wip
This commit is contained in:
Matthew Deville
parent
55a94f0a75
commit
8389e88a80
7 changed files with 250 additions and 58 deletions
69
Cargo.lock
generated
69
Cargo.lock
generated
|
|
@ -275,6 +275,17 @@ dependencies = [
|
|||
"portable-atomic-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "audioshader"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"bevy",
|
||||
"cpal 0.16.0",
|
||||
"crossbeam-channel",
|
||||
"rodio 0.21.1",
|
||||
"rustfft",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.5.0"
|
||||
|
|
@ -1134,16 +1145,6 @@ dependencies = [
|
|||
"wasm-bindgen-futures",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bevy_test"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"bevy",
|
||||
"cpal 0.16.0",
|
||||
"crossbeam-channel",
|
||||
"rodio 0.21.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bevy_text"
|
||||
version = "0.16.1"
|
||||
|
|
@ -3031,6 +3032,15 @@ dependencies = [
|
|||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-complex"
|
||||
version = "0.4.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-derive"
|
||||
version = "0.4.2"
|
||||
|
|
@ -3610,6 +3620,15 @@ dependencies = [
|
|||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "primal-check"
|
||||
version = "0.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08"
|
||||
dependencies = [
|
||||
"num-integer",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro-crate"
|
||||
version = "3.3.0"
|
||||
|
|
@ -3834,6 +3853,20 @@ version = "2.1.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
|
||||
|
||||
[[package]]
|
||||
name = "rustfft"
|
||||
version = "6.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c6f140db74548f7c9d7cce60912c9ac414e74df5e718dc947d514b051b42f3f4"
|
||||
dependencies = [
|
||||
"num-complex",
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
"primal-check",
|
||||
"strength_reduce",
|
||||
"transpose",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "0.38.44"
|
||||
|
|
@ -4054,6 +4087,12 @@ version = "1.1.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
|
||||
|
||||
[[package]]
|
||||
name = "strength_reduce"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82"
|
||||
|
||||
[[package]]
|
||||
name = "strum"
|
||||
version = "0.26.3"
|
||||
|
|
@ -4462,6 +4501,16 @@ dependencies = [
|
|||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "transpose"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e"
|
||||
dependencies = [
|
||||
"num-integer",
|
||||
"strength_reduce",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ttf-parser"
|
||||
version = "0.20.0"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[package]
|
||||
edition = "2024"
|
||||
name = "bevy_test"
|
||||
name = "audioshader"
|
||||
version = "0.1.0"
|
||||
|
||||
|
||||
|
|
@ -9,6 +9,7 @@
|
|||
cpal = "0.16"
|
||||
crossbeam-channel = "0.5"
|
||||
rodio = "0.21"
|
||||
rustfft = "6"
|
||||
|
||||
# Enable a small amount of optimization in the dev profile.
|
||||
[profile.dev]
|
||||
|
|
|
|||
|
|
@ -3,46 +3,105 @@
|
|||
@group(2) @binding(0) var<uniform> u_time: f32;
|
||||
@group(2) @binding(1) var<uniform> u_resolution: vec2f;
|
||||
@group(2) @binding(2) var<uniform> u_mouse_position: vec2f;
|
||||
@group(2) @binding(3) var fft_texture: texture_2d<f32>;
|
||||
@group(2) @binding(4) var fft_texture_sampler: sampler;
|
||||
|
||||
|
||||
fn lerp(v0: f32, v1: f32, t: f32) -> f32 {
|
||||
return v0 + t * (v1 - v0);
|
||||
}
|
||||
|
||||
// Utility: clamp to 0..1 range
|
||||
fn saturate(x: f32) -> f32 { return clamp(x, 0.0, 1.0); }
|
||||
|
||||
// Cosine palette (like IQ's)
|
||||
fn palette(t: f32, a: vec3f, b: vec3f, c: vec3f, d: vec3f) -> vec3f {
|
||||
return a + b * cos(6.2831853 * (c * t + d));
|
||||
}
|
||||
|
||||
// Sample the FFT texture at normalized x in [0,1]
|
||||
fn sample_fft(x: f32) -> f32 {
|
||||
// 5-tap Gaussian blur in texture space using actual texel width
|
||||
let size = textureDimensions(fft_texture);
|
||||
let tex_w = max(f32(size.x), 1.0);
|
||||
let one_texel = 1.0 / tex_w;
|
||||
let u = saturate(x);
|
||||
let v = 0.5;
|
||||
let w0 = 0.0625; // gaussian-ish weights: 1,4,6,4,1 normalized
|
||||
let w1 = 0.25;
|
||||
let w2 = 0.375;
|
||||
let s0 = textureSample(fft_texture, fft_texture_sampler, vec2f(u - 2.0 * one_texel, v)).r;
|
||||
let s1 = textureSample(fft_texture, fft_texture_sampler, vec2f(u - 1.0 * one_texel, v)).r;
|
||||
let s2 = textureSample(fft_texture, fft_texture_sampler, vec2f(u, v)).r;
|
||||
let s3 = textureSample(fft_texture, fft_texture_sampler, vec2f(u + 1.0 * one_texel, v)).r;
|
||||
let s4 = textureSample(fft_texture, fft_texture_sampler, vec2f(u + 2.0 * one_texel, v)).r;
|
||||
return w0 * s0 + w1 * s1 + w2 * s2 + w1 * s3 + w0 * s4;
|
||||
}
|
||||
|
||||
@fragment
|
||||
fn fragment(mesh: VertexOutput) -> @location(0) vec4f {
|
||||
var uv = vec2f(lerp(-2, 0.47, mesh.uv.x), lerp(-1.12, 1.12, mesh.uv.y));
|
||||
// Normalize UV to -1..1 with aspect correction
|
||||
let aspect = u_resolution.x / max(u_resolution.y, 1.0);
|
||||
let p = (mesh.uv * 2.0 - vec2f(1.0, 1.0)) * vec2f(aspect, 1.0);
|
||||
|
||||
// Progressive time-based zoom toward the known interesting point
|
||||
// c = -0.743643887037151 + 0.13182590420533i
|
||||
let center = vec2f(-0.743643887037151, 0.13182590420533);
|
||||
let zoomSpeed: f32 = 0.2; // tune this to control how fast we zoom
|
||||
let zoom = exp(u_time * zoomSpeed); // zoom factor grows with time
|
||||
let c = center + (uv - center) / zoom; // move coordinates toward center as zoom increases
|
||||
// Polar coords
|
||||
let r = length(p);
|
||||
let ang = atan2(p.y, p.x); // [-pi, pi]
|
||||
var a01 = ang / 6.2831853 + 0.5; // [0,1]
|
||||
|
||||
var zx = 0.0;
|
||||
var zy = 0.0;
|
||||
let maxIter: i32 = 100 + i32(u_time);
|
||||
var it: i32 = 0;
|
||||
// Scroll the angle slowly for motion
|
||||
a01 = fract(a01 + 0.05 * u_time);
|
||||
|
||||
for (var i: i32 = 0; i < maxIter; i = i + 1) {
|
||||
let x = zx * zx - zy * zy + c.x;
|
||||
let y = 2.0 * zx * zy + c.y;
|
||||
zx = x;
|
||||
zy = y;
|
||||
if (zx * zx + zy * zy > 4.0) {
|
||||
it = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Log-like frequency remap for better distribution of low freqs
|
||||
let freq = pow(a01, 0.8);
|
||||
|
||||
let t = f32(it) / f32(maxIter);
|
||||
// Smooth FFT sample with a few taps
|
||||
let amp0 = sample_fft(freq);
|
||||
let amp1 = sample_fft(saturate(freq + 1.0 / 512.0));
|
||||
let amp2 = sample_fft(saturate(freq - 1.0 / 512.0));
|
||||
var amp = (amp0 + amp1 + amp2) / 3.0;
|
||||
// Optional shaping
|
||||
amp = pow(amp, 0.6);
|
||||
|
||||
let palette = vec3f(0.5) + 0.5 * cos(6.2831853 * (vec3f(0.0, 0.33, 0.66) + t * 1.5) + u_time * 0.2);
|
||||
// Target ring radius driven by amplitude
|
||||
let base_r = 0.35;
|
||||
let ring_r = base_r + amp * 0.45;
|
||||
|
||||
var color = vec3f(0, 0, 0);
|
||||
if (t < 0.999999) {
|
||||
color = palette;
|
||||
}
|
||||
|
||||
return vec4f(color, 1);
|
||||
// Ring intensity with derivative-based anti-aliasing
|
||||
let band_w = mix(0.02, 0.06, 1.0 - amp);
|
||||
let aa = fwidth(r);
|
||||
let dist = abs(r - ring_r);
|
||||
let ring = 1.0 - smoothstep(band_w - aa, band_w + aa, dist);
|
||||
|
||||
// Radial spokes modulated by FFT at harmonics
|
||||
let spokes = 0.5 + 0.5 * cos(ang * (8.0 + 24.0 * amp) + u_time * (1.0 + 2.0 * amp));
|
||||
let spokes_mask = pow(spokes, 3.0);
|
||||
|
||||
// Glow falloff
|
||||
let glow = 0.015 / (r * r + 0.03);
|
||||
|
||||
// Background subtle gradient with slow drift
|
||||
let bg = 0.08 + 0.06 * cos(6.28318 * (p.x * 0.15 + p.y * 0.12 + 0.03 * u_time));
|
||||
|
||||
// Color palette over angle
|
||||
let col = palette(
|
||||
a01,
|
||||
vec3f(0.35, 0.3, 0.35), // a
|
||||
vec3f(0.35, 0.35, 0.35), // b
|
||||
vec3f(1.0, 1.0, 1.0), // c
|
||||
vec3f(0.0, 0.33, 0.67) // d
|
||||
);
|
||||
|
||||
// Combine layers
|
||||
var color = vec3f(bg) * 0.6;
|
||||
color += col * ring * (0.6 + 0.6 * spokes_mask);
|
||||
color += vec3f(0.9, 0.95, 1.0) * glow * (0.3 + 1.5 * amp);
|
||||
|
||||
// Subtle vignette
|
||||
let vig = smoothstep(1.2, 0.3, r);
|
||||
color *= vig;
|
||||
|
||||
// Final boost
|
||||
color = clamp(color, vec3f(0.0), vec3f(1.0));
|
||||
return vec4f(color, 1.0);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,8 +5,28 @@ use {
|
|||
traits::{DeviceTrait, HostTrait, StreamTrait},
|
||||
},
|
||||
crossbeam_channel::{Receiver, Sender},
|
||||
rustfft::{Fft, FftPlanner, num_complex::Complex},
|
||||
std::sync::Arc,
|
||||
};
|
||||
|
||||
/// Resource holding the latest FFT result (magnitude spectrum)
|
||||
#[derive(Resource, Clone, Debug)]
|
||||
pub struct AudioFft {
|
||||
pub spectrum: Vec<f32>,
|
||||
pub sample_rate: u32,
|
||||
pub channels: u16,
|
||||
}
|
||||
|
||||
impl Default for AudioFft {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
spectrum: vec![0.0; 1024],
|
||||
sample_rate: 44100,
|
||||
channels: 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Event carrying interleaved mic samples as f32 in the device's native
|
||||
/// sample rate and channel layout.
|
||||
#[derive(Event, Debug, Clone)]
|
||||
|
|
@ -16,19 +36,6 @@ pub struct AudioCaptureData {
|
|||
pub channels: u16,
|
||||
}
|
||||
|
||||
pub fn handle_audio_capture_events(mut reader: EventReader<AudioCaptureData>) {
|
||||
for event in reader.read() {
|
||||
// Process mic data here.
|
||||
// For example, print the number of samples received.
|
||||
println!(
|
||||
"Received mic data: {} samples at {} Hz, {} channels",
|
||||
event.samples.len(),
|
||||
event.sample_rate,
|
||||
event.channels
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Holds the CPAL stream alive and a channel for transferring audio to Bevy world.
|
||||
#[derive(Resource)]
|
||||
struct AudioCaptureStream {
|
||||
|
|
@ -44,6 +51,7 @@ pub struct AudioCapturePlugin;
|
|||
impl Plugin for AudioCapturePlugin {
|
||||
fn build(&self, app: &mut App) {
|
||||
app.add_event::<AudioCaptureData>()
|
||||
.init_resource::<AudioFft>()
|
||||
.add_systems(Startup, init_audio_input_stream)
|
||||
.add_systems(Update, dispatch_audio_events);
|
||||
}
|
||||
|
|
@ -77,7 +85,7 @@ fn init_audio_input_stream(mut commands: Commands) {
|
|||
stream.play().expect("Failed to play input stream");
|
||||
|
||||
info!(
|
||||
"Mic stream started: {} ch @ {} Hz (default OS input)",
|
||||
"Audio capture stream started: {} ch @ {} Hz (default OS input)",
|
||||
channels, sample_rate
|
||||
);
|
||||
|
||||
|
|
@ -124,13 +132,54 @@ where
|
|||
)
|
||||
}
|
||||
|
||||
fn dispatch_audio_events(mic: Res<AudioCaptureStream>, mut writer: EventWriter<AudioCaptureData>) {
|
||||
fn dispatch_audio_events(
|
||||
mic: Res<AudioCaptureStream>,
|
||||
mut writer: EventWriter<AudioCaptureData>,
|
||||
mut fft_res: ResMut<AudioFft>,
|
||||
) {
|
||||
// FFT parameters
|
||||
const FFT_SIZE: usize = 1024;
|
||||
// Drain any available audio buffers without blocking the frame.
|
||||
while let Ok(samples) = mic.rx.try_recv() {
|
||||
writer.write(AudioCaptureData {
|
||||
samples,
|
||||
samples: samples.clone(),
|
||||
sample_rate: mic.sample_rate,
|
||||
channels: mic.channels,
|
||||
});
|
||||
|
||||
// Compute FFT on the first channel (mono mix)
|
||||
let mut mono: Vec<f32> = if mic.channels > 1 {
|
||||
samples
|
||||
.chunks(mic.channels as usize)
|
||||
.map(|frame| frame[0])
|
||||
.collect()
|
||||
} else {
|
||||
samples.clone()
|
||||
};
|
||||
// Pad or truncate to FFT_SIZE
|
||||
if mono.len() < FFT_SIZE {
|
||||
mono.resize(FFT_SIZE, 0.0);
|
||||
} else if mono.len() > FFT_SIZE {
|
||||
mono.truncate(FFT_SIZE);
|
||||
}
|
||||
|
||||
// Prepare input for FFT
|
||||
let mut buffer: Vec<Complex<f32>> = mono
|
||||
.into_iter()
|
||||
.map(|x| Complex { re: x, im: 0.0 })
|
||||
.collect();
|
||||
|
||||
// Compute FFT
|
||||
let mut planner = FftPlanner::<f32>::new();
|
||||
let fft: Arc<dyn Fft<f32>> = planner.plan_fft_forward(FFT_SIZE);
|
||||
fft.process(&mut buffer);
|
||||
|
||||
// Compute magnitude spectrum
|
||||
let spectrum: Vec<f32> = buffer.iter().map(|c| c.norm()).collect();
|
||||
|
||||
// Update AudioFft resource
|
||||
fft_res.spectrum = spectrum;
|
||||
fft_res.sample_rate = mic.sample_rate;
|
||||
fft_res.channels = mic.channels;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ fn main() {
|
|||
Update,
|
||||
(
|
||||
material::update_material_time,
|
||||
audio_input::handle_audio_capture_events,
|
||||
material::update_material_audio_fft,
|
||||
systems::screen_resized,
|
||||
systems::mouse_moved,
|
||||
systems::exit_app.run_if(input_just_pressed(KeyCode::Escape)),
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
use bevy::{
|
||||
asset::Assets,
|
||||
prelude::*,
|
||||
reflect::TypePath,
|
||||
render::render_resource::{AsBindGroup, ShaderRef},
|
||||
render::render_resource::{AsBindGroup, Extent3d, ShaderRef},
|
||||
sprite::{AlphaMode2d, Material2d},
|
||||
};
|
||||
|
||||
|
|
@ -15,6 +16,9 @@ pub struct CustomMaterial {
|
|||
pub resolution: Vec2,
|
||||
#[uniform(2)]
|
||||
pub mouse_position: Vec2,
|
||||
#[texture(3)]
|
||||
#[sampler(4)]
|
||||
pub audio_fft_tex: Handle<Image>,
|
||||
}
|
||||
|
||||
pub fn update_material_time(mut materials: ResMut<Assets<CustomMaterial>>, time: Res<Time>) {
|
||||
|
|
@ -24,6 +28,35 @@ pub fn update_material_time(mut materials: ResMut<Assets<CustomMaterial>>, time:
|
|||
}
|
||||
}
|
||||
|
||||
pub fn update_material_audio_fft(
|
||||
mut materials: ResMut<Assets<CustomMaterial>>,
|
||||
mut images: ResMut<Assets<Image>>,
|
||||
audio_fft: Res<crate::audio_input::AudioFft>,
|
||||
) {
|
||||
let spectrum = &audio_fft.spectrum;
|
||||
if spectrum.is_empty() {
|
||||
return;
|
||||
}
|
||||
let image = Image::new(
|
||||
Extent3d {
|
||||
width: spectrum.len() as u32,
|
||||
height: 1,
|
||||
depth_or_array_layers: 1,
|
||||
},
|
||||
bevy::render::render_resource::TextureDimension::D2,
|
||||
spectrum
|
||||
.iter()
|
||||
.map(|&v| (v * 255.0).clamp(0.0, 255.0) as u8)
|
||||
.collect(),
|
||||
bevy::render::render_resource::TextureFormat::R8Unorm,
|
||||
bevy::asset::RenderAssetUsages::RENDER_WORLD,
|
||||
);
|
||||
let handle = images.add(image);
|
||||
for (_handle, material) in materials.iter_mut() {
|
||||
material.audio_fft_tex = handle.clone();
|
||||
}
|
||||
}
|
||||
|
||||
impl Material2d for CustomMaterial {
|
||||
fn fragment_shader() -> ShaderRef {
|
||||
DEFAULT_SHADER_PATH.into()
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ pub fn setup(
|
|||
time: 0.0,
|
||||
resolution: screen_size,
|
||||
mouse_position: Vec2::ZERO,
|
||||
audio_fft_tex: Default::default(),
|
||||
});
|
||||
|
||||
commands.spawn((
|
||||
|
|
|
|||
Loading…
Reference in a new issue