From 1ca4225edef1863d947306fb4c52d8dda84c4740 Mon Sep 17 00:00:00 2001 From: JoJo Date: Sun, 12 Sep 2021 23:34:41 +0200 Subject: [PATCH] Add command AudioVis to visualise playback audio spectrum Implementation based on heliecho, w some changes & fixes. Still needs tweaking, I'd say. --- server/Cargo.lock | 150 ++++++++++++++++++++++++++++ server/Cargo.toml | 3 + server/src/audiovis.rs | 210 +++++++++++++++++++++++++++++++++++++++ server/src/main.rs | 56 +++++++---- server/static/index.html | 1 + server/static/main.js | 3 +- 6 files changed, 401 insertions(+), 22 deletions(-) create mode 100644 server/src/audiovis.rs diff --git a/server/Cargo.lock b/server/Cargo.lock index 8e3a2f7..65bef27 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -875,7 +875,10 @@ version = "0.1.0" dependencies = [ "async-std", "byteorder", + "num", "palette", + "pulse-simple", + "rustfft", "serde", "tide", ] @@ -910,6 +913,29 @@ version = "0.2.101" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3cb00336871be5ed2c8ed44b60ae9959dc5b9f08539422ed43f09e34ecaeba21" +[[package]] +name = "libpulse-simple-sys" +version = "1.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b8b0fcb9665401cc7c156c337c8edc7eb4e797b9d3ae1667e1e9e17b29e0c7c" +dependencies = [ + "libpulse-sys", + "pkg-config", +] + +[[package]] +name = "libpulse-sys" +version = "1.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f12950b69c1b66233a900414befde36c8d4ea49deec1e1f34e4cd2f586e00c7d" +dependencies = [ + "libc", + "num-derive", + "num-traits", + "pkg-config", + "winapi", +] + [[package]] name = "log" version = "0.4.14" @@ -932,6 +958,51 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +[[package]] +name = "num" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43db66d1170d347f9a065114077f7dccb00c1b9478c89384490a3425279a4606" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74e768dff5fb39a41b3bcd30bb25cf989706c90d028d1ad71971987aa309d535" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26873667bbbb7c5182d4a37c1add32cdf09f841af72da53318fdb81543c15085" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-derive" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "num-integer" version = "0.1.44" @@ -942,6 +1013,29 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2021c8337a54d21aca0d59a92577a029af9431cb59b909b03252b9c164fad59" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d41702bd167c2df5520b384281bc111a4b5efcf7fbc4c9c222c815b07e0a6a6a" +dependencies = [ + "autocfg", + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.14" @@ -1091,6 +1185,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + [[package]] name = "polling" version = "2.1.0" @@ -1121,6 +1221,15 @@ version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" +[[package]] +name = "primal-check" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01419cee72c1a1ca944554e23d83e483e1bccf378753344e881de28b5487511d" +dependencies = [ + "num-integer", +] + [[package]] name = "proc-macro-hack" version = "0.5.19" @@ -1142,6 +1251,17 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "pulse-simple" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "846332afcb040c3c781ff4e4c3b1e3952e8ddcb6f2b3be46125981d233e02d05" +dependencies = [ + "libc", + "libpulse-simple-sys", + "libpulse-sys", +] + [[package]] name = "quote" version = "1.0.9" @@ -1247,6 +1367,20 @@ dependencies = [ "semver", ] +[[package]] +name = "rustfft" +version = "6.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1d089e5c57521629a59f5f39bca7434849ff89bd6873b521afe389c1c602543" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", +] + [[package]] name = "ryu" version = "1.0.5" @@ -1455,6 +1589,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" +[[package]] +name = "strength_reduce" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3ff2f71c82567c565ba4b3009a9350a96a7269eaa4001ebedae926230bc2254" + [[package]] name = "subtle" version = "2.4.1" @@ -1594,6 +1734,16 @@ dependencies = [ "serde", ] +[[package]] +name = "transpose" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95f9c900aa98b6ea43aee227fd680550cdec726526aab8ac801549eadb25e39f" +dependencies = [ + "num-integer", + "strength_reduce", +] + [[package]] name = "typenum" version = "1.14.0" diff --git a/server/Cargo.toml b/server/Cargo.toml index c390092..9bbf435 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -11,3 +11,6 @@ tide = "0.16" async-std = { version = "1.10", features = ["attributes"] } serde = { version = "1.0", features = ["derive"] } palette = "0.6" +rustfft = "6.0" +pulse-simple = "1.0" +num = "0.4" diff --git a/server/src/audiovis.rs b/server/src/audiovis.rs new file mode 100644 index 0000000..4b02f47 --- /dev/null +++ b/server/src/audiovis.rs @@ -0,0 +1,210 @@ +use num::Complex; +use palette::{Blend, FromColor, Hsv, Mix, Saturate, Srgb}; +use pulse_simple::Record; +use rustfft::FftPlanner; +use std::{ + sync::mpsc::{channel, Receiver}, + thread, +}; + +const SAMPLE_RATE: usize = 48000; +// Must be power of 2 +const SAMPLES_PER_PERIOD: usize = 256; + +const BASS_CUTOFF: f32 = 400.0; +const HIGH_CUTOFF: f32 = 2900.0; + +const MAX_FREQ: f32 = 20_000.0; + +pub struct AudioVis { + rx: Receiver, + color: Hsv, +} + +impl AudioVis { + pub fn new() -> Self { + let (tx, rx) = channel(); + thread::spawn(move || { + let recorder = Record::new( + "jojos-hue", + "Capture audio to stream as color data to jojos-hue devices", + None, + SAMPLE_RATE as u32, + ); + let mut stereo_data = [[0.0f32; 2]; SAMPLES_PER_PERIOD]; + let (bass_cutoff_bin, high_cutoff_bin) = + (freq_to_bin(BASS_CUTOFF), freq_to_bin(HIGH_CUTOFF)); + let mut i = 0u16; + loop { + recorder.read(&mut stereo_data); + let bin_amps = stereo_pcm_to_bins(&stereo_data); + let mut bass_amps = [0.0; SAMPLES_PER_PERIOD >> 1]; + for (bin, &) in bin_amps.iter().enumerate() { + bass_amps[bin] = bass_pass(bin_to_freq(bin), amp); + } + let mut mid_amps = [0.0; SAMPLES_PER_PERIOD >> 1]; + for (bin, &) in bin_amps.iter().enumerate() { + mid_amps[bin] = mid_pass(bin_to_freq(bin), amp); + } + let mut high_amps = [0.0; SAMPLES_PER_PERIOD >> 1]; + for (bin, &) in bin_amps.iter().enumerate() { + high_amps[bin] = high_pass(bin_to_freq(bin), amp); + } + let max_amp_bass = max_amp(&bass_amps[..bass_cutoff_bin]).1; + let m_m = max_amp(&mid_amps[bass_cutoff_bin..high_cutoff_bin]); + let m_h = max_amp(&high_amps[high_cutoff_bin..]); + let max_amp_mid = (m_m.0 + bass_cutoff_bin, m_m.1).1; + let max_amp_high = (m_h.0 + high_cutoff_bin, m_h.1).1; + let (bass_lvl, mid_lvl, high_lvl) = ( + norm_amp(max_amp_bass).powf(1.22), // .powf(2.5), + norm_amp(max_amp_mid * 1.6).powf(1.25), // .powf(2.6), + norm_amp(max_amp_high * 3.2).powf(1.12), // .powf(2.4), + ); + //let brightness = (bass_lvl * 1.6 + mid_lvl * 0.7 + high_lvl * 0.7) / 3.0; + let color = Hsv::from_color(Srgb::new(bass_lvl, mid_lvl, high_lvl)).saturate(0.3); + //let c1 = color.clone(); + //color.value = brightness; + + i += 1; + if i > 40 { + i = 0; + let (max_bin_all, max_amp_all) = max_amp(&bin_amps); + println!( + "f: {:6.0}, amp: {:4.1}, vol: {:1.3}\namp: bass: {}, mid: {}, high: {}\nlvl: bass: {}, mid: {}, high: {}\nc2: {:?}\n", + bin_to_freq(max_bin_all), + max_amp_all, + norm_amp(max_amp_all), + max_amp_bass, + max_amp_mid, + max_amp_high, + bass_lvl, + mid_lvl, + high_lvl, + //c1, + color + ); + } + + tx.send(color).unwrap(); + } + }); + Self { + rx, + color: Hsv::new(0.0, 0.0, 0.0), + } + } + pub fn color(&mut self) -> Hsv { + if let Some(c2) = self.rx.try_iter().last() { + let c1 = self.color; + let mut c3 = c1.mix(&c2, 0.6); + let r = 0.2 + 0.8 * c2.value; // become bright fast, and dark more slowly + c3.value = (1.0 - r) * c1.value + r * c2.value; + self.color = c3; + // let c1 = Srgb::from_color(self.color).into_linear(); + // let c2 = Srgb::from_color(c2).into_linear(); + // let c3 = c1.dodge(c2).mix(&c2, 0.6); + // self.color = Hsv::from_color(Srgb::from_linear(c3)).saturate(0.15); + } + self.color + } +} +fn bin_to_freq(i: usize) -> f32 { + (i * SAMPLE_RATE) as f32 / SAMPLES_PER_PERIOD as f32 +} + +fn freq_to_bin(f: f32) -> usize { + (f * SAMPLES_PER_PERIOD as f32 / SAMPLE_RATE as f32 + 0.5) as usize +} + +fn pass_to(freq: f32, amp: f32, cut: f32) -> f32 { + assert!(freq >= 0.0); + if freq > cut { + 0.0 + } else { + let x = freq / cut; + let sharpness = (cut / 400.0).powf(2.0); + amp * f32::max(1.0 - (3.0 * sharpness).powf(7.0 * (x - 1.0)), 0.0) + } +} + +fn band_pass(freq: f32, amp: f32, low_cut: f32, high_cut: f32) -> f32 { + pass_from(freq, pass_to(freq, amp, high_cut), low_cut) +} + +fn pass_from(freq: f32, amp: f32, cut: f32) -> f32 { + assert!(freq >= 0.0 && cut < MAX_FREQ); + if freq < cut { + 0.0 + } else { + let x = (freq - cut) / (MAX_FREQ - cut); + let sharpness = ((MAX_FREQ - cut) / 2700.0).powf(10.0); + amp * f32::max(1.0 - (3.0 * sharpness).powf(-7.0 * x), 0.0) + } +} + +fn bass_pass(freq: f32, amp: f32) -> f32 { + pass_to(freq, amp, BASS_CUTOFF) +} + +fn mid_pass(freq: f32, amp: f32) -> f32 { + band_pass(freq, amp, BASS_CUTOFF - 170.0, HIGH_CUTOFF + 300.0) +} + +fn high_pass(freq: f32, amp: f32) -> f32 { + pass_from(freq, amp, HIGH_CUTOFF) +} + +/// Normalize an amplitude to [0, 1] +fn norm_amp(x: f32) -> f32 { + (x / (x.powi(2) + 8.0).sqrt()).powf(1.8) + // db / (db + 1.0) + // let x = db / 52.0; + // if x > 1.0 { + // 1.0 + // } else { + // (1.0 - x) * x + x * (1.0 / (1.0 + (-10.0 * (x - 0.5)).exp())) + // } +} + +/// bin == index of fft where there is a corresponding frequence +fn stereo_pcm_to_bins( + stereo_data: &[[f32; 2]; SAMPLES_PER_PERIOD], +) -> [f32; SAMPLES_PER_PERIOD >> 1] { + let complex_zero = Complex { + re: 0.0f32, + im: 0.0f32, + }; + let mut avg_data = [complex_zero; SAMPLES_PER_PERIOD]; + for (i, &[l, r]) in stereo_data.iter().enumerate() { + avg_data[i] = Complex { + re: (l + r) / 2.0, + im: 0.0, + }; + } + let mut planner = FftPlanner::new(); + let fft = planner.plan_fft_forward(SAMPLES_PER_PERIOD); + fft.process(&mut avg_data as &mut [_]); + let bin_amps_complex = avg_data; + let mut bin_amps = [0.0f32; SAMPLES_PER_PERIOD >> 1]; + for (i, &c) in bin_amps_complex + .iter() + .take(SAMPLES_PER_PERIOD >> 1) + .enumerate() + { + let amp = (c.re.powi(2) + c.im.powi(2)).sqrt(); + //bin_amps[i] = 20.0 * amp.log(10.0); + bin_amps[i] = amp; + } + bin_amps +} + +fn max_amp(amps: &[f32]) -> (usize, f32) { + let (mut max_bin, mut max_amp) = (0, 0.0); + for i in 0..amps.len() { + if amps[i] > max_amp { + max_amp = amps[i]; + max_bin = i; + } + } + (max_bin, max_amp) +} diff --git a/server/src/main.rs b/server/src/main.rs index 6a91042..804cae1 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -1,3 +1,4 @@ +use audiovis::AudioVis; use byteorder::{ByteOrder, LittleEndian}; use palette::{FromColor, Hsv, Hue, RgbHue, Shade, Srgb}; use std::{ @@ -14,6 +15,8 @@ use std::{ }; use tide::{prelude::*, Request}; +mod audiovis; + const FPS: f32 = 100.0; const DEADLINE: Duration = Duration::from_secs(10); @@ -32,14 +35,36 @@ enum QueryCmd { Off, Cycle, Breathing, + AudioVis, } -#[derive(Clone, Copy)] enum Cmd { All, Off, Cycle, Breathing, + AudioVis(AudioVis), +} + +impl Cmd { + fn render_cmd(&mut self, base: Hsv, t: f64) -> RenderCmd { + match self { + Cmd::All => RenderCmd::All(Srgb::from_color(base)), + Cmd::Off => RenderCmd::All(Srgb::from_color(base.darken(t.min(1.0) as f32))), + Cmd::Cycle => RenderCmd::All(Srgb::from_color( + base.shift_hue(hue_f64_to_f32(RgbHue::from_radians(t / 6.0))), + )), + Cmd::Breathing => RenderCmd::All(Srgb::from_color( + base.darken(0.9 * (t / 2.0).sin().powi(2) as f32), + )), + Cmd::AudioVis(av) => RenderCmd::All(Srgb::from_color(av.color())), + } + } +} + +#[derive(Clone, Copy)] +enum RenderCmd { + All(Srgb), } #[async_std::main] @@ -92,6 +117,7 @@ async fn proc_handler(req: Request<()>, tx: SyncSender) -> tide::Resul "off" => tx.send(QueryCmd::Off).unwrap(), "cycle" => tx.send(QueryCmd::Cycle).unwrap(), "breathing" => tx.send(QueryCmd::Breathing).unwrap(), + "audiovis" => tx.send(QueryCmd::AudioVis).unwrap(), _ => {} } Ok("cool, ty".into()) @@ -109,6 +135,7 @@ fn controller(heartbeat_rx: Receiver, cmd_rx: Receiver) { match clients.get_mut(&addr) { Some(client) => client.0 = Instant::now(), None => { + println!("Registered client {} with {} LEDs", addr, nleds); clients.insert( addr, (Instant::now(), setup_client(addr, nleds, sender.clone())), @@ -134,11 +161,12 @@ fn controller(heartbeat_rx: Receiver, cmd_rx: Receiver) { QueryCmd::Off => cmd = Cmd::Off, QueryCmd::Cycle => cmd = Cmd::Cycle, QueryCmd::Breathing => cmd = Cmd::Breathing, + QueryCmd::AudioVis => cmd = Cmd::AudioVis(AudioVis::new()), } } for (_, tx) in clients.values() { let t = tcmd.elapsed().as_secs_f64(); - tx.send((cmd, base, t)).ok(); + tx.send(cmd.render_cmd(base, t)).ok(); } let dt = tprev.elapsed(); tprev = Instant::now(); @@ -149,14 +177,14 @@ fn controller(heartbeat_rx: Receiver, cmd_rx: Receiver) { } } -fn setup_client(addr: SocketAddr, nleds: u16, sender: Arc) -> Sender<(Cmd, Hsv, f64)> { +fn setup_client(addr: SocketAddr, nleds: u16, sender: Arc) -> Sender { let (tx, rx) = channel(); thread::spawn(move || loop { let magic_cookie = b"satan"; - let (cmd, base, t) = rx + let cmd = rx .recv() .expect("Client channel closed. Time for this thread to die!"); - let color_data = lightf(cmd, base, t, nleds) + let color_data = lightf(cmd, nleds) .into_iter() .flat_map(|c| once(c.red).chain(once(c.green)).chain(once(c.blue))) .map(|x| (x * 256.0).min(256.0).max(0.0) as u8); @@ -173,29 +201,15 @@ fn setup_client(addr: SocketAddr, nleds: u16, sender: Arc) -> Sender< fn parse_heartbeat(msg: &[u8; 13], addr: SocketAddr) -> Option { if msg.starts_with(b"heart<") && msg.ends_with(b">beat") { let nleds = LittleEndian::read_u16(&msg[6..8]); - println!("Heartbeat from {} with {} LEDs", addr, nleds); Some((addr, nleds)) } else { None } } -fn lightf(cmd: Cmd, base: Hsv, t: f64, nleds: u16) -> Vec { +fn lightf(cmd: RenderCmd, nleds: u16) -> Vec { match cmd { - Cmd::All => vec![Srgb::from_color(base); nleds as usize], - Cmd::Off => vec![Srgb::from_color(base.darken(t.min(1.0) as f32)); nleds as usize], - Cmd::Cycle => { - vec![ - Srgb::from_color(base.shift_hue(hue_f64_to_f32(RgbHue::from_radians(t / 6.0)))); - nleds as usize - ] - } - Cmd::Breathing => { - vec![ - Srgb::from_color(base.darken(0.9 * (t / 2.0).sin().powi(2) as f32)); - nleds as usize - ] - } // Cmd::AudioVis => {} + RenderCmd::All(color) => vec![color; nleds as usize], } } diff --git a/server/static/index.html b/server/static/index.html index 1fe6011..16afbc0 100644 --- a/server/static/index.html +++ b/server/static/index.html @@ -13,6 +13,7 @@ + diff --git a/server/static/main.js b/server/static/main.js index b72faff..67ab54d 100644 --- a/server/static/main.js +++ b/server/static/main.js @@ -31,7 +31,8 @@ var colorWheel = new ReinventedColorWheel({ let nullaryButtons = [["off-button", "off"], ["cycle-button", "cycle"], - ["breathing-button", "breathing"]]; + ["breathing-button", "breathing"], + ["audiovis-button", "audiovis"]]; nullaryButtons.forEach(([btn, proc]) => { document.getElementById(btn).onclick = () => { -- 2.45.2