~jojo/jojos-hue

1ca4225edef1863d947306fb4c52d8dda84c4740 — JoJo a month ago cf43207
Add command AudioVis to visualise playback audio spectrum

Implementation based on heliecho, w some changes & fixes. Still needs
tweaking, I'd say.
M server/Cargo.lock => server/Cargo.lock +150 -0
@@ 875,7 875,10 @@ version = "0.1.0"
dependencies = [
 "async-std",
 "byteorder",
 "num",
 "palette",
 "pulse-simple",
 "rustfft",
 "serde",
 "tide",
]


@@ 911,6 914,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3cb00336871be5ed2c8ed44b60ae9959dc5b9f08539422ed43f09e34ecaeba21"

[[package]]
name = "libpulse-simple-sys"
version = "1.19.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b8b0fcb9665401cc7c156c337c8edc7eb4e797b9d3ae1667e1e9e17b29e0c7c"
dependencies = [
 "libpulse-sys",
 "pkg-config",
]

[[package]]
name = "libpulse-sys"
version = "1.19.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f12950b69c1b66233a900414befde36c8d4ea49deec1e1f34e4cd2f586e00c7d"
dependencies = [
 "libc",
 "num-derive",
 "num-traits",
 "pkg-config",
 "winapi",
]

[[package]]
name = "log"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"


@@ 933,6 959,51 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"

[[package]]
name = "num"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43db66d1170d347f9a065114077f7dccb00c1b9478c89384490a3425279a4606"
dependencies = [
 "num-bigint",
 "num-complex",
 "num-integer",
 "num-iter",
 "num-rational",
 "num-traits",
]

[[package]]
name = "num-bigint"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "74e768dff5fb39a41b3bcd30bb25cf989706c90d028d1ad71971987aa309d535"
dependencies = [
 "autocfg",
 "num-integer",
 "num-traits",
]

[[package]]
name = "num-complex"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26873667bbbb7c5182d4a37c1add32cdf09f841af72da53318fdb81543c15085"
dependencies = [
 "num-traits",
]

[[package]]
name = "num-derive"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d"
dependencies = [
 "proc-macro2",
 "quote",
 "syn",
]

[[package]]
name = "num-integer"
version = "0.1.44"
source = "registry+https://github.com/rust-lang/crates.io-index"


@@ 943,6 1014,29 @@ dependencies = [
]

[[package]]
name = "num-iter"
version = "0.1.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2021c8337a54d21aca0d59a92577a029af9431cb59b909b03252b9c164fad59"
dependencies = [
 "autocfg",
 "num-integer",
 "num-traits",
]

[[package]]
name = "num-rational"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d41702bd167c2df5520b384281bc111a4b5efcf7fbc4c9c222c815b07e0a6a6a"
dependencies = [
 "autocfg",
 "num-bigint",
 "num-integer",
 "num-traits",
]

[[package]]
name = "num-traits"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"


@@ 1092,6 1186,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"

[[package]]
name = "pkg-config"
version = "0.3.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c"

[[package]]
name = "polling"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"


@@ 1122,6 1222,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"

[[package]]
name = "primal-check"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01419cee72c1a1ca944554e23d83e483e1bccf378753344e881de28b5487511d"
dependencies = [
 "num-integer",
]

[[package]]
name = "proc-macro-hack"
version = "0.5.19"
source = "registry+https://github.com/rust-lang/crates.io-index"


@@ 1143,6 1252,17 @@ dependencies = [
]

[[package]]
name = "pulse-simple"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "846332afcb040c3c781ff4e4c3b1e3952e8ddcb6f2b3be46125981d233e02d05"
dependencies = [
 "libc",
 "libpulse-simple-sys",
 "libpulse-sys",
]

[[package]]
name = "quote"
version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"


@@ 1248,6 1368,20 @@ dependencies = [
]

[[package]]
name = "rustfft"
version = "6.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1d089e5c57521629a59f5f39bca7434849ff89bd6873b521afe389c1c602543"
dependencies = [
 "num-complex",
 "num-integer",
 "num-traits",
 "primal-check",
 "strength_reduce",
 "transpose",
]

[[package]]
name = "ryu"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"


@@ 1456,6 1590,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0"

[[package]]
name = "strength_reduce"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3ff2f71c82567c565ba4b3009a9350a96a7269eaa4001ebedae926230bc2254"

[[package]]
name = "subtle"
version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"


@@ 1595,6 1735,16 @@ dependencies = [
]

[[package]]
name = "transpose"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95f9c900aa98b6ea43aee227fd680550cdec726526aab8ac801549eadb25e39f"
dependencies = [
 "num-integer",
 "strength_reduce",
]

[[package]]
name = "typenum"
version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"

M server/Cargo.toml => server/Cargo.toml +3 -0
@@ 11,3 11,6 @@ tide = "0.16"
async-std = { version = "1.10", features = ["attributes"] }
serde = { version = "1.0", features = ["derive"] }
palette = "0.6"
rustfft = "6.0"
pulse-simple = "1.0"
num = "0.4"

A server/src/audiovis.rs => server/src/audiovis.rs +210 -0
@@ 0,0 1,210 @@
use num::Complex;
use palette::{Blend, FromColor, Hsv, Mix, Saturate, Srgb};
use pulse_simple::Record;
use rustfft::FftPlanner;
use std::{
    sync::mpsc::{channel, Receiver},
    thread,
};

const SAMPLE_RATE: usize = 48000;
// Must be power of 2
const SAMPLES_PER_PERIOD: usize = 256;

const BASS_CUTOFF: f32 = 400.0;
const HIGH_CUTOFF: f32 = 2900.0;

const MAX_FREQ: f32 = 20_000.0;

pub struct AudioVis {
    rx: Receiver<Hsv>,
    color: Hsv,
}

impl AudioVis {
    pub fn new() -> Self {
        let (tx, rx) = channel();
        thread::spawn(move || {
            let recorder = Record::new(
                "jojos-hue",
                "Capture audio to stream as color data to jojos-hue devices",
                None,
                SAMPLE_RATE as u32,
            );
            let mut stereo_data = [[0.0f32; 2]; SAMPLES_PER_PERIOD];
            let (bass_cutoff_bin, high_cutoff_bin) =
                (freq_to_bin(BASS_CUTOFF), freq_to_bin(HIGH_CUTOFF));
            let mut i = 0u16;
            loop {
                recorder.read(&mut stereo_data);
                let bin_amps = stereo_pcm_to_bins(&stereo_data);
                let mut bass_amps = [0.0; SAMPLES_PER_PERIOD >> 1];
                for (bin, &amp) in bin_amps.iter().enumerate() {
                    bass_amps[bin] = bass_pass(bin_to_freq(bin), amp);
                }
                let mut mid_amps = [0.0; SAMPLES_PER_PERIOD >> 1];
                for (bin, &amp) in bin_amps.iter().enumerate() {
                    mid_amps[bin] = mid_pass(bin_to_freq(bin), amp);
                }
                let mut high_amps = [0.0; SAMPLES_PER_PERIOD >> 1];
                for (bin, &amp) in bin_amps.iter().enumerate() {
                    high_amps[bin] = high_pass(bin_to_freq(bin), amp);
                }
                let max_amp_bass = max_amp(&bass_amps[..bass_cutoff_bin]).1;
                let m_m = max_amp(&mid_amps[bass_cutoff_bin..high_cutoff_bin]);
                let m_h = max_amp(&high_amps[high_cutoff_bin..]);
                let max_amp_mid = (m_m.0 + bass_cutoff_bin, m_m.1).1;
                let max_amp_high = (m_h.0 + high_cutoff_bin, m_h.1).1;
                let (bass_lvl, mid_lvl, high_lvl) = (
                    norm_amp(max_amp_bass).powf(1.22),       // .powf(2.5),
                    norm_amp(max_amp_mid * 1.6).powf(1.25),  // .powf(2.6),
                    norm_amp(max_amp_high * 3.2).powf(1.12), // .powf(2.4),
                );
                //let brightness = (bass_lvl * 1.6 + mid_lvl * 0.7 + high_lvl * 0.7) / 3.0;
                let color = Hsv::from_color(Srgb::new(bass_lvl, mid_lvl, high_lvl)).saturate(0.3);
                //let c1 = color.clone();
                //color.value = brightness;

                i += 1;
                if i > 40 {
                    i = 0;
                    let (max_bin_all, max_amp_all) = max_amp(&bin_amps);
                    println!(
                        "f: {:6.0}, amp: {:4.1}, vol: {:1.3}\namp: bass: {}, mid: {}, high: {}\nlvl: bass: {}, mid: {}, high: {}\nc2: {:?}\n",
                        bin_to_freq(max_bin_all),
                        max_amp_all,
                        norm_amp(max_amp_all),
                        max_amp_bass,
                        max_amp_mid,
                        max_amp_high,
                        bass_lvl,
                        mid_lvl,
                        high_lvl,
                        //c1,
                        color
                    );
                }

                tx.send(color).unwrap();
            }
        });
        Self {
            rx,
            color: Hsv::new(0.0, 0.0, 0.0),
        }
    }
    pub fn color(&mut self) -> Hsv {
        if let Some(c2) = self.rx.try_iter().last() {
            let c1 = self.color;
            let mut c3 = c1.mix(&c2, 0.6);
            let r = 0.2 + 0.8 * c2.value; // become bright fast, and dark more slowly
            c3.value = (1.0 - r) * c1.value + r * c2.value;
            self.color = c3;
            // let c1 = Srgb::from_color(self.color).into_linear();
            // let c2 = Srgb::from_color(c2).into_linear();
            // let c3 = c1.dodge(c2).mix(&c2, 0.6);
            // self.color = Hsv::from_color(Srgb::from_linear(c3)).saturate(0.15);
        }
        self.color
    }
}
fn bin_to_freq(i: usize) -> f32 {
    (i * SAMPLE_RATE) as f32 / SAMPLES_PER_PERIOD as f32
}

fn freq_to_bin(f: f32) -> usize {
    (f * SAMPLES_PER_PERIOD as f32 / SAMPLE_RATE as f32 + 0.5) as usize
}

fn pass_to(freq: f32, amp: f32, cut: f32) -> f32 {
    assert!(freq >= 0.0);
    if freq > cut {
        0.0
    } else {
        let x = freq / cut;
        let sharpness = (cut / 400.0).powf(2.0);
        amp * f32::max(1.0 - (3.0 * sharpness).powf(7.0 * (x - 1.0)), 0.0)
    }
}

fn band_pass(freq: f32, amp: f32, low_cut: f32, high_cut: f32) -> f32 {
    pass_from(freq, pass_to(freq, amp, high_cut), low_cut)
}

fn pass_from(freq: f32, amp: f32, cut: f32) -> f32 {
    assert!(freq >= 0.0 && cut < MAX_FREQ);
    if freq < cut {
        0.0
    } else {
        let x = (freq - cut) / (MAX_FREQ - cut);
        let sharpness = ((MAX_FREQ - cut) / 2700.0).powf(10.0);
        amp * f32::max(1.0 - (3.0 * sharpness).powf(-7.0 * x), 0.0)
    }
}

fn bass_pass(freq: f32, amp: f32) -> f32 {
    pass_to(freq, amp, BASS_CUTOFF)
}

fn mid_pass(freq: f32, amp: f32) -> f32 {
    band_pass(freq, amp, BASS_CUTOFF - 170.0, HIGH_CUTOFF + 300.0)
}

fn high_pass(freq: f32, amp: f32) -> f32 {
    pass_from(freq, amp, HIGH_CUTOFF)
}

/// Normalize an amplitude to [0, 1]
fn norm_amp(x: f32) -> f32 {
    (x / (x.powi(2) + 8.0).sqrt()).powf(1.8)
    //    db / (db + 1.0)
    // let x = db / 52.0;
    // if x > 1.0 {
    //     1.0
    // } else {
    //     (1.0 - x) * x + x * (1.0 / (1.0 + (-10.0 * (x - 0.5)).exp()))
    // }
}

/// bin == index of fft where there is a corresponding frequence
fn stereo_pcm_to_bins(
    stereo_data: &[[f32; 2]; SAMPLES_PER_PERIOD],
) -> [f32; SAMPLES_PER_PERIOD >> 1] {
    let complex_zero = Complex {
        re: 0.0f32,
        im: 0.0f32,
    };
    let mut avg_data = [complex_zero; SAMPLES_PER_PERIOD];
    for (i, &[l, r]) in stereo_data.iter().enumerate() {
        avg_data[i] = Complex {
            re: (l + r) / 2.0,
            im: 0.0,
        };
    }
    let mut planner = FftPlanner::new();
    let fft = planner.plan_fft_forward(SAMPLES_PER_PERIOD);
    fft.process(&mut avg_data as &mut [_]);
    let bin_amps_complex = avg_data;
    let mut bin_amps = [0.0f32; SAMPLES_PER_PERIOD >> 1];
    for (i, &c) in bin_amps_complex
        .iter()
        .take(SAMPLES_PER_PERIOD >> 1)
        .enumerate()
    {
        let amp = (c.re.powi(2) + c.im.powi(2)).sqrt();
        //bin_amps[i] = 20.0 * amp.log(10.0);
        bin_amps[i] = amp;
    }
    bin_amps
}

fn max_amp(amps: &[f32]) -> (usize, f32) {
    let (mut max_bin, mut max_amp) = (0, 0.0);
    for i in 0..amps.len() {
        if amps[i] > max_amp {
            max_amp = amps[i];
            max_bin = i;
        }
    }
    (max_bin, max_amp)
}

M server/src/main.rs => server/src/main.rs +35 -21
@@ 1,3 1,4 @@
use audiovis::AudioVis;
use byteorder::{ByteOrder, LittleEndian};
use palette::{FromColor, Hsv, Hue, RgbHue, Shade, Srgb};
use std::{


@@ 14,6 15,8 @@ use std::{
};
use tide::{prelude::*, Request};

mod audiovis;

const FPS: f32 = 100.0;
const DEADLINE: Duration = Duration::from_secs(10);



@@ 32,14 35,36 @@ enum QueryCmd {
    Off,
    Cycle,
    Breathing,
    AudioVis,
}

#[derive(Clone, Copy)]
enum Cmd {
    All,
    Off,
    Cycle,
    Breathing,
    AudioVis(AudioVis),
}

impl Cmd {
    fn render_cmd(&mut self, base: Hsv, t: f64) -> RenderCmd {
        match self {
            Cmd::All => RenderCmd::All(Srgb::from_color(base)),
            Cmd::Off => RenderCmd::All(Srgb::from_color(base.darken(t.min(1.0) as f32))),
            Cmd::Cycle => RenderCmd::All(Srgb::from_color(
                base.shift_hue(hue_f64_to_f32(RgbHue::from_radians(t / 6.0))),
            )),
            Cmd::Breathing => RenderCmd::All(Srgb::from_color(
                base.darken(0.9 * (t / 2.0).sin().powi(2) as f32),
            )),
            Cmd::AudioVis(av) => RenderCmd::All(Srgb::from_color(av.color())),
        }
    }
}

#[derive(Clone, Copy)]
enum RenderCmd {
    All(Srgb),
}

#[async_std::main]


@@ 92,6 117,7 @@ async fn proc_handler(req: Request<()>, tx: SyncSender<QueryCmd>) -> tide::Resul
        "off" => tx.send(QueryCmd::Off).unwrap(),
        "cycle" => tx.send(QueryCmd::Cycle).unwrap(),
        "breathing" => tx.send(QueryCmd::Breathing).unwrap(),
        "audiovis" => tx.send(QueryCmd::AudioVis).unwrap(),
        _ => {}
    }
    Ok("cool, ty".into())


@@ 109,6 135,7 @@ fn controller(heartbeat_rx: Receiver<Heartbeat>, cmd_rx: Receiver<QueryCmd>) {
            match clients.get_mut(&addr) {
                Some(client) => client.0 = Instant::now(),
                None => {
                    println!("Registered client {} with {} LEDs", addr, nleds);
                    clients.insert(
                        addr,
                        (Instant::now(), setup_client(addr, nleds, sender.clone())),


@@ 134,11 161,12 @@ fn controller(heartbeat_rx: Receiver<Heartbeat>, cmd_rx: Receiver<QueryCmd>) {
                QueryCmd::Off => cmd = Cmd::Off,
                QueryCmd::Cycle => cmd = Cmd::Cycle,
                QueryCmd::Breathing => cmd = Cmd::Breathing,
                QueryCmd::AudioVis => cmd = Cmd::AudioVis(AudioVis::new()),
            }
        }
        for (_, tx) in clients.values() {
            let t = tcmd.elapsed().as_secs_f64();
            tx.send((cmd, base, t)).ok();
            tx.send(cmd.render_cmd(base, t)).ok();
        }
        let dt = tprev.elapsed();
        tprev = Instant::now();


@@ 149,14 177,14 @@ fn controller(heartbeat_rx: Receiver<Heartbeat>, cmd_rx: Receiver<QueryCmd>) {
    }
}

fn setup_client(addr: SocketAddr, nleds: u16, sender: Arc<UdpSocket>) -> Sender<(Cmd, Hsv, f64)> {
fn setup_client(addr: SocketAddr, nleds: u16, sender: Arc<UdpSocket>) -> Sender<RenderCmd> {
    let (tx, rx) = channel();
    thread::spawn(move || loop {
        let magic_cookie = b"satan";
        let (cmd, base, t) = rx
        let cmd = rx
            .recv()
            .expect("Client channel closed. Time for this thread to die!");
        let color_data = lightf(cmd, base, t, nleds)
        let color_data = lightf(cmd, nleds)
            .into_iter()
            .flat_map(|c| once(c.red).chain(once(c.green)).chain(once(c.blue)))
            .map(|x| (x * 256.0).min(256.0).max(0.0) as u8);


@@ 173,29 201,15 @@ fn setup_client(addr: SocketAddr, nleds: u16, sender: Arc<UdpSocket>) -> Sender<
fn parse_heartbeat(msg: &[u8; 13], addr: SocketAddr) -> Option<Heartbeat> {
    if msg.starts_with(b"heart<") && msg.ends_with(b">beat") {
        let nleds = LittleEndian::read_u16(&msg[6..8]);
        println!("Heartbeat from {} with {} LEDs", addr, nleds);
        Some((addr, nleds))
    } else {
        None
    }
}

fn lightf(cmd: Cmd, base: Hsv, t: f64, nleds: u16) -> Vec<Srgb> {
fn lightf(cmd: RenderCmd, nleds: u16) -> Vec<Srgb> {
    match cmd {
        Cmd::All => vec![Srgb::from_color(base); nleds as usize],
        Cmd::Off => vec![Srgb::from_color(base.darken(t.min(1.0) as f32)); nleds as usize],
        Cmd::Cycle => {
            vec![
                Srgb::from_color(base.shift_hue(hue_f64_to_f32(RgbHue::from_radians(t / 6.0))));
                nleds as usize
            ]
        }
        Cmd::Breathing => {
            vec![
                Srgb::from_color(base.darken(0.9 * (t / 2.0).sin().powi(2) as f32));
                nleds as usize
            ]
        } // Cmd::AudioVis => {}
        RenderCmd::All(color) => vec![color; nleds as usize],
    }
}


M server/static/index.html => server/static/index.html +1 -0
@@ 13,6 13,7 @@
      <button id="off-button">OFF</button>
      <button id="cycle-button">Cycle</button>
      <button id="breathing-button">Breathing</button>
      <button id="audiovis-button">AudioVis</button>
    </main>

    <script lang="javascript" src="/static/reinvented-color-wheel.js"></script>

M server/static/main.js => server/static/main.js +2 -1
@@ 31,7 31,8 @@ var colorWheel = new ReinventedColorWheel({
let nullaryButtons =
    [["off-button", "off"],
     ["cycle-button", "cycle"],
     ["breathing-button", "breathing"]];
     ["breathing-button", "breathing"],
     ["audiovis-button", "audiovis"]];

nullaryButtons.forEach(([btn, proc]) => {
    document.getElementById(btn).onclick = () => {