diff --git a/Cargo.lock b/Cargo.lock index 8167654..f4aeb04 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,6 +41,28 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "alsa" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812947049edcd670a82cd5c73c3661d2e58468577ba8489de58e1a73c04cbd5d" +dependencies = [ + "alsa-sys", + "bitflags 2.11.0", + "cfg-if", + "libc", +] + +[[package]] +name = "alsa-sys" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad7569085a265dd3f607ebecce7458eaab2132a84393534c95b18dcbc3f31e04" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "anstream" version = "0.6.21" @@ -782,6 +804,20 @@ dependencies = [ "libm", ] +[[package]] +name = "coreaudio-rs" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5d7dca3ebcf65a035582c9ad4385371a9d9ee6537474d2a278f4e1e475bb58" +dependencies = [ + "bitflags 2.11.0", + "libc", + "objc2-audio-toolbox", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", +] + [[package]] name = "cosmic-text" version = "0.19.0" @@ -806,6 +842,36 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "cpal" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8942da362c0f0d895d7cac616263f2f9424edc5687364dfd1d25ef7eba506d7" +dependencies = [ + "alsa", + "coreaudio-rs", + "dasp_sample", + "jni 0.21.1", + "js-sys", + "libc", + "mach2", + "ndk", + "ndk-context", + "num-derive", + "num-traits", + "objc2", + "objc2-audio-toolbox", + "objc2-avf-audio", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -1008,6 +1074,12 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "dasp_sample" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" + [[package]] name = "data-encoding" version = "2.11.0" @@ -3185,6 +3257,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" +[[package]] +name = "mach2" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a1b95cd5421ec55b445b5ae102f5ea0e768de1f82bd3001e11f426c269c3aea" +dependencies = [ + "libc", +] + [[package]] name = "macro-string" version = "0.1.4" @@ -3693,6 +3774,54 @@ dependencies = [ "objc2-foundation", ] +[[package]] +name = "objc2-audio-toolbox" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6948501a91121d6399b79abaa33a8aa4ea7857fe019f341b8c23ad6e81b79b08" +dependencies = [ + "bitflags 2.11.0", + "libc", + "objc2", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation", +] + +[[package]] +name = "objc2-avf-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be" +dependencies = [ + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1eebcea8b0dbff5f7c8504f3107c68fc061a3eb44932051c8cf8a68d969c3b2" +dependencies = [ + "dispatch2", + "objc2", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-audio-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c" +dependencies = [ + "bitflags 2.11.0", + "objc2", +] + [[package]] name = "objc2-core-foundation" version = "0.3.2" @@ -3700,7 +3829,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ "bitflags 2.11.0", + "block2", "dispatch2", + "libc", "objc2", ] @@ -3737,6 +3868,7 @@ checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" dependencies = [ "bitflags 2.11.0", "block2", + "libc", "objc2", "objc2-core-foundation", ] @@ -4791,6 +4923,18 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rodio" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a536bb79db59098ef71a4dd4246c02eb87b316deceb1b68e0cde7167ec01eb" +dependencies = [ + "cpal", + "dasp_sample", + "num-rational", + "thiserror 2.0.18", +] + [[package]] name = "roxmltree" version = "0.20.0" @@ -5020,6 +5164,7 @@ dependencies = [ "notify", "palette", "rfd", + "rodio", "rustmotion", "schemars", "serde_json", diff --git a/crates/rustmotion-studio/Cargo.toml b/crates/rustmotion-studio/Cargo.toml index 63698f5..df154ed 100644 --- a/crates/rustmotion-studio/Cargo.toml +++ b/crates/rustmotion-studio/Cargo.toml @@ -22,6 +22,7 @@ serde_json = { version = "1", features = ["preserve_order"] } tokio = { version = "1", features = ["time"] } dioxus-icons = { version = "0.1.0", default-features = false } rfd = "0.17" +rodio = { version = "0.22", default-features = false, features = ["playback"] } dirs = "6.0.0" palette = { version = "0.7.6", default-features = false } schemars = "0.8" diff --git a/crates/rustmotion-studio/src/editor/audio.rs b/crates/rustmotion-studio/src/editor/audio.rs new file mode 100644 index 0000000..5b01547 --- /dev/null +++ b/crates/rustmotion-studio/src/editor/audio.rs @@ -0,0 +1,256 @@ +//! Preview audio for the transport bar. +//! +//! What plays is the output of [`mix_audio_tracks`] — the same function the +//! encoder calls — so `start`/`end`, fades, per-track volume and volume +//! keyframes are heard in the studio exactly as they will be muxed. Anything +//! less would make the preview lie about the cut. +//! +//! The mix spans the whole scenario, silence included, which is what lets the +//! playhead follow the audio clock from the first frame to the last instead of +//! drifting against a timer (and freezing when a track ends early). +//! +//! cpal's output stream is not `Send`, so it lives on a dedicated thread driven +//! by a command channel; position and state are published as atomics the UI +//! reads without ever touching that thread. + +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::sync::mpsc::{channel, RecvTimeoutError, Sender}; +use std::sync::{Arc, OnceLock}; +use std::time::Duration; + +use rustmotion::schema::ResolvedScenario; + +enum Cmd { + /// Interleaved stereo f32 for the whole scenario, and its sample rate. + Load(Arc>, u32), + /// Drop the current mix — the scenario has no audio at all. + Clear, + PlayFrom(f64), + Stop, + SetMuted(bool), +} + +/// Playhead position in milliseconds, valid only while [`PLAYING`] is true. +static POSITION_MS: AtomicU64 = AtomicU64::new(0); +static PLAYING: AtomicBool = AtomicBool::new(false); +static HAS_AUDIO: AtomicBool = AtomicBool::new(false); +static MUTED: AtomicBool = AtomicBool::new(false); +/// Bumped per prepared scenario so a slow mix that finishes after a newer one +/// started cannot install itself over the newer result. +static MIX_TOKEN: AtomicUsize = AtomicUsize::new(0); + +static TX: OnceLock>> = OnceLock::new(); + +fn tx() -> Option<&'static Sender> { + TX.get_or_init(spawn_audio_thread).as_ref() +} + +fn spawn_audio_thread() -> Option> { + let (tx, rx) = channel::(); + std::thread::Builder::new() + .name("rm-preview-audio".into()) + .spawn(move || { + // No output device (CI, headless, a machine with sound disabled) is + // not an error: keep draining so senders never block, just silent. + let stream = match rodio::stream::DeviceSinkBuilder::from_default_device() + .and_then(|b| b.open_stream()) + { + Ok(s) => s, + Err(e) => { + eprintln!("rustmotion-studio: no audio output device ({e}); preview is silent"); + while rx.recv().is_ok() {} + return; + } + }; + + let mut mix: Option<(Arc>, u32)> = None; + let mut sink: Option = None; + let mut start_offset = 0.0f64; + + loop { + // Short timeout so the published position stays fresh enough to + // drive the playhead at any sane fps. + match rx.recv_timeout(Duration::from_millis(10)) { + Ok(Cmd::Load(pcm, rate)) => { + mix = Some((pcm, rate)); + HAS_AUDIO.store(true, Ordering::Relaxed); + } + Ok(Cmd::Clear) => { + mix = None; + sink = None; + HAS_AUDIO.store(false, Ordering::Relaxed); + PLAYING.store(false, Ordering::Relaxed); + } + Ok(Cmd::PlayFrom(seconds)) => { + sink = None; + let Some((pcm, rate)) = mix.as_ref() else { + continue; + }; + let frame = (seconds.max(0.0) * *rate as f64) as usize; + let offset = (frame * CHANNELS as usize).min(pcm.len()); + let s = rodio::Player::connect_new(stream.mixer()); + s.set_volume(if MUTED.load(Ordering::Relaxed) { + 0.0 + } else { + 1.0 + }); + s.append(rodio::buffer::SamplesBuffer::new( + channel_count(), + sample_rate(*rate), + pcm[offset..].to_vec(), + )); + s.play(); + start_offset = seconds.max(0.0); + sink = Some(s); + POSITION_MS.store((start_offset * 1000.0) as u64, Ordering::Relaxed); + PLAYING.store(true, Ordering::Relaxed); + } + Ok(Cmd::Stop) => { + sink = None; + PLAYING.store(false, Ordering::Relaxed); + } + Ok(Cmd::SetMuted(m)) => { + if let Some(s) = sink.as_ref() { + s.set_volume(if m { 0.0 } else { 1.0 }); + } + } + Err(RecvTimeoutError::Timeout) => {} + Err(RecvTimeoutError::Disconnected) => return, + } + + if let Some(s) = sink.as_ref() { + if s.empty() { + // Ran off the end of the mix: stop claiming a position + // so the transport falls back to its own clock. + sink = None; + PLAYING.store(false, Ordering::Relaxed); + } else { + let pos = start_offset + s.get_pos().as_secs_f64(); + POSITION_MS.store((pos * 1000.0) as u64, Ordering::Relaxed); + } + } + } + }) + .ok()?; + Some(tx) +} + +const CHANNELS: u16 = 2; + +fn channel_count() -> rodio::ChannelCount { + rodio::ChannelCount::new(CHANNELS).expect("stereo is non-zero") +} + +fn sample_rate(rate: u32) -> rodio::SampleRate { + rodio::SampleRate::new(rate).unwrap_or(rodio::SampleRate::new(44_100).unwrap()) +} + +/// Mix the scenario's audio in the background and hand it to the audio thread. +/// Cheap to call on every model reload: a scenario with no tracks just clears. +pub fn prepare(scenario: Arc, total_duration: f64) { + let Some(tx) = tx() else { return }; + if scenario.audio.is_empty() { + let _ = tx.send(Cmd::Clear); + return; + } + let token = MIX_TOKEN.fetch_add(1, Ordering::SeqCst) + 1; + let tx = tx.clone(); + // Mixing decodes and resamples every track: seconds on a long scenario, so + // never on the UI thread. + std::thread::spawn(move || { + let mixed = rustmotion::encode::audio::mix_audio_tracks(&scenario.audio, total_duration); + if MIX_TOKEN.load(Ordering::SeqCst) != token { + return; // a newer scenario is already being prepared + } + match mixed { + Ok(Some(pcm_bytes)) => { + let pcm: Vec = pcm_bytes + .chunks_exact(2) + .map(|b| i16::from_le_bytes([b[0], b[1]]) as f32 / 32768.0) + .collect(); + let _ = tx.send(Cmd::Load(Arc::new(pcm), SAMPLE_RATE)); + } + Ok(None) => { + let _ = tx.send(Cmd::Clear); + } + Err(e) => { + eprintln!("rustmotion-studio: preview audio unavailable: {e}"); + let _ = tx.send(Cmd::Clear); + } + } + }); +} + +const SAMPLE_RATE: u32 = rustmotion::encode::audio::OUTPUT_SAMPLE_RATE; + +/// Start (or restart) playback at `frame`. A no-op when the scenario is silent. +/// +/// The position is published here rather than waiting for the audio thread to +/// pick the command up: a seek must not read back as the old position for the +/// handful of milliseconds in between, or the transport — which follows this +/// clock — would snap the playhead back before the sound catches up. +pub fn play_from_frame(frame: u32, fps: u32) { + let seconds = frame as f64 / fps.max(1) as f64; + POSITION_MS.store((seconds * 1000.0) as u64, Ordering::Relaxed); + PLAYING.store(true, Ordering::Relaxed); + if let Some(tx) = tx() { + let _ = tx.send(Cmd::PlayFrom(seconds)); + } +} + +pub fn stop() { + if let Some(tx) = tx() { + let _ = tx.send(Cmd::Stop); + } +} + +pub fn set_muted(muted: bool) { + MUTED.store(muted, Ordering::Relaxed); + if let Some(tx) = tx() { + let _ = tx.send(Cmd::SetMuted(muted)); + } +} + +pub fn has_audio() -> bool { + HAS_AUDIO.load(Ordering::Relaxed) +} + +/// The frame the audio is currently at, or `None` when nothing is playing — +/// in which case the transport advances on its own timer. +pub fn position_frame(fps: u32) -> Option { + if !PLAYING.load(Ordering::Relaxed) { + return None; + } + let secs = POSITION_MS.load(Ordering::Relaxed) as f64 / 1000.0; + Some((secs * fps.max(1) as f64).round() as u32) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn position_is_none_while_stopped() { + PLAYING.store(false, Ordering::Relaxed); + assert_eq!(position_frame(30), None); + } + + #[test] + fn position_converts_milliseconds_to_frames() { + PLAYING.store(true, Ordering::Relaxed); + POSITION_MS.store(2_000, Ordering::Relaxed); + assert_eq!(position_frame(30), Some(60)); + POSITION_MS.store(0, Ordering::Relaxed); + assert_eq!(position_frame(30), Some(0)); + PLAYING.store(false, Ordering::Relaxed); + } + + /// fps 0 would divide by zero on the way in and out. + #[test] + fn zero_fps_is_clamped() { + PLAYING.store(true, Ordering::Relaxed); + POSITION_MS.store(1_000, Ordering::Relaxed); + assert_eq!(position_frame(0), Some(1)); + PLAYING.store(false, Ordering::Relaxed); + } +} diff --git a/crates/rustmotion-studio/src/editor/mod.rs b/crates/rustmotion-studio/src/editor/mod.rs index 8872daf..eb432a8 100644 --- a/crates/rustmotion-studio/src/editor/mod.rs +++ b/crates/rustmotion-studio/src/editor/mod.rs @@ -2,6 +2,7 @@ //! inspector, and the comments/annotations panel. mod annotations; +pub mod audio; mod diff_panel; mod export; pub mod frames; diff --git a/crates/rustmotion-studio/src/editor/playback.rs b/crates/rustmotion-studio/src/editor/playback.rs index 5b50669..078fb93 100644 --- a/crates/rustmotion-studio/src/editor/playback.rs +++ b/crates/rustmotion-studio/src/editor/playback.rs @@ -1,7 +1,7 @@ use std::time::Duration; use dioxus::prelude::*; -use dioxus_icons::lucide::{Pause, Play}; +use dioxus_icons::lucide::{Pause, Play, Volume2, VolumeX}; use crate::components::button::{Button, ButtonSize, ButtonVariant}; use crate::components::select::{Select, SelectOption}; @@ -37,12 +37,20 @@ pub fn playback_action(key: &Key, mods: Modifiers) -> Option { } } -/// Advance the playhead at the scenario fps while `playing` is true. A custom -/// hook so the editor view stays focused on layout. +/// Advance the playhead while `playing` is true, and keep the sound with it. +/// +/// When the scenario has audio the playhead follows the *audio* clock rather +/// than its own timer: a timer ticking at 1/fps drifts against the sound card +/// over a long scenario, and by the end the picture no longer matches what you +/// hear. With no track (or no output device) it falls back to the timer. pub fn use_playback_clock(shared: Shared, mut current: Signal, playing: Signal) { use_future(move || { let shared = shared.clone(); async move { + // Whether the sound for this playback run has been started. Reset on + // pause and on loop, so a mix that finishes preparing mid-playback + // still gets picked up. + let mut audio_armed = false; loop { let fps = shared .lock() @@ -52,34 +60,61 @@ pub fn use_playback_clock(shared: Shared, mut current: Signal, playing: Sig .fps .max(1); tokio::time::sleep(Duration::from_secs_f64(1.0 / fps as f64)).await; - if playing() { - let total = shared - .lock() - .unwrap_or_else(|e| e.into_inner()) - .total_frames - .max(1); - let next = (current() + 1) % total; - current.set(next); + if !playing() { + if audio_armed { + super::audio::stop(); + audio_armed = false; + } + continue; + } + let total = shared + .lock() + .unwrap_or_else(|e| e.into_inner()) + .total_frames + .max(1); + if super::audio::has_audio() && !audio_armed { + super::audio::play_from_frame(current(), fps); + audio_armed = true; } + let from = current(); + let next = match super::audio::position_frame(fps) { + Some(f) if f < total => f, + _ => (from + 1) % total, + }; + // Wrapping past the last frame restarts the track with the picture. + if next < from { + audio_armed = false; + } + current.set(next); } } }); } /// Bump `rev` whenever the watcher swaps in a reloaded model, so the `` -/// refetches the (now changed) current frame. +/// refetches the (now changed) current frame — and re-mix the preview audio for +/// whatever scenario is now loaded. The mix has to follow the model: opening +/// another file, or editing this one's tracks, otherwise keeps playing the +/// previous scenario's sound. pub fn use_hot_reload(shared: Shared, mut rev: Signal) { use_future(move || { let shared = shared.clone(); async move { - let mut last_gen = 0u64; + let mut last_gen: Option = None; loop { - tokio::time::sleep(Duration::from_millis(250)).await; - let g = shared.lock().unwrap_or_else(|e| e.into_inner()).generation; - if g != last_gen { - last_gen = g; - rev.set(rev() + 1); + let (g, scenario, total) = { + let m = shared.lock().unwrap_or_else(|e| e.into_inner()); + (m.generation, m.scenario.clone(), m.total_frames) + }; + if last_gen != Some(g) { + if last_gen.is_some() { + rev.set(rev() + 1); + } + last_gen = Some(g); + let fps = scenario.video.fps.max(1); + super::audio::prepare(scenario, total as f64 / fps as f64); } + tokio::time::sleep(Duration::from_millis(250)).await; } } }); @@ -94,6 +129,8 @@ pub fn PlaybackBar( current: Signal, playing: Signal, total: u32, + fps: u32, + mut muted: Signal, diff_active: Signal, mut diff_side: Signal, mut preview_scale: Signal, @@ -103,6 +140,7 @@ pub fn PlaybackBar( let max = total.saturating_sub(1); let cur = current().min(max); let is_playing = playing(); + let is_muted = muted(); let side = diff_side(); rsx! { @@ -124,6 +162,21 @@ pub fn PlaybackBar( Play { size: 15 } } } + Button { + variant: ButtonVariant::Secondary, + size: ButtonSize::IconSm, + title: if is_muted { "Unmute" } else { "Mute" }, + onclick: move |_| { + let next = !muted(); + muted.set(next); + super::audio::set_muted(next); + }, + if is_muted { + VolumeX { size: 15 } + } else { + Volume2 { size: 15 } + } + } if diff_active() { div { class: "rm-seg", style: "width:auto; flex:none;", Button { @@ -163,6 +216,9 @@ pub fn PlaybackBar( oninput: move |e| { if let Ok(v) = e.value().parse::() { current.set(v); + if playing() { + super::audio::play_from_frame(v, fps); + } } }, } diff --git a/crates/rustmotion-studio/src/editor/view.rs b/crates/rustmotion-studio/src/editor/view.rs index 13db46d..9ee6052 100644 --- a/crates/rustmotion-studio/src/editor/view.rs +++ b/crates/rustmotion-studio/src/editor/view.rs @@ -50,6 +50,7 @@ pub fn StudioApp(view: Signal) -> Element { let current = use_signal(|| 0u32); let playing = use_signal(|| false); + let muted = use_signal(|| false); let rev = use_signal(|| 0u64); // Selection stores (node_id, pointer, kind) so the inspector stays open via // the stored pointer even if the element collapses out of the hit-map. @@ -157,7 +158,7 @@ pub fn StudioApp(view: Signal) -> Element { diff_side, ); - let (total, err, write_err, audio_err, title, annotations) = { + let (total, fps, err, write_err, audio_err, title, annotations) = { let m = shared.lock().unwrap_or_else(|e| e.into_inner()); let title = m .path @@ -168,6 +169,7 @@ pub fn StudioApp(view: Signal) -> Element { .to_string(); ( m.total_frames, + m.scenario.video.fps.max(1), m.error.clone(), m.write_error.clone(), m.audio_error.clone(), @@ -328,7 +330,7 @@ pub fn StudioApp(view: Signal) -> Element { div { style: "flex:1; display:flex; flex-direction:row; flex-wrap:nowrap; align-items:stretch; min-height:0; overflow:hidden;", div { style: "flex:1; min-width:0; display:flex; flex-direction:column; min-height:0;", Canvas { current, rev, show_hits, playing, selected, diff_active, diff_side, diff_marks, preview_scale } - PlaybackBar { current, playing, total, diff_active, diff_side, preview_scale } + PlaybackBar { current, playing, total, fps, muted, diff_active, diff_side, preview_scale } } div { style: "flex:none; display:flex; overflow:hidden; transition:width 220ms ease; width:{panel_w};",