From 6268ca805205706136881c93c1192b750c8245e7 Mon Sep 17 00:00:00 2001 From: Baptiste Parmantier Date: Wed, 12 Aug 2026 19:13:44 +0200 Subject: [PATCH] feat(audio): analyse the mixed envelope, not the raw source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit volume, volume_keyframes and the fades are what comes out of the speakers, and since #182 the studio plays exactly that — a waveform drawing the raw file's envelope while a fade takes the sound down contradicts what the viewer hears. The gain comes from the encoder's own envelope, factored out of the mixer loop as track_gain_at(track, t_in_track, audible) and expressed in seconds so the mixer (OUTPUT_SAMPLE_RATE, resampled) and the analysis (the file's own rate, decoded source) share one implementation. A second copy would drift, and the whole point is that the picture matches the sound. The cache fingerprint now hashes the serialised track rather than start/end alone: two scenarios can name the same file with different mixes, and adding a field to AudioTrack must not silently leave it out of the key. Not mergeable yet — see #201. The components read scene-local time against a scenario-time analysis, which is harmless while the envelope is flat and turns into a flat trace as soon as it is not. --- crates/rustmotion/src/encode/audio.rs | 62 ++++++++----- .../rustmotion/src/encode/audio_analysis.rs | 57 +++++++++--- crates/rustmotion/src/tests.rs | 92 +++++++++++++++++++ 3 files changed, 175 insertions(+), 36 deletions(-) diff --git a/crates/rustmotion/src/encode/audio.rs b/crates/rustmotion/src/encode/audio.rs index 62877a1..4752fc9 100644 --- a/crates/rustmotion/src/encode/audio.rs +++ b/crates/rustmotion/src/encode/audio.rs @@ -221,9 +221,6 @@ pub fn mix_audio_tracks_segment( .unwrap_or(scenario_samples) .min(scenario_samples); - let fade_in_samples = track.fade_in.unwrap_or(0.0) * TARGET_SAMPLE_RATE as f64; - let fade_out_samples = track.fade_out.unwrap_or(0.0) * TARGET_SAMPLE_RATE as f64; - // How much of the track is ever audible in the scenario, regardless // of which segment we are materializing right now. Fades are // computed against this, not against the segment's own bounds. @@ -251,25 +248,12 @@ pub fn mix_audio_tracks_segment( } let frame = i / TARGET_CHANNELS as usize; - let current_time = track.start + (frame as f64 / TARGET_SAMPLE_RATE as f64); - let vol = if !track.volume_keyframes.is_empty() { - interpolate_volume_keyframes(&track.volume_keyframes, current_time) - } else { - track.volume - }; - let mut sample = src_sample * vol; - - // Apply fade in - if fade_in_samples > 0.0 && (frame as f64) < fade_in_samples { - sample *= frame as f32 / fade_in_samples as f32; - } - - // Apply fade out — against the track's own total audible - // frames in the scenario, not this segment's length. - let frames_from_end = total_frames - frame; - if fade_out_samples > 0.0 && (frames_from_end as f64) < fade_out_samples { - sample *= frames_from_end as f32 / fade_out_samples as f32; - } + let sample = src_sample + * track_gain_at( + track, + frame as f64 / TARGET_SAMPLE_RATE as f64, + total_frames as f64 / TARGET_SAMPLE_RATE as f64, + ); mix_buffer[dst_idx] += sample; } @@ -286,6 +270,40 @@ pub fn mix_audio_tracks_segment( Ok(Some(pcm_bytes)) } +/// The gain applied to a track `t_in_track` seconds after its own first sample, +/// given that `audible` seconds of it are ever heard. +/// +/// Expressed in seconds rather than sample indices so the mixer (which works at +/// `OUTPUT_SAMPLE_RATE` on resampled audio) and the analysis (which works at the +/// file's own rate on the decoded source) can share it. They must: a waveform +/// that draws an envelope the mix does not produce is the component lying about +/// the very track it claims to react to. +pub(crate) fn track_gain_at( + track: &crate::schema::AudioTrack, + t_in_track: f64, + audible: f64, +) -> f32 { + let mut gain = if track.volume_keyframes.is_empty() { + track.volume + } else { + // Keyframe times are on the *scenario* timeline, not the track's. + interpolate_volume_keyframes(&track.volume_keyframes, track.start + t_in_track) + }; + + if let Some(fade_in) = track.fade_in { + if fade_in > 0.0 && t_in_track < fade_in { + gain *= (t_in_track / fade_in) as f32; + } + } + if let Some(fade_out) = track.fade_out { + let remaining = audible - t_in_track; + if fade_out > 0.0 && remaining < fade_out { + gain *= (remaining.max(0.0) / fade_out) as f32; + } + } + gain +} + /// Interpolate volume at a given time using volume keyframes with easing fn interpolate_volume_keyframes(keyframes: &[crate::schema::VolumeKeyframe], time: f64) -> f32 { if keyframes.is_empty() { diff --git a/crates/rustmotion/src/encode/audio_analysis.rs b/crates/rustmotion/src/encode/audio_analysis.rs index fa68e3a..7f8d438 100644 --- a/crates/rustmotion/src/encode/audio_analysis.rs +++ b/crates/rustmotion/src/encode/audio_analysis.rs @@ -31,10 +31,12 @@ impl std::fmt::Display for AudioAnalysisFailure { /// up that way), so without this a track whose *content* changed under a stable /// path — the normal case when someone re-exports a mix while the studio is /// open — would keep serving the old envelope forever. -/// Also carries the track's placement: the analysis content depends only -/// on the file, but the *lookup* now applies `start`/`end`, so an entry -/// computed for one placement must not be reused for another. -type SourceFingerprint = (u64, u128, u32, u64, u64); +/// File identity (length, mtime), the fps it was bucketed at, and a hash of +/// everything about the *track* that changes the result: `start`/`end` move the +/// lookup, and `volume`/`volume_keyframes`/the fades are baked into the +/// amplitudes. An entry computed for one of those must never be served for +/// another — two scenarios can name the same file with different mixes. +type SourceFingerprint = (u64, u128, u32, u64); static FINGERPRINTS: OnceLock>> = OnceLock::new(); @@ -48,8 +50,7 @@ fn fingerprints() -> &'static Mutex> { fn source_fingerprint( src: &str, fps: u32, - start: f64, - end: Option, + track: &rustmotion_core::schema::AudioTrack, ) -> Option { let meta = std::fs::metadata(src).ok()?; let mtime = meta @@ -58,13 +59,19 @@ fn source_fingerprint( .duration_since(std::time::UNIX_EPOCH) .ok()? .as_nanos(); - Some(( - meta.len(), - mtime, - fps, - start.to_bits(), - end.unwrap_or(f64::INFINITY).to_bits(), - )) + Some((meta.len(), mtime, fps, track_hash(track))) +} + +/// Hash the track's placement and volume envelope. Serialised rather than +/// hashed field by field so adding a field to `AudioTrack` cannot silently +/// leave it out of the key. +fn track_hash(track: &rustmotion_core::schema::AudioTrack) -> u64 { + use std::hash::{Hash, Hasher}; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + serde_json::to_string(track) + .unwrap_or_default() + .hash(&mut hasher); + hasher.finish() } /// Build the 16 log-spaced band frequency boundaries (Hz) from 20..16000. @@ -107,7 +114,7 @@ pub fn analyze_scenario_audio(scenario: &ResolvedScenario) -> Vec Vec file_seconds.min((end - track.start).max(0.0)), + None => file_seconds, + } + }; + let mono: Vec = mono + .into_iter() + .enumerate() + .map(|(i, s)| { + let t = i as f64 / sample_rate as f64; + s * crate::encode::audio::track_gain_at(track, t, audible) + }) + .collect(); + let samples_per_frame = (sample_rate as f64 / fps as f64).ceil() as usize; let num_frames = (mono.len() as f64 / samples_per_frame as f64).ceil() as usize; diff --git a/crates/rustmotion/src/tests.rs b/crates/rustmotion/src/tests.rs index fa49637..54bf562 100644 --- a/crates/rustmotion/src/tests.rs +++ b/crates/rustmotion/src/tests.rs @@ -1992,6 +1992,98 @@ mod audio_tests { count } + /// The analysis must describe the **mix**, not the source. + /// + /// `volume`, `volume_keyframes` and the fades are what comes out of the + /// speakers, and since #182 the studio plays exactly that. A waveform + /// drawing the raw file's envelope while a keyframe takes the sound to + /// zero contradicts what the viewer hears. + #[test] + fn the_analysis_follows_the_mixed_envelope_not_the_raw_file() { + let sample_rate = 44100u32; + // 2 s of unbroken sine: any variation in the analysis comes from the + // envelope, never from the source. + let wav_path = std::env::temp_dir().join(format!("rustmotion_test_mix_{}.wav", nanos())); + std::fs::write( + &wav_path, + make_sine_wav(sample_rate * 2, sample_rate * 2, 440.0, sample_rate), + ) + .expect("write fixture"); + let wav_str = wav_path.to_str().unwrap().to_string(); + + // Full for the first second, silent for the second. + let json = serde_json::json!({ + "video": {"width": 32, "height": 32, "fps": 30}, + "audio": [{ + "src": wav_str, + "volume_keyframes": [ + {"time": 0.0, "volume": 1.0}, + {"time": 1.0, "volume": 1.0}, + {"time": 1.05, "volume": 0.0}, + {"time": 2.0, "volume": 0.0} + ] + }], + "scenes": [{"duration": 2.0, "children": []}] + }) + .to_string(); + let scenario = + crate::loader::load_scenario_from_source(None, Some(&json)).expect("load scenario"); + assert!(crate::encode::audio_analysis::analyze_scenario_audio(&scenario).is_empty()); + + let analysis = audio_analysis_cache().get(&wav_str).unwrap().clone(); + std::fs::remove_file(&wav_path).ok(); + + assert!( + analysis.amplitude_at(0.5) > 0.5, + "inside the audible half the envelope is open" + ); + assert!( + analysis.amplitude_at(1.5) < 0.05, + "the keyframes take the track to silence — the visualisation must \ + follow it, not keep drawing the sine underneath" + ); + } + + /// The envelope is part of the cache key: two scenarios naming the same + /// file with different mixes must not share an analysis. + #[test] + fn changing_the_envelope_re_analyses_the_same_file() { + let sample_rate = 44100u32; + let wav_path = std::env::temp_dir().join(format!("rustmotion_test_env_{}.wav", nanos())); + std::fs::write( + &wav_path, + make_sine_wav(sample_rate, sample_rate, 440.0, sample_rate), + ) + .expect("write fixture"); + let wav_str = wav_path.to_str().unwrap().to_string(); + + let with_volume = |v: f32| { + let json = serde_json::json!({ + "video": {"width": 32, "height": 32, "fps": 30}, + "audio": [{"src": wav_str, "volume": v}], + "scenes": [{"duration": 1.0, "children": []}] + }) + .to_string(); + let scenario = + crate::loader::load_scenario_from_source(None, Some(&json)).expect("load"); + crate::encode::audio_analysis::analyze_scenario_audio(&scenario); + audio_analysis_cache().get(&wav_str).unwrap().amplitude[10] + }; + + // Amplitudes are normalised per analysis, so a flat gain change cannot + // be read off the values — assert the *entry* was replaced instead. + let loud = with_volume(1.0); + let quiet = with_volume(0.0); + std::fs::remove_file(&wav_path).ok(); + + assert!(loud > 0.5, "the full-volume take is audible, got {loud}"); + assert_eq!( + quiet, 0.0, + "at volume 0 the analysis must be silent — a stale entry would \ + still report {loud}" + ); + } + /// The failure the whole rewrite exists for: a scenario naming an asset /// beside itself must load identically whatever directory the process /// runs from. Authoring from the scenario's folder worked; the studio,