From 71a1a26daedc86cda75a6b53410eeb6562c95813 Mon Sep 17 00:00:00 2001 From: Akash Parmar Date: Tue, 11 Aug 2026 19:12:35 +0530 Subject: [PATCH] fix(editor): allow muting split segments independently Persist mute state per timeline segment instead of per clip, so splitting a segment lets each half be muted independently. Centralizes the mute check in the audio renderer for normal-speed, retimed, and transition audio, and threads audioMuted through Cap-project import so it isn't lost when appending a muted segment into another project. --- apps/cli/src/selftest/playback.rs | 1 + .../desktop-display-transport-benchmark.rs | 2 + apps/desktop/src-tauri/src/import.rs | 137 +++++++++++++++++- apps/desktop/src-tauri/src/recording.rs | 1 + .../src/routes/editor/ConfigSidebar.tsx | 14 ++ apps/desktop/src/routes/editor/context.ts | 3 + apps/desktop/src/utils/tauri.ts | 2 +- .../examples/editor-playback-benchmark.rs | 2 + .../examples/playback-pipeline-benchmark.rs | 2 + crates/editor/src/audio.rs | 66 +++++++-- crates/editor/src/editor_instance.rs | 2 + crates/export/src/lib.rs | 1 + crates/project/src/configuration.rs | 17 +++ crates/recording/src/recovery.rs | 1 + crates/recording/src/studio_recording.rs | 1 + crates/recording/src/track_heal.rs | 4 + crates/rendering/src/zoom_spring.rs | 3 + 17 files changed, 240 insertions(+), 19 deletions(-) diff --git a/apps/cli/src/selftest/playback.rs b/apps/cli/src/selftest/playback.rs index 83845d08aa..2bb0230844 100644 --- a/apps/cli/src/selftest/playback.rs +++ b/apps/cli/src/selftest/playback.rs @@ -903,6 +903,7 @@ mod fixture { timescale: 1.0, name: None, speed_audio_mode: None, + audio_muted: false, }], transitions: Vec::new(), zoom_segments: Vec::new(), diff --git a/apps/desktop/src-tauri/examples/desktop-display-transport-benchmark.rs b/apps/desktop/src-tauri/examples/desktop-display-transport-benchmark.rs index 35693bf8cb..5565d8779a 100644 --- a/apps/desktop/src-tauri/examples/desktop-display-transport-benchmark.rs +++ b/apps/desktop/src-tauri/examples/desktop-display-transport-benchmark.rs @@ -133,6 +133,7 @@ async fn load_recording( timescale: 1.0, name: None, speed_audio_mode: None, + audio_muted: false, }] } StudioRecordingMeta::MultipleSegments { inner } => inner @@ -152,6 +153,7 @@ async fn load_recording( timescale: 1.0, name: None, speed_audio_mode: None, + audio_muted: false, }) }) .collect(), diff --git a/apps/desktop/src-tauri/src/import.rs b/apps/desktop/src-tauri/src/import.rs index 94b4b89f72..f06e10ae30 100644 --- a/apps/desktop/src-tauri/src/import.rs +++ b/apps/desktop/src-tauri/src/import.rs @@ -324,6 +324,7 @@ fn full_timeline_for_segments( end: duration, name: None, speed_audio_mode: None, + audio_muted: false, }) }) .collect() @@ -358,6 +359,7 @@ fn full_timeline_for_source_segments( end: duration, name: None, speed_audio_mode: None, + audio_muted: false, }) }) .collect() @@ -920,6 +922,7 @@ fn source_timeline_segments_for_import( end, name: None, speed_audio_mode: None, + audio_muted: segment.audio_muted, }); } @@ -1718,6 +1721,7 @@ async fn append_mp4_to_editor_project( end: duration, name: None, speed_audio_mode: None, + audio_muted: false, }); add_clip_configs( &mut config, @@ -1767,13 +1771,21 @@ async fn append_cap_project_to_editor_project( }; }; + append_studio_project_to_editor_project(target_project_path, &source_meta, source_studio_meta) +} + +fn append_studio_project_to_editor_project( + target_project_path: PathBuf, + source_meta: &RecordingMeta, + source_studio_meta: &StudioRecordingMeta, +) -> Result { let source_segments = studio_segments_for_import(source_studio_meta); if source_segments.is_empty() { return Err("Source Cap project has no recording segments".to_string()); } - let source_timeline = source_timeline_segments_for_import(&source_meta, &source_segments)?; - let source_cursors = match source_studio_meta.as_ref() { + let source_timeline = source_timeline_segments_for_import(source_meta, &source_segments)?; + let source_cursors = match source_studio_meta { StudioRecordingMeta::MultipleSegments { inner } => Some(&inner.cursors), StudioRecordingMeta::SingleSegment { .. } => None, }; @@ -1849,6 +1861,7 @@ async fn append_cap_project_to_editor_project( end: source_segment.end, name: None, speed_audio_mode: source_segment.speed_audio_mode, + audio_muted: source_segment.audio_muted, }); } } @@ -2083,6 +2096,126 @@ pub async fn check_import_ready(project_path: PathBuf) -> Result { mod tests { use super::*; + #[test] + fn cap_project_import_preserves_segment_mute() { + let _ = ffmpeg::init(); + let source_project = tempfile::tempdir().unwrap(); + let target_project = tempfile::tempdir().unwrap(); + let display_path = RelativePathBuf::from("content/segments/segment-0/display.mp4"); + let absolute_display_path = display_path.to_path(source_project.path()); + std::fs::create_dir_all(absolute_display_path.parent().unwrap()).unwrap(); + std::fs::write( + &absolute_display_path, + include_bytes!("../../../media-server/src/__tests__/fixtures/test-no-audio.mp4"), + ) + .unwrap(); + + let source_segment = MultipleSegment { + display: VideoMeta { + path: display_path, + fps: 30, + start_time: Some(0.0), + device_id: None, + }, + camera: None, + mic: None, + system_audio: None, + cursor: None, + keyboard: None, + display_notch: None, + }; + let source_meta = RecordingMeta { + platform: Some(Platform::default()), + project_path: source_project.path().to_path_buf(), + pretty_name: "Muted import fixture".to_string(), + sharing: None, + inner: RecordingMetaInner::Studio(Box::new(StudioRecordingMeta::MultipleSegments { + inner: MultipleSegments { + segments: vec![source_segment], + cursors: Cursors::default(), + status: Some(StudioRecordingStatus::Complete), + }, + })), + upload: None, + }; + source_meta.save_for_project().unwrap(); + ProjectConfiguration { + timeline: Some(TimelineConfiguration { + segments: vec![TimelineSegment { + recording_clip: 0, + timescale: 1.0, + start: 0.0, + end: 1.0, + name: None, + speed_audio_mode: None, + audio_muted: true, + }], + transitions: Vec::new(), + zoom_segments: Vec::new(), + scene_segments: Vec::new(), + mask_segments: Vec::new(), + text_segments: Vec::new(), + caption_segments: Vec::new(), + keyboard_segments: Vec::new(), + audio_segments: Vec::new(), + camera3d_segments: Vec::new(), + }), + ..Default::default() + } + .write(source_project.path()) + .unwrap(); + let source_meta = RecordingMeta::load_for_project(source_project.path()).unwrap(); + + RecordingMeta { + platform: Some(Platform::default()), + project_path: target_project.path().to_path_buf(), + pretty_name: "Target import fixture".to_string(), + sharing: None, + inner: RecordingMetaInner::Studio(Box::new(StudioRecordingMeta::MultipleSegments { + inner: MultipleSegments { + segments: Vec::new(), + cursors: Cursors::default(), + status: Some(StudioRecordingStatus::Complete), + }, + })), + upload: None, + } + .save_for_project() + .unwrap(); + + let RecordingMetaInner::Studio(source_studio) = &source_meta.inner else { + panic!("expected Studio source metadata"); + }; + let imported = append_studio_project_to_editor_project( + target_project.path().to_path_buf(), + &source_meta, + source_studio, + ) + .unwrap(); + + let target_config = ProjectConfiguration::load(target_project.path()).unwrap(); + let target_timeline = target_config.timeline.unwrap(); + let target_meta = RecordingMeta::load_for_project(target_project.path()).unwrap(); + let RecordingMetaInner::Studio(target_studio) = target_meta.inner else { + panic!("expected Studio recording metadata"); + }; + let StudioRecordingMeta::MultipleSegments { inner } = *target_studio else { + panic!("expected multiple recording segments"); + }; + + assert_eq!(imported, 1); + assert_eq!(target_timeline.segments.len(), 1); + assert!(target_timeline.segments[0].audio_muted); + assert_eq!(inner.segments.len(), 1); + assert!( + inner.segments[0] + .display + .path + .to_path(target_project.path()) + .is_file() + ); + } + #[test] fn source_asset_path_allows_file_inside_source_project() { let source_project = tempfile::tempdir().unwrap(); diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index 2d0ef1a004..656ddd734a 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -3844,6 +3844,7 @@ fn project_config_from_recording( timescale: 1.0, name: None, speed_audio_mode: None, + audio_muted: false, }) .collect::>(); diff --git a/apps/desktop/src/routes/editor/ConfigSidebar.tsx b/apps/desktop/src/routes/editor/ConfigSidebar.tsx index 08cf05fc7d..723b6eb6ad 100644 --- a/apps/desktop/src/routes/editor/ConfigSidebar.tsx +++ b/apps/desktop/src/routes/editor/ConfigSidebar.tsx @@ -5606,6 +5606,20 @@ function ClipSegmentConfig(props: {

+ }> + + + projectActions.setClipSegmentAudioMuted( + props.segmentIndex, + audioMuted, + ) + } + /> + + + }> { + setProject("timeline", "segments", index, "audioMuted", audioMuted); + }, }; let projectSaveTimeout: number | undefined; diff --git a/apps/desktop/src/utils/tauri.ts b/apps/desktop/src/utils/tauri.ts index 7ad31e0951..1e891df708 100644 --- a/apps/desktop/src/utils/tauri.ts +++ b/apps/desktop/src/utils/tauri.ts @@ -1068,7 +1068,7 @@ export type SystemDiagnostics = { macosVersion: MacOSVersionInfo | null; availab export type TargetUnderCursor = { display_id: DisplayId | null; window: WindowUnderCursor | null } export type TextSegment = { start: number; end: number; track?: number; enabled?: boolean; content?: string; center?: XY; size?: XY; fontFamily?: string; fontSize?: number; fontWeight?: number; italic?: boolean; color?: string; fadeDuration?: number } export type TimelineConfiguration = { segments: TimelineSegment[]; transitions: ClipTransition[]; zoomSegments: ZoomSegment[]; sceneSegments?: SceneSegment[]; maskSegments?: MaskSegment[]; textSegments?: TextSegment[]; captionSegments?: CaptionTrackSegment[]; keyboardSegments?: KeyboardTrackSegment[]; audioSegments?: AudioTrackSegment[]; camera3dSegments?: Camera3DSegment[] } -export type TimelineSegment = { recordingSegment?: number; timescale: number; start: number; end: number; name?: string | null; speedAudioMode?: ClipSpeedAudioMode | null } +export type TimelineSegment = { recordingSegment?: number; timescale: number; start: number; end: number; name?: string | null; speedAudioMode?: ClipSpeedAudioMode | null; audioMuted?: boolean } export type TranscriptionEngine = "Whisper" | "Parakeet" export type Trigger = "screenshotTaken" | "studioRecordingFinished" | "instantRecordingFinished" | "recordingStarted" | "uploadCompleted" | "videoImported" | "recordingDeleted" export type UpdateChannel = "stable" | "nightly" diff --git a/crates/editor/examples/editor-playback-benchmark.rs b/crates/editor/examples/editor-playback-benchmark.rs index 0d65a29738..96a2d96219 100644 --- a/crates/editor/examples/editor-playback-benchmark.rs +++ b/crates/editor/examples/editor-playback-benchmark.rs @@ -353,6 +353,7 @@ async fn load_recording( timescale: 1.0, name: None, speed_audio_mode: None, + audio_muted: false, }] } StudioRecordingMeta::MultipleSegments { inner } => inner @@ -372,6 +373,7 @@ async fn load_recording( timescale: 1.0, name: None, speed_audio_mode: None, + audio_muted: false, }) }) .collect(), diff --git a/crates/editor/examples/playback-pipeline-benchmark.rs b/crates/editor/examples/playback-pipeline-benchmark.rs index 5601ac3231..ae30fb7394 100644 --- a/crates/editor/examples/playback-pipeline-benchmark.rs +++ b/crates/editor/examples/playback-pipeline-benchmark.rs @@ -276,6 +276,7 @@ async fn load_recording( timescale: 1.0, name: None, speed_audio_mode: None, + audio_muted: false, }] } StudioRecordingMeta::MultipleSegments { inner } => inner @@ -298,6 +299,7 @@ async fn load_recording( timescale: 1.0, name: None, speed_audio_mode: None, + audio_muted: false, }) }) .collect(), diff --git a/crates/editor/src/audio.rs b/crates/editor/src/audio.rs index a655886d66..39e8e524cb 100644 --- a/crates/editor/src/audio.rs +++ b/crates/editor/src/audio.rs @@ -279,21 +279,20 @@ impl AudioRenderer { samples: self.playhead_to_samples(cursor.segment_time), }; + self.render_segment_chunk( + project, + TimelineSource { + source_time: cursor.segment_time, + segment_index: cursor.segment_index, + segment: cursor.segment, + }, + chunk_samples, + written * 2, + &mut ret, + ); + if cursor.segment.timescale == 1.0 { - self.render_current_chunk(project, chunk_samples, written * 2, &mut ret); self.cursor.samples += chunk_samples; - } else { - self.render_speed_audio_chunk( - project, - TimelineSource { - source_time: cursor.segment_time, - segment_index: cursor.segment_index, - segment: cursor.segment, - }, - chunk_samples, - written * 2, - &mut ret, - ); } self.elapsed_samples += chunk_samples; @@ -487,6 +486,10 @@ impl AudioRenderer { out_offset: usize, out: &mut [f32], ) -> usize { + if samples == 0 || project.audio.mute || source.segment.audio_muted { + return 0; + } + if source.segment.timescale == 1.0 { let cursor = source_cursor(source, self.playhead_to_samples(source.source_time)); return self.render_chunk_at_cursor(project, cursor, samples, out_offset, out); @@ -503,9 +506,7 @@ impl AudioRenderer { out_offset: usize, out: &mut [f32], ) -> usize { - if samples == 0 - || project.audio.mute - || source.segment.speed_audio_mode.unwrap_or_default() == ClipSpeedAudioMode::Mute + if source.segment.speed_audio_mode.unwrap_or_default() == ClipSpeedAudioMode::Mute || !source.segment.timescale.is_finite() || !(0.25..=8.0).contains(&source.segment.timescale) { @@ -1891,6 +1892,7 @@ mod tests { end: 1.0, name: None, speed_audio_mode: None, + audio_muted: false, }, TimelineSegment { recording_clip: 0, @@ -1899,6 +1901,7 @@ mod tests { end: 2.0, name: None, speed_audio_mode: None, + audio_muted: false, }, TimelineSegment { recording_clip: 0, @@ -1907,6 +1910,7 @@ mod tests { end: 3.0, name: None, speed_audio_mode: None, + audio_muted: false, }, TimelineSegment { recording_clip: 1, @@ -1915,6 +1919,7 @@ mod tests { end: 1.0, name: None, speed_audio_mode: None, + audio_muted: false, }, TimelineSegment { recording_clip: 1, @@ -1923,6 +1928,7 @@ mod tests { end: 2.0, name: None, speed_audio_mode: None, + audio_muted: false, }, TimelineSegment { recording_clip: 1, @@ -1931,6 +1937,7 @@ mod tests { end: 3.0, name: None, speed_audio_mode: None, + audio_muted: false, }, ], transitions: Vec::new(), @@ -2130,6 +2137,32 @@ mod tests { assert!(renderer.speed_audio_processors.iter().all(Option::is_none)); } + #[test] + fn segment_mute_does_not_mute_adjacent_segments() { + let mut muted = segment(0, 0.0, 1.0, 1.0); + muted.audio_muted = true; + let (_dir, mut renderer, project) = + single_clip_fixture(&[8_000, 16_000], vec![muted, segment(0, 1.0, 2.0, 1.0)]); + + let stream = render_export_audio(&mut renderer, &project, 30, 60); + + assert!(left_at_second(&stream, 0).abs() < 0.0001); + assert!((left_at_second(&stream, 1) - expected(16_000)).abs() < 0.01); + } + + #[test] + fn segment_mute_silences_retimed_audio() { + let mut muted = segment(0, 0.0, 2.0, 2.0); + muted.speed_audio_mode = Some(ClipSpeedAudioMode::MaintainPitch); + muted.audio_muted = true; + let (_dir, mut renderer, project) = single_clip_fixture(&[8_000, 16_000], vec![muted]); + + let stream = render_export_audio(&mut renderer, &project, 30, 30); + + assert!(mean_abs(&stream) < 0.0001); + assert!(renderer.speed_audio_processors.iter().all(Option::is_none)); + } + /// One clip per second `section_values`, on a timeline made of `segments`. fn single_clip_fixture( section_values: &[i16], @@ -2181,6 +2214,7 @@ mod tests { end, name: None, speed_audio_mode: None, + audio_muted: false, } } diff --git a/crates/editor/src/editor_instance.rs b/crates/editor/src/editor_instance.rs index 5214531101..820be3243c 100644 --- a/crates/editor/src/editor_instance.rs +++ b/crates/editor/src/editor_instance.rs @@ -172,6 +172,7 @@ impl EditorInstance { timescale: 1.0, name: None, speed_audio_mode: None, + audio_muted: false, }], _ => { warn!( @@ -205,6 +206,7 @@ impl EditorInstance { timescale: 1.0, name: None, speed_audio_mode: None, + audio_muted: false, }) }) .collect(), diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index 3ced3f0ca1..70b6ca66bb 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -118,6 +118,7 @@ impl ExporterBuilder { timescale: 1.0, name: None, speed_audio_mode: None, + audio_muted: false, }) }) .collect(); diff --git a/crates/project/src/configuration.rs b/crates/project/src/configuration.rs index 1e5234c82d..bd080c7519 100644 --- a/crates/project/src/configuration.rs +++ b/crates/project/src/configuration.rs @@ -747,6 +747,8 @@ pub struct TimelineSegment { pub name: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub speed_audio_mode: Option, + #[serde(default)] + pub audio_muted: bool, } #[derive(Type, Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)] @@ -2239,6 +2241,7 @@ mod tests { end: 4.0, name: None, speed_audio_mode: None, + audio_muted: false, }, TimelineSegment { recording_clip: 1, @@ -2247,6 +2250,7 @@ mod tests { end: 16.0, name: None, speed_audio_mode: None, + audio_muted: false, }, ], transitions, @@ -2282,6 +2286,19 @@ mod tests { ); } + #[test] + fn timeline_segment_defaults_audio_muted_for_existing_projects() { + let segment: TimelineSegment = serde_json::from_value(serde_json::json!({ + "recordingSegment": 0, + "timescale": 1.0, + "start": 0.0, + "end": 1.0 + })) + .unwrap(); + + assert!(!segment.audio_muted); + } + #[test] fn timeline_maps_both_sources_inside_transition() { let timeline = timeline_with_transitions(vec![ClipTransition { diff --git a/crates/recording/src/recovery.rs b/crates/recording/src/recovery.rs index b754bac018..aa0e476ca4 100644 --- a/crates/recording/src/recovery.rs +++ b/crates/recording/src/recovery.rs @@ -1498,6 +1498,7 @@ impl RecoveryManager { timescale: 1.0, name: None, speed_audio_mode: None, + audio_muted: false, }) }) .collect(); diff --git a/crates/recording/src/studio_recording.rs b/crates/recording/src/studio_recording.rs index 1130576913..a62b2f6267 100644 --- a/crates/recording/src/studio_recording.rs +++ b/crates/recording/src/studio_recording.rs @@ -1120,6 +1120,7 @@ async fn stop_recording( timescale: 1.0, name: None, speed_audio_mode: None, + audio_muted: false, }) }) .collect(); diff --git a/crates/recording/src/track_heal.rs b/crates/recording/src/track_heal.rs index 2534ec7cda..8a42d5bcd2 100644 --- a/crates/recording/src/track_heal.rs +++ b/crates/recording/src/track_heal.rs @@ -1052,6 +1052,7 @@ mod tests { end: 202.220711, name: None, speed_audio_mode: None, + audio_muted: false, }], transitions: Vec::new(), zoom_segments: vec![ZoomSegment { @@ -1106,6 +1107,7 @@ mod tests { end: 202.220711, name: None, speed_audio_mode: None, + audio_muted: false, }], transitions: Vec::new(), zoom_segments: vec![], @@ -1180,6 +1182,7 @@ mod tests { end: 100.0, name: None, speed_audio_mode: None, + audio_muted: false, }, TimelineSegment { recording_clip: 1, @@ -1188,6 +1191,7 @@ mod tests { end: 50.0, name: None, speed_audio_mode: None, + audio_muted: false, }, ], transitions: Vec::new(), diff --git a/crates/rendering/src/zoom_spring.rs b/crates/rendering/src/zoom_spring.rs index 540e71c7de..cd980acb62 100644 --- a/crates/rendering/src/zoom_spring.rs +++ b/crates/rendering/src/zoom_spring.rs @@ -1492,6 +1492,7 @@ mod tests { end: 20.0, name: None, speed_audio_mode: None, + audio_muted: false, }], transitions: vec![], zoom_segments: vec![], @@ -1549,6 +1550,7 @@ mod tests { end: 4.0, name: None, speed_audio_mode: None, + audio_muted: false, }, TimelineSegment { recording_clip: 0, @@ -1557,6 +1559,7 @@ mod tests { end: 14.0, name: None, speed_audio_mode: None, + audio_muted: false, }, ], transitions: vec![ClipTransition {