diff --git a/src/lib/exporter/audioEncoder.test.ts b/src/lib/exporter/audioEncoder.test.ts index f3e8b8f61..7c73b687c 100644 --- a/src/lib/exporter/audioEncoder.test.ts +++ b/src/lib/exporter/audioEncoder.test.ts @@ -6,6 +6,7 @@ type OfflineRenderTestHarness = AudioProcessor & { decodeAudioFromUrl(url: string): Promise; getMediaDurationSec(url: string): Promise; loadAudioFileDemuxer(audioPath: string): Promise; + processTrimOnlyAudio(demuxer: unknown, muxer: unknown, trimRegions: never[]): Promise; prepareOfflineRender( videoUrl: string, trimRegions: never[], @@ -55,14 +56,37 @@ function fakeAudioBuffer(channels: Float32Array[]): AudioBuffer { describe("AudioProcessor offline render preparation", () => { it("routes a muted full-track clip through offline audio rendering", async () => { const processor = new AudioProcessor(); - const render = vi.spyOn(processor as unknown as OfflineRenderTestHarness, - "renderAndMuxOfflineAudio").mockResolvedValue(); - const clips = [{ id: "clip", startMs: 0, endMs: 1000, sourceStartMs: 0, speed: 1, muted: true }]; + const render = vi + .spyOn(processor as unknown as OfflineRenderTestHarness, "renderAndMuxOfflineAudio") + .mockResolvedValue(); + const clips = [ + { id: "clip", startMs: 0, endMs: 1000, sourceStartMs: 0, speed: 1, muted: true }, + ]; const muxer = {} as never; - await processor.process(null, muxer, "recording.mp4", [], [], undefined, - [], [], undefined, undefined, clips); - expect(render).toHaveBeenCalledWith("recording.mp4", [], [], [], [], - undefined, undefined, clips, muxer); + await processor.process( + null, + muxer, + "recording.mp4", + [], + [], + undefined, + [], + [], + undefined, + undefined, + clips, + ); + expect(render).toHaveBeenCalledWith( + "recording.mp4", + [], + [], + [], + [], + undefined, + undefined, + clips, + muxer, + ); }); it("rejects a cancelled chunked render instead of returning a partial WAV", async () => { @@ -208,6 +232,134 @@ describe("AudioProcessor offline render preparation", () => { expect(renderAndMuxOfflineAudio).toHaveBeenCalled(); }); + it("mixes embedded desktop audio with a WAV microphone sidecar instead of demuxing only the sidecar", async () => { + const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness; + const videoUrl = "file:///C:/recordly/recording.mp4"; + const videoPath = "C:/recordly/recording.mp4"; + const micPath = "C:\\recordly\\recording.mic.wav"; + const muxer = {} as never; + const loadAudioFileDemuxer = vi.spyOn(processor, "loadAudioFileDemuxer"); + const renderAndMuxOfflineAudio = vi + .spyOn(processor, "renderAndMuxOfflineAudio") + .mockResolvedValue(); + + await processor.process(null, muxer, videoUrl, [], [], undefined, [], [videoPath, micPath]); + + expect(loadAudioFileDemuxer).not.toHaveBeenCalled(); + expect(renderAndMuxOfflineAudio).toHaveBeenCalledWith( + videoUrl, + [], + [], + [], + [videoPath, micPath], + undefined, + undefined, + undefined, + muxer, + ); + + renderAndMuxOfflineAudio.mockRestore(); + + const mainBuffer = { duration: 10, numberOfChannels: 2 } as AudioBuffer; + const micBuffer = { duration: 9.5, numberOfChannels: 1 } as AudioBuffer; + const decodeAudioFromUrl = vi + .spyOn(processor, "decodeAudioFromUrl") + .mockImplementation(async (url: string) => { + if (url === videoUrl) { + return mainBuffer; + } + if (url === micPath) { + return micBuffer; + } + return null; + }); + vi.spyOn(processor, "getMediaDurationSec").mockResolvedValue(10); + + const prepared = await processor.prepareOfflineRender( + videoUrl, + [], + [], + [], + [videoPath, micPath], + ); + + expect(prepared.mainBufferEntry?.buffer).toBe(mainBuffer); + expect(prepared.companionEntries).toHaveLength(1); + expect(prepared.companionEntries[0]?.buffer).toBe(micBuffer); + expect(decodeAudioFromUrl).toHaveBeenCalledWith(videoUrl); + expect(decodeAudioFromUrl).toHaveBeenCalledWith(micPath); + expect(decodeAudioFromUrl).not.toHaveBeenCalledWith(videoPath); + }); + + it("keeps a single microphone sidecar on the direct demux path when the video has no embedded audio", async () => { + const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness; + const micPath = "C:\\recordly\\recording.mic.wav"; + const loadAudioFileDemuxer = vi + .spyOn(processor, "loadAudioFileDemuxer") + .mockResolvedValue({ destroy: vi.fn() }); + const processTrimOnlyAudio = vi + .spyOn(processor, "processTrimOnlyAudio") + .mockResolvedValue(); + const renderAndMuxOfflineAudio = vi + .spyOn(processor, "renderAndMuxOfflineAudio") + .mockResolvedValue(); + + await processor.process( + null, + {} as never, + "file:///C:/recordly/recording.mp4", + [], + [], + undefined, + [], + [micPath], + ); + + expect(loadAudioFileDemuxer).toHaveBeenCalledWith(micPath); + expect(processTrimOnlyAudio).toHaveBeenCalled(); + expect(renderAndMuxOfflineAudio).not.toHaveBeenCalled(); + }); + + it("does not mix an embedded copy when dedicated system and microphone sidecars are present", async () => { + const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness; + const videoUrl = "file:///C:/recordly/recording.mp4"; + const videoPath = "C:/recordly/recording.mp4"; + const systemPath = "C:\\recordly\\recording.system.wav"; + const micPath = "C:\\recordly\\recording.mic.wav"; + const systemBuffer = { duration: 10, numberOfChannels: 2 } as AudioBuffer; + const micBuffer = { duration: 9.5, numberOfChannels: 1 } as AudioBuffer; + const decodeAudioFromUrl = vi + .spyOn(processor, "decodeAudioFromUrl") + .mockImplementation(async (url: string) => { + if (url === systemPath) { + return systemBuffer; + } + if (url === micPath) { + return micBuffer; + } + return null; + }); + vi.spyOn(processor, "getMediaDurationSec").mockResolvedValue(10); + + const prepared = await processor.prepareOfflineRender( + videoUrl, + [], + [], + [], + [videoPath, systemPath, micPath], + ); + + expect(prepared.mainBufferEntry).toBeNull(); + expect(prepared.companionEntries.map((entry) => entry.buffer)).toEqual([ + systemBuffer, + micBuffer, + ]); + expect(decodeAudioFromUrl).not.toHaveBeenCalledWith(videoUrl); + expect(decodeAudioFromUrl).not.toHaveBeenCalledWith(videoPath); + expect(decodeAudioFromUrl).toHaveBeenCalledWith(systemPath); + expect(decodeAudioFromUrl).toHaveBeenCalledWith(micPath); + }); + it("soft-limits mixed peaks before encoding or WAV conversion", () => { const samples = new Float32Array([ -1.6, diff --git a/src/lib/exporter/modernVideoExporter.fallback.test.ts b/src/lib/exporter/modernVideoExporter.fallback.test.ts index f4d291ab7..00334a3a0 100644 --- a/src/lib/exporter/modernVideoExporter.fallback.test.ts +++ b/src/lib/exporter/modernVideoExporter.fallback.test.ts @@ -1,4 +1,5 @@ -import { afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { AudioProcessor } from "./audioEncoder"; import type { ModernVideoExporter as ModernVideoExporterClass } from "./modernVideoExporter"; const mocks = vi.hoisted(() => { @@ -519,4 +520,222 @@ describe("ModernVideoExporter native fallback routing", () => { }), ); }); + + describe("browser source audio routing", () => { + const videoUrl = "file:///C:/recordly/recording.mp4"; + const micPath = "C:\\recordly\\recording.mic.wav"; + const browserExportConfig = { + videoUrl, + width: 1920, + height: 1080, + frameRate: 30, + bitrate: 8_000_000, + wallpaper: "#101010", + padding: 0, + borderRadius: 0, + backgroundBlur: 0, + shadowIntensity: 0, + showShadow: false, + cropRegion: { x: 0, y: 0, width: 1, height: 1 }, + backendPreference: "webcodecs", + sourceAudioFallbackPaths: [micPath], + } as never; + + let processAudio: ReturnType; + + beforeEach(() => { + vi.stubGlobal("AudioEncoder", { + isConfigSupported: vi.fn(async () => ({ supported: true })), + }); + mocks.streamingDecoderGetEffectiveDuration.mockReturnValue(1); + processAudio = vi.spyOn(AudioProcessor.prototype, "process").mockResolvedValue(); + }); + + afterEach(() => { + processAudio.mockRestore(); + mocks.streamingDecoderLoadMetadata.mockImplementation(async () => mocks.videoInfo); + mocks.streamingDecoderGetDemuxer.mockReturnValue(null); + }); + + function stubEmbeddedDesktopAudio() { + mocks.streamingDecoderLoadMetadata.mockResolvedValue({ + ...mocks.videoInfo, + hasAudio: true, + audioCodec: "aac", + audioSampleRate: 48_000, + }); + } + + function createBrowserExporter(overrides: Record = {}) { + return new ModernVideoExporter({ + ...browserExportConfig, + ...overrides, + } as never) as unknown as { + export: () => Promise<{ success: boolean; blob?: Blob; error?: string }>; + initializeEncoder: () => Promise; + tryStartNativeVideoExport: () => Promise; + finishNativeVideoExport: () => Promise; + }; + } + + async function exportWithWebCodecs( + overrides: Record = {}, + exporter = createBrowserExporter(overrides), + ) { + vi.spyOn(exporter, "initializeEncoder").mockResolvedValue({ + codec: "avc1.640034", + hardwareAcceleration: "prefer-hardware", + }); + return { exporter, result: await exporter.export() }; + } + + it("passes embedded desktop audio and a WAV microphone sidecar into browser export mixing", async () => { + stubEmbeddedDesktopAudio(); + + const { result } = await exportWithWebCodecs(); + + expect(result.success).toBe(true); + expect(processAudio).toHaveBeenCalledTimes(1); + expect(processAudio).toHaveBeenCalledWith( + null, + expect.anything(), + videoUrl, + undefined, + undefined, + undefined, + undefined, + [expect.stringMatching(/recording\.mp4$/i), micPath], + undefined, + undefined, + undefined, + ); + }); + + it("preserves companion delay, edits, and source settings when normalizing browser audio sources", async () => { + stubEmbeddedDesktopAudio(); + const trimRegions = [{ id: "trim-1", startMs: 1_000, endMs: 2_000 }]; + const speedRegions = [{ id: "speed-1", startMs: 3_000, endMs: 4_000, speed: 1.5 }]; + const sourceAudioFallbackStartDelayMsByPath = { [micPath]: 250 }; + const sourceAudioTrackSettings = { + mic: { volume: 0.8, normalize: false }, + system: { volume: 1, normalize: false }, + }; + const clipRegions = [ + { id: "clip", startMs: 0, endMs: 1_000, sourceStartMs: 0, speed: 1, muted: true }, + ]; + + const { result } = await exportWithWebCodecs({ + trimRegions, + speedRegions, + sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrackSettings, + clipRegions, + }); + + expect(result.success).toBe(true); + expect(processAudio).toHaveBeenCalledTimes(1); + expect(processAudio).toHaveBeenCalledWith( + null, + expect.anything(), + videoUrl, + trimRegions, + speedRegions, + undefined, + undefined, + [expect.stringMatching(/recording\.mp4$/i), micPath], + sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrackSettings, + clipRegions, + ); + }); + + it("retries native export once in the browser without dropping the embedded desktop source", async () => { + vi.stubGlobal("navigator", { platform: "Win32" }); + stubEmbeddedDesktopAudio(); + const log = vi.spyOn(console, "error").mockImplementation(() => {}); + const exporter = createBrowserExporter({ backendPreference: "auto" }); + const startNative = vi + .spyOn(exporter, "tryStartNativeVideoExport") + .mockResolvedValue(true); + const initializeEncoder = vi.spyOn(exporter, "initializeEncoder").mockResolvedValue({ + codec: "avc1.640034", + hardwareAcceleration: "prefer-hardware", + }); + vi.spyOn(exporter, "finishNativeVideoExport").mockResolvedValue({ + success: false, + error: "Native finish failed", + }); + + const result = await exporter.export(); + + expect(result.success).toBe(true); + expect(startNative).toHaveBeenCalledTimes(1); + expect(initializeEncoder).toHaveBeenCalledTimes(1); + expect(processAudio).toHaveBeenCalledTimes(1); + expect(processAudio).toHaveBeenCalledWith( + null, + expect.anything(), + videoUrl, + undefined, + undefined, + undefined, + undefined, + [expect.stringMatching(/recording\.mp4$/i), micPath], + undefined, + undefined, + undefined, + ); + expect(log).toHaveBeenCalledWith( + expect.stringContaining("restarting once with WebCodecs"), + ); + }); + + it("does not add the local video twice when it is already present as a Windows path or file URL", async () => { + stubEmbeddedDesktopAudio(); + + const { result } = await exportWithWebCodecs({ + sourceAudioFallbackPaths: [ + "C:\\recordly\\recording.mp4", + "file:///C:/recordly/recording.mp4", + micPath, + ], + }); + + expect(result.success).toBe(true); + expect(processAudio.mock.calls[0]?.[7]).toEqual([ + expect.stringMatching(/recording\.mp4$/i), + micPath, + ]); + }); + + it("keeps a microphone-only sidecar list when the source video has no embedded audio", async () => { + const { result } = await exportWithWebCodecs(); + + expect(result.success).toBe(true); + expect(processAudio).toHaveBeenCalledTimes(1); + expect(processAudio.mock.calls[0]?.[7]).toEqual([micPath]); + }); + + it("keeps embedded-only browser export on the source demuxer without inventing companion paths", async () => { + stubEmbeddedDesktopAudio(); + mocks.streamingDecoderGetDemuxer.mockReturnValue({}); + + const { result } = await exportWithWebCodecs({ + sourceAudioFallbackPaths: undefined, + }); + + expect(result.success).toBe(true); + expect(processAudio).toHaveBeenCalledTimes(1); + expect(processAudio.mock.calls[0]?.[7]).toEqual([]); + }); + + it("skips browser audio processing for a genuinely silent source video", async () => { + const { result } = await exportWithWebCodecs({ + sourceAudioFallbackPaths: undefined, + }); + + expect(result.success).toBe(true); + expect(processAudio).not.toHaveBeenCalled(); + }); + }); }); diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts index 81137b267..2f945ece2 100644 --- a/src/lib/exporter/modernVideoExporter.ts +++ b/src/lib/exporter/modernVideoExporter.ts @@ -815,10 +815,12 @@ export class ModernVideoExporter { !this.cancelled ) { const demuxer = this.streamingDecoder.getDemuxer(); + const sourceAudioFallbackPaths = + this.getNormalizedAudioFallbackPaths(videoInfo); if ( demuxer || (this.config.audioRegions ?? []).length > 0 || - (this.config.sourceAudioFallbackPaths ?? []).length > 0 + sourceAudioFallbackPaths.length > 0 ) { this.audioProcessor = new AudioProcessor(); this.audioProcessor.setOnProgress((progress) => { @@ -835,7 +837,7 @@ export class ModernVideoExporter { this.config.speedRegions, undefined, this.config.audioRegions, - this.config.sourceAudioFallbackPaths, + sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, this.config.sourceAudioTrackSettings, this.config.clipRegions, @@ -1416,7 +1418,7 @@ export class ModernVideoExporter { return buildNativeStaticLayoutTimelineSegments(sourceSegments); } - private getNativeAudioFallbackPaths(videoInfo: DecodedVideoInfo): string[] { + private getNormalizedAudioFallbackPaths(videoInfo: DecodedVideoInfo): string[] { const sourceAudioFallbackPaths = (this.config.sourceAudioFallbackPaths ?? []).filter( (audioPath) => typeof audioPath === "string" && audioPath.trim().length > 0, ); @@ -1455,7 +1457,7 @@ export class ModernVideoExporter { private buildNativeAudioPlan(videoInfo: DecodedVideoInfo): NativeAudioPlan { const speedRegions = this.config.speedRegions ?? []; const audioRegions = this.config.audioRegions ?? []; - const sourceAudioFallbackPaths = this.getNativeAudioFallbackPaths(videoInfo); + const sourceAudioFallbackPaths = this.getNormalizedAudioFallbackPaths(videoInfo); const hasTimedSourceAudioFallback = sourceAudioFallbackPaths.some( (audioPath) => (this.config.sourceAudioFallbackStartDelayMsByPath?.[audioPath] ?? 0) > 0,