diff --git a/src/lib/exporter/audioEncoder.test.ts b/src/lib/exporter/audioEncoder.test.ts index f3e8b8f61..89cce529e 100644 --- a/src/lib/exporter/audioEncoder.test.ts +++ b/src/lib/exporter/audioEncoder.test.ts @@ -1,11 +1,19 @@ import { describe, expect, it, vi } from "vitest"; -import { AudioProcessor, softLimitOfflineMixPeaksInPlace } from "./audioEncoder"; +import { AudioProcessor, isWavAudioPath, softLimitOfflineMixPeaksInPlace } from "./audioEncoder"; type OfflineRenderTestHarness = AudioProcessor & { + bulkDecodeFromUrl(url: string, sampleRate: number): Promise; decodeAudioFromUrl(url: string): Promise; getMediaDurationSec(url: string): Promise; loadAudioFileDemuxer(audioPath: string): Promise; + processTrimOnlyAudio( + demuxer: unknown, + muxer: unknown, + sortedTrims: unknown[], + readEndSec?: number, + ): Promise; + streamDecodeFromUrl(url: string): Promise; prepareOfflineRender( videoUrl: string, trimRegions: never[], @@ -55,14 +63,37 @@ function fakeAudioBuffer(channels: Float32Array[]): AudioBuffer { describe("AudioProcessor offline render preparation", () => { it("routes a muted full-track clip through offline audio rendering", async () => { const processor = new AudioProcessor(); - const render = vi.spyOn(processor as unknown as OfflineRenderTestHarness, - "renderAndMuxOfflineAudio").mockResolvedValue(); - const clips = [{ id: "clip", startMs: 0, endMs: 1000, sourceStartMs: 0, speed: 1, muted: true }]; + const render = vi + .spyOn(processor as unknown as OfflineRenderTestHarness, "renderAndMuxOfflineAudio") + .mockResolvedValue(); + const clips = [ + { id: "clip", startMs: 0, endMs: 1000, sourceStartMs: 0, speed: 1, muted: true }, + ]; const muxer = {} as never; - await processor.process(null, muxer, "recording.mp4", [], [], undefined, - [], [], undefined, undefined, clips); - expect(render).toHaveBeenCalledWith("recording.mp4", [], [], [], [], - undefined, undefined, clips, muxer); + await processor.process( + null, + muxer, + "recording.mp4", + [], + [], + undefined, + [], + [], + undefined, + undefined, + clips, + ); + expect(render).toHaveBeenCalledWith( + "recording.mp4", + [], + [], + [], + [], + undefined, + undefined, + clips, + muxer, + ); }); it("rejects a cancelled chunked render instead of returning a partial WAV", async () => { @@ -208,6 +239,101 @@ describe("AudioProcessor offline render preparation", () => { expect(renderAndMuxOfflineAudio).toHaveBeenCalled(); }); + it("detects WAV audio paths and URLs correctly", () => { + expect(isWavAudioPath("C:\\Recordly\\recording.system.wav")).toBe(true); + expect(isWavAudioPath("/tmp/recording.mic.WAV")).toBe(true); + expect(isWavAudioPath("http://localhost:4321/video?path=audio.wav&sig=123")).toBe(true); + expect(isWavAudioPath("file:///tmp/recording.mp4")).toBe(false); + expect(isWavAudioPath("/tmp/recording.mic.m4a")).toBe(false); + expect(isWavAudioPath("/tmp/recording.mic.webm")).toBe(false); + expect(isWavAudioPath(null)).toBe(false); + expect(isWavAudioPath(undefined)).toBe(false); + }); + + it("avoids the single-sidecar fast path for WAV companion audio (such as system audio) and routes to offline rendering", async () => { + const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness; + const loadAudioFileDemuxer = vi.spyOn(processor, "loadAudioFileDemuxer"); + const renderAndMuxOfflineAudio = vi + .spyOn(processor, "renderAndMuxOfflineAudio") + .mockResolvedValue(); + + await processor.process( + null, + {} as never, + "file:///tmp/recording.mp4", + [], + [], + undefined, + [], + ["C:\\Recordly\\recording.system.wav"], + ); + + expect(loadAudioFileDemuxer).not.toHaveBeenCalled(); + expect(renderAndMuxOfflineAudio).toHaveBeenCalledWith( + "file:///tmp/recording.mp4", + [], + [], + [], + ["C:\\Recordly\\recording.system.wav"], + undefined, + undefined, + undefined, + expect.anything(), + ); + }); + + it("falls back to offline rendering when fast sidecar demux fails or outputs no audio", async () => { + const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness; + const mockDemuxer = { destroy: vi.fn() }; + vi.spyOn(processor, "loadAudioFileDemuxer").mockResolvedValue(mockDemuxer); + vi.spyOn(processor, "processTrimOnlyAudio").mockResolvedValue(false); + const renderAndMuxOfflineAudio = vi + .spyOn(processor, "renderAndMuxOfflineAudio") + .mockResolvedValue(); + + await processor.process( + null, + {} as never, + "file:///tmp/recording.mp4", + [], + [], + undefined, + [], + ["/tmp/recording.mic.webm"], + ); + + expect(mockDemuxer.destroy).toHaveBeenCalled(); + expect(renderAndMuxOfflineAudio).toHaveBeenCalledWith( + "file:///tmp/recording.mp4", + [], + [], + [], + ["/tmp/recording.mic.webm"], + undefined, + undefined, + undefined, + expect.anything(), + ); + }); + + it("bypasses streaming decode and uses bulk decode directly for WAV files", async () => { + const processor = new AudioProcessor() as unknown as OfflineRenderTestHarness; + const fakeBuffer = { duration: 10, numberOfChannels: 2 } as AudioBuffer; + const streamDecodeFromUrl = vi.spyOn(processor, "streamDecodeFromUrl"); + const bulkDecodeFromUrl = vi + .spyOn(processor, "bulkDecodeFromUrl") + .mockResolvedValue(fakeBuffer); + + const result = await processor.decodeAudioFromUrl("C:\\Recordly\\recording.system.wav"); + + expect(result).toBe(fakeBuffer); + expect(streamDecodeFromUrl).not.toHaveBeenCalled(); + expect(bulkDecodeFromUrl).toHaveBeenCalledWith( + "C:\\Recordly\\recording.system.wav", + expect.any(Number), + ); + }); + it("soft-limits mixed peaks before encoding or WAV conversion", () => { const samples = new Float32Array([ -1.6, diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index 26050c0a2..1e78f3892 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -10,6 +10,7 @@ import { resolveSourceTrackRoutingPolicy } from "./sourceTrackRoutingPolicy"; import { AudioTranscodeProcessor } from "./audioTranscodeProcessor"; import { hasNonDefaultSourceTrackSettings, + isWavAudioPath, MIN_SPEED_REGION_DELTA_MS, MP4_AUDIO_CODEC, type TrimLikeRegion, @@ -20,6 +21,7 @@ export { getSourceTrackIdFromPath, hasNonDefaultSourceTrackSettings, isAacAudioEncodingSupported, + isWavAudioPath, softLimitOfflineMixPeaksInPlace, } from "./audioProcessorShared"; @@ -142,11 +144,15 @@ export class AudioProcessor extends AudioTranscodeProcessor { const hasTimedCompanionAudio = routingPolicy.playbackPaths.some( (audioPath) => (sourceAudioFallbackStartDelayMsByPath?.[audioPath] ?? 0) > 0, ); + const hasWavCompanionAudio = routingPolicy.playbackPaths.some((audioPath) => + isWavAudioPath(audioPath), + ); const needsSourceAudioMixing = routingPolicy.playbackPaths.length > 1 || (routingPolicy.hasEmbeddedSourceAudio && routingPolicy.playbackPaths.length > 0) || requiresLegacyMacMicSidecarMix || - hasTimedCompanionAudio; + hasTimedCompanionAudio || + hasWavCompanionAudio; // When speed edits, audio regions, or multiple audio sources need mixing, use offline AudioContext pipeline. if ( @@ -173,22 +179,34 @@ export class AudioProcessor extends AudioTranscodeProcessor { // Single sidecar audio with no speed/audio edits: demux directly (skips slow real-time rendering). if (!routingPolicy.hasEmbeddedSourceAudio && routingPolicy.playbackPaths.length === 1) { - const sidecarDemuxer = await this.loadAudioFileDemuxer(routingPolicy.playbackPaths[0]); - if (sidecarDemuxer) { - try { - await this.processTrimOnlyAudio(sidecarDemuxer, muxer, sortedTrims); - } finally { + const sidecarPath = routingPolicy.playbackPaths[0]; + if (!isWavAudioPath(sidecarPath)) { + const sidecarDemuxer = await this.loadAudioFileDemuxer(sidecarPath); + if (sidecarDemuxer) { + let wroteAudio = false; try { - sidecarDemuxer.destroy(); - } catch { - /* cleanup */ + wroteAudio = await this.processTrimOnlyAudio( + sidecarDemuxer, + muxer, + sortedTrims, + ); + } catch (error) { + console.warn("[AudioProcessor] Fast sidecar demux failed:", error); + } finally { + try { + sidecarDemuxer.destroy(); + } catch { + /* cleanup */ + } + } + if (wroteAudio) { + return; } } - return; } - // Fallback to offline rendering if demuxer creation failed + // Fallback to offline rendering if demuxer creation failed, unsupported codec, or no audio was written console.warn( - "[AudioProcessor] Fast sidecar demux failed, falling back to offline rendering", + "[AudioProcessor] Fast sidecar demux unavailable or failed, falling back to offline rendering", ); await this.renderAndMuxOfflineAudio( videoUrl, @@ -235,7 +253,28 @@ export class AudioProcessor extends AudioTranscodeProcessor { } } - await this.processTrimOnlyAudio(demuxer, muxer, sortedTrims, readEndSec); + const wroteTrimOnlyAudio = await this.processTrimOnlyAudio( + demuxer, + muxer, + sortedTrims, + readEndSec, + ); + if (!wroteTrimOnlyAudio && routingPolicy.playbackPaths.length > 0) { + console.warn( + "[AudioProcessor] Main demuxer audio trim failed, falling back to offline rendering for playback paths", + ); + await this.renderAndMuxOfflineAudio( + videoUrl, + sortedTrims, + [], + [], + routingPolicy.playbackPaths, + sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrackSettings, + clipRegions, + muxer, + ); + } } async renderEditedAudioTrack( diff --git a/src/lib/exporter/audioMediaProcessor.ts b/src/lib/exporter/audioMediaProcessor.ts index 3359873a2..13da10124 100644 --- a/src/lib/exporter/audioMediaProcessor.ts +++ b/src/lib/exporter/audioMediaProcessor.ts @@ -1,10 +1,17 @@ import { WebDemuxer } from "web-demuxer"; +import { + DECODE_BACKPRESSURE_LIMIT, + isWavAudioPath, + OFFLINE_AUDIO_SAMPLE_RATE, +} from "./audioProcessorShared"; import { AudioTimelineProcessor } from "./audioTimelineProcessor"; -import { DECODE_BACKPRESSURE_LIMIT, OFFLINE_AUDIO_SAMPLE_RATE } from "./audioProcessorShared"; import { resolveMediaElementSource } from "./localMediaSource"; export class AudioMediaProcessor extends AudioTimelineProcessor { protected async decodeAudioFromUrl(url: string): Promise { + if (isWavAudioPath(url)) { + return this.bulkDecodeFromUrl(url, OFFLINE_AUDIO_SAMPLE_RATE); + } try { const buffer = await this.streamDecodeFromUrl(url); if (buffer) return buffer; @@ -36,6 +43,11 @@ export class AudioMediaProcessor extends AudioTimelineProcessor { return null; // No audio track } + const codecCheck = await AudioDecoder.isConfigSupported(audioConfig); + if (!codecCheck.supported) { + return null; + } + const sampleRate = audioConfig.sampleRate || 48_000; const numChannels = Math.min(audioConfig.numberOfChannels || 2, 2); @@ -137,7 +149,14 @@ export class AudioMediaProcessor extends AudioTimelineProcessor { } if (decoder.state === "configured") { - await decoder.flush(); + try { + await decoder.flush(); + } catch (flushError) { + console.warn( + "[AudioMediaProcessor] Non-fatal audio decoder flush warning:", + flushError, + ); + } } if (decodeError) throw decodeError; } finally { @@ -236,6 +255,9 @@ export class AudioMediaProcessor extends AudioTimelineProcessor { // Get the duration of a media file by loading only its metadata. protected async getMediaDurationSec(url: string): Promise { + if (typeof document === "undefined") { + return 0; + } const source = await resolveMediaElementSource(url); try { const media = document.createElement("video"); diff --git a/src/lib/exporter/audioProcessorShared.ts b/src/lib/exporter/audioProcessorShared.ts index a48062a78..3af9c244c 100644 --- a/src/lib/exporter/audioProcessorShared.ts +++ b/src/lib/exporter/audioProcessorShared.ts @@ -96,6 +96,32 @@ export function getSourceTrackIdFromPath(audioPath: string): SourceTrackId { return "mixed"; } +export function isWavAudioPath(audioPath: string | null | undefined): boolean { + if (!audioPath) { + return false; + } + const normalized = audioPath.toLowerCase().trim(); + if (normalized.endsWith(".wav")) { + return true; + } + try { + const parsedUrl = new URL(audioPath, "http://localhost"); + const pathParam = parsedUrl.searchParams.get("path"); + if (pathParam && pathParam.toLowerCase().trim().endsWith(".wav")) { + return true; + } + if (parsedUrl.pathname.toLowerCase().endsWith(".wav")) { + return true; + } + } catch { + const beforeQuery = normalized.split("?")[0]; + if (beforeQuery.endsWith(".wav")) { + return true; + } + } + return false; +} + export function hasNonDefaultSourceTrackSettings( sourceAudioTrackSettings?: SourceAudioTrackSettings, ) { diff --git a/src/lib/exporter/audioTranscodeProcessor.ts b/src/lib/exporter/audioTranscodeProcessor.ts index 209a280ac..fb95f541f 100644 --- a/src/lib/exporter/audioTranscodeProcessor.ts +++ b/src/lib/exporter/audioTranscodeProcessor.ts @@ -1,5 +1,4 @@ import type { WebDemuxer } from "web-demuxer"; -import { OfflineAudioProcessor } from "./offlineAudioProcessor"; import { AUDIO_BITRATE, DECODE_BACKPRESSURE_LIMIT, @@ -8,6 +7,7 @@ import { type TrimLikeRegion, } from "./audioProcessorShared"; import type { VideoMuxer } from "./muxer"; +import { OfflineAudioProcessor } from "./offlineAudioProcessor"; export class AudioTranscodeProcessor extends OfflineAudioProcessor { protected async processTrimOnlyAudio( @@ -15,19 +15,19 @@ export class AudioTranscodeProcessor extends OfflineAudioProcessor { muxer: VideoMuxer, sortedTrims: TrimLikeRegion[], readEndSec?: number, - ): Promise { + ): Promise { let audioConfig: AudioDecoderConfig; try { audioConfig = (await demuxer.getDecoderConfig("audio")) as AudioDecoderConfig; } catch { console.warn("[AudioProcessor] No audio track found, skipping"); - return; + return false; } const codecCheck = await AudioDecoder.isConfigSupported(audioConfig); if (!codecCheck.supported) { console.warn("[AudioProcessor] Audio codec not supported:", audioConfig.codec); - return; + return false; } const audioStream = @@ -37,7 +37,7 @@ export class AudioTranscodeProcessor extends OfflineAudioProcessor { let sourceTimestampOffsetUs: number | null = null; - await this.transcodeAudioStream( + return await this.transcodeAudioStream( audioStream as ReadableStream, audioConfig, muxer, @@ -68,7 +68,7 @@ export class AudioTranscodeProcessor extends OfflineAudioProcessor { shouldSkipChunk?: (timestampMs: number) => boolean; transformAudioData?: (data: AudioData) => AudioData | null; } = {}, - ): Promise { + ): Promise { const pendingFrames: AudioData[] = []; let decodeError: Error | null = null; let encodeError: Error | null = null; @@ -138,11 +138,13 @@ export class AudioTranscodeProcessor extends OfflineAudioProcessor { const encodeSupport = await AudioEncoder.isConfigSupported(encodeConfig); if (!encodeSupport.supported) { console.warn("[AudioProcessor] AAC encoding not supported, skipping audio"); - return; + return false; } + let wroteAudio = false; const encoder = new AudioEncoder({ output: (chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) => { + wroteAudio = true; pendingMuxing = pendingMuxing .then(async () => { if (this.cancelled) { @@ -223,7 +225,14 @@ export class AudioTranscodeProcessor extends OfflineAudioProcessor { } if (decoder.state === "configured") { - await decoder.flush(); + try { + await decoder.flush(); + } catch (flushError) { + console.warn( + "[AudioTranscodeProcessor] Non-fatal audio decoder flush warning:", + flushError, + ); + } } while (!this.cancelled && (pendingFrames.length > 0 || encoder.encodeQueueSize > 0)) { @@ -237,7 +246,14 @@ export class AudioTranscodeProcessor extends OfflineAudioProcessor { failIfNeeded(); if (encoder.state === "configured") { - await encoder.flush(); + try { + await encoder.flush(); + } catch (flushError) { + console.warn( + "[AudioTranscodeProcessor] Non-fatal audio encoder flush warning:", + flushError, + ); + } } await pendingMuxing; @@ -264,8 +280,10 @@ export class AudioTranscodeProcessor extends OfflineAudioProcessor { } if (this.cancelled) { - return; + return false; } + + return wroteAudio; } // ---------- Offline audio rendering pipeline ---------- diff --git a/src/lib/exporter/offlineAudioProcessor.ts b/src/lib/exporter/offlineAudioProcessor.ts index e1080db3b..8d19fbc5f 100644 --- a/src/lib/exporter/offlineAudioProcessor.ts +++ b/src/lib/exporter/offlineAudioProcessor.ts @@ -104,9 +104,12 @@ export class OfflineAudioProcessor extends AudioMediaProcessor { startDelaySec: number; gain: number; }> = []; - const refDuration = + const mediaDuration = mainBuffer?.duration ?? - (resolvedPlan.playbackPaths.length > 0 ? await this.getMediaDurationSec(videoUrl) : 0); + (resolvedPlan.playbackPaths.length > 0 || audioRegions.length > 0 + ? await this.getMediaDurationSec(videoUrl) + : 0); + const refDuration = mediaDuration; for (const audioPath of resolvedPlan.playbackPaths) { if (this.cancelled) throw new Error("Export cancelled"); const buffer = await this.decodeAudioFromUrl(audioPath); @@ -147,7 +150,10 @@ export class OfflineAudioProcessor extends AudioMediaProcessor { if (mainBufferEntry?.buffer) { sourceDurationSec = mainBufferEntry.buffer.duration; } else if (resolvedPlan.playbackPaths.length > 0 || regionEntries.length > 0) { - sourceDurationSec = await this.getMediaDurationSec(videoUrl); + sourceDurationSec = Math.max( + mediaDuration > 0 ? mediaDuration : 0, + primaryBuffer?.duration ?? 0, + ); } else { sourceDurationSec = primaryBuffer?.duration ?? 0; } @@ -215,6 +221,7 @@ export class OfflineAudioProcessor extends AudioMediaProcessor { let encodeError: Error | null = null; let muxError: Error | null = null; + const getEncodeError = (): Error | null => encodeError; let pendingMuxing = Promise.resolve(); let wroteFirstChunk = false; @@ -249,6 +256,8 @@ export class OfflineAudioProcessor extends AudioMediaProcessor { }); encoder.configure(encodeConfig); + let totalFramesEncoded = 0; + try { await this.renderChunked( prepared, @@ -256,20 +265,35 @@ export class OfflineAudioProcessor extends AudioMediaProcessor { async (rendered, outputOffsetSec) => { if (encodeError) throw encodeError; if (muxError) throw muxError; - await this.feedBufferToEncoder(encoder, rendered, outputOffsetSec); + const frames = await this.feedBufferToEncoder( + encoder, + rendered, + outputOffsetSec, + ); + totalFramesEncoded += frames; }, ); if (encodeError) throw encodeError; if (muxError) throw muxError; - if (encoder.state === "configured") { - await encoder.flush(); + if (totalFramesEncoded > 0 && encoder.state === "configured") { + try { + await encoder.flush(); + } catch (flushError) { + console.warn( + "[OfflineAudioProcessor] Non-fatal audio encoder flush warning:", + flushError, + ); + } } await pendingMuxing; - if (encodeError) throw encodeError; + const finalEncodeError = getEncodeError(); + const isIgnorableFlushError = + wroteFirstChunk && Boolean(finalEncodeError?.message.includes("Flushing error")); + if (finalEncodeError && !isIgnorableFlushError) throw finalEncodeError; if (muxError) throw muxError; } finally { if (encoder.state === "configured") { @@ -425,10 +449,11 @@ export class OfflineAudioProcessor extends AudioMediaProcessor { encoder: AudioEncoder, buffer: AudioBuffer, timestampOffsetSec: number, - ): Promise { + ): Promise { const sampleRate = buffer.sampleRate; const numChannels = buffer.numberOfChannels; const totalFrames = buffer.length; + let encodedFrames = 0; for ( let offset = 0; @@ -454,11 +479,14 @@ export class OfflineAudioProcessor extends AudioMediaProcessor { encoder.encode(audioData); audioData.close(); + encodedFrames += frameCount; while (encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT && !this.cancelled) { await new Promise((r) => setTimeout(r, 1)); } } + + return encodedFrames; } // Decode audio from a URL using streaming WebCodecs decode with bulk fallback.