diff --git a/.changeset/mixer-keep-timed-out-read.md b/.changeset/mixer-keep-timed-out-read.md new file mode 100644 index 00000000..45b68fec --- /dev/null +++ b/.changeset/mixer-keep-timed-out-read.md @@ -0,0 +1,5 @@ +--- +'@livekit/rtc-node': patch +--- + +Keep the audio of an `AudioMixer` input whose frame arrives after `streamTimeoutMs`, instead of dropping that frame and the ones queued behind it. diff --git a/packages/livekit-rtc/src/audio_mixer.test.ts b/packages/livekit-rtc/src/audio_mixer.test.ts index d4dfbfaa..f91434c0 100644 --- a/packages/livekit-rtc/src/audio_mixer.test.ts +++ b/packages/livekit-rtc/src/audio_mixer.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: 2025 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { AudioFrame } from './audio_frame.js'; import { AudioMixer } from './audio_mixer.js'; @@ -266,4 +266,37 @@ describe('AudioMixer', () => { console.warn = originalWarn; } }); + + it('keeps the frames of a stream that was late once', async () => { + const sampleRate = 48000; + const numChannels = 1; + const samplesPerChannel = 480; + const mixer = new AudioMixer(sampleRate, numChannels, { + blocksize: samplesPerChannel, + streamTimeoutMs: 10, + }); + + // five frames, the third of which arrives well after the timeout + async function* lateOnce(): AsyncGenerator { + for (let i = 0; i < 5; i++) { + if (i === 2) await new Promise((resolve) => setTimeout(resolve, 50)); + const data = new Int16Array(numChannels * samplesPerChannel).fill(100); + yield new AudioFrame(data, sampleRate, numChannels, samplesPerChannel); + } + } + + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + try { + mixer.addStream(lateOnce()); + let samples = 0; + for await (const frame of mixer) { + samples += frame.data.filter((s) => s === 100).length; + } + + expect(warn).toHaveBeenCalled(); + expect(samples).toBe(5 * samplesPerChannel); + } finally { + warn.mockRestore(); + } + }); }); diff --git a/packages/livekit-rtc/src/audio_mixer.ts b/packages/livekit-rtc/src/audio_mixer.ts index e2aaef84..00a415fd 100644 --- a/packages/livekit-rtc/src/audio_mixer.ts +++ b/packages/livekit-rtc/src/audio_mixer.ts @@ -70,6 +70,8 @@ export class AudioMixer { private streams: Set; private buffers: Map; private streamIterators: Map> }>; + // reads that timed out but are still pending: their frame is the stream's next one + private pendingReads: Map>>; private sampleRate: number; private numChannels: number; private chunkSize: number; @@ -91,6 +93,7 @@ export class AudioMixer { this.streams = new Set(); this.buffers = new Map(); this.streamIterators = new Map(); + this.pendingReads = new Map(); this.sampleRate = sampleRate; this.numChannels = numChannels; this.chunkSize = @@ -141,6 +144,7 @@ export class AudioMixer { this.streams.delete(stream); this.buffers.delete(stream); this.streamIterators.delete(stream); + this.pendingReads.delete(stream); } /** @@ -311,9 +315,13 @@ export class AudioMixer { // Accumulate data until we have at least chunkSize samples while (buf.length < this.chunkSize * this.numChannels && !exhausted && !this.closed) { try { - const result = await this.timeoutRace(iterator.next(), this.streamTimeoutMs); + const read = this.pendingReads.get(stream) ?? iterator.next(); + this.pendingReads.delete(stream); + const result = await this.timeoutRace(read, this.streamTimeoutMs); if (result === 'timeout') { + // await this read again next round: a new next() would skip the frame it resolves with + if (this.streams.has(stream)) this.pendingReads.set(stream, read); console.warn(`AudioMixer: stream timeout after ${this.streamTimeoutMs}ms`); break; }