Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/mixer-keep-timed-out-read.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'@livekit/rtc-node': patch
---

Keep the audio of an `AudioMixer` input whose frame arrives after `streamTimeoutMs`, instead of dropping that frame and the ones queued behind it.
35 changes: 34 additions & 1 deletion packages/livekit-rtc/src/audio_mixer.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
//
// SPDX-License-Identifier: Apache-2.0
import { describe, expect, it } from 'vitest';
import { describe, expect, it, vi } from 'vitest';
import { AudioFrame } from './audio_frame.js';
import { AudioMixer } from './audio_mixer.js';

Expand Down Expand Up @@ -266,4 +266,37 @@ describe('AudioMixer', () => {
console.warn = originalWarn;
}
});

it('keeps the frames of a stream that was late once', async () => {
const sampleRate = 48000;
const numChannels = 1;
const samplesPerChannel = 480;
const mixer = new AudioMixer(sampleRate, numChannels, {
blocksize: samplesPerChannel,
streamTimeoutMs: 10,
});

// five frames, the third of which arrives well after the timeout
async function* lateOnce(): AsyncGenerator<AudioFrame> {
for (let i = 0; i < 5; i++) {
if (i === 2) await new Promise((resolve) => setTimeout(resolve, 50));
const data = new Int16Array(numChannels * samplesPerChannel).fill(100);
yield new AudioFrame(data, sampleRate, numChannels, samplesPerChannel);
}
}

const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
try {
mixer.addStream(lateOnce());
let samples = 0;
for await (const frame of mixer) {
samples += frame.data.filter((s) => s === 100).length;
}

expect(warn).toHaveBeenCalled();
expect(samples).toBe(5 * samplesPerChannel);
} finally {
warn.mockRestore();
}
});
});
10 changes: 9 additions & 1 deletion packages/livekit-rtc/src/audio_mixer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ export class AudioMixer {
private streams: Set<AudioStream>;
private buffers: Map<AudioStream, Int16Array>;
private streamIterators: Map<AudioStream, { next(): Promise<IteratorResult<AudioFrame>> }>;
// reads that timed out but are still pending: their frame is the stream's next one
private pendingReads: Map<AudioStream, Promise<IteratorResult<AudioFrame>>>;
private sampleRate: number;
private numChannels: number;
private chunkSize: number;
Expand All @@ -91,6 +93,7 @@ export class AudioMixer {
this.streams = new Set();
this.buffers = new Map();
this.streamIterators = new Map();
this.pendingReads = new Map();
this.sampleRate = sampleRate;
this.numChannels = numChannels;
this.chunkSize =
Expand Down Expand Up @@ -141,6 +144,7 @@ export class AudioMixer {
this.streams.delete(stream);
this.buffers.delete(stream);
this.streamIterators.delete(stream);
this.pendingReads.delete(stream);
}

/**
Expand Down Expand Up @@ -311,9 +315,13 @@ export class AudioMixer {
// Accumulate data until we have at least chunkSize samples
while (buf.length < this.chunkSize * this.numChannels && !exhausted && !this.closed) {
try {
const result = await this.timeoutRace(iterator.next(), this.streamTimeoutMs);
const read = this.pendingReads.get(stream) ?? iterator.next();
this.pendingReads.delete(stream);
const result = await this.timeoutRace(read, this.streamTimeoutMs);

if (result === 'timeout') {
// await this read again next round: a new next() would skip the frame it resolves with
if (this.streams.has(stream)) this.pendingReads.set(stream, read);
console.warn(`AudioMixer: stream timeout after ${this.streamTimeoutMs}ms`);
break;
}
Expand Down
Loading