Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changeset/audio_receive_aggregation.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
libwebrtc: patch
livekit-ffi: patch
---

Aggregate explicitly requested unbounded audio receive frames before waking the Rust consumer, preserving PCM, filtered streams, and bounded queue behavior.
21 changes: 21 additions & 0 deletions libwebrtc/src/audio_stream.rs
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,27 @@ pub mod native {
}
}

/// Aggregates unbounded sink frames before waking the consumer.
///
/// Bounded queues and frame durations at or below 10 ms retain their existing behavior.
pub fn with_frame_size(
audio_track: RtcAudioTrack,
sample_rate: i32,
num_channels: i32,
options: NativeAudioStreamOptions,
frame_size_ms: Option<u32>,
) -> Self {
Self {
handle: stream_imp::NativeAudioStream::new_with_frame_size(
audio_track,
sample_rate,
num_channels,
normalize_queue_size_frames(options.queue_size_frames),
frame_size_ms,
),
}
}

pub fn track(&self) -> RtcAudioTrack {
self.handle.track()
}
Expand Down
106 changes: 104 additions & 2 deletions libwebrtc/src/native/audio_stream.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ pub struct NativeAudioStream {
native_sink: SharedPtr<sys_at::ffi::NativeAudioSink>,
audio_track: RtcAudioTrack,
frame_queue: Arc<AudioFrameQueue>,
observer: Arc<AudioTrackObserver>,
}

impl NativeAudioStream {
Expand All @@ -42,9 +43,27 @@ impl NativeAudioStream {
sample_rate: i32,
num_channels: i32,
queue_size_frames: Option<usize>,
) -> Self {
Self::new_with_frame_size(audio_track, sample_rate, num_channels, queue_size_frames, None)
}

/// Creates a sink with optional aggregation for an unbounded queue.
pub fn new_with_frame_size(
audio_track: RtcAudioTrack,
sample_rate: i32,
num_channels: i32,
queue_size_frames: Option<usize>,
frame_size_ms: Option<u32>,
) -> Self {
let frame_queue = Arc::new(AudioFrameQueue::new(queue_size_frames));
let observer = Arc::new(AudioTrackObserver { frame_queue: frame_queue.clone() });
// Bounded queues count original sink frames; keep their overflow semantics unchanged.
let target = frame_size_ms
.filter(|ms| *ms > 10 && queue_size_frames.is_none())
.map(|ms| sample_rate as usize * ms as usize / 1000 * num_channels as usize);
let observer = Arc::new(AudioTrackObserver {
frame_queue: frame_queue.clone(),
batcher: target.map(|samples| Mutex::new(FrameBatcher::new(samples))),
});
let native_sink = sys_at::ffi::new_native_audio_sink(
Box::new(sys_at::AudioSinkWrapper::new(observer.clone())),
sample_rate,
Expand All @@ -54,7 +73,7 @@ impl NativeAudioStream {
let audio = unsafe { sys_at::ffi::media_to_audio(audio_track.sys_handle()) };
audio.add_sink(&native_sink);

Self { native_sink, audio_track, frame_queue }
Self { native_sink, audio_track, frame_queue, observer }
}

pub fn track(&self) -> RtcAudioTrack {
Expand All @@ -66,6 +85,9 @@ impl NativeAudioStream {
audio.remove_sink(&self.native_sink);

self.frame_queue.close();
if let Some(batcher) = &self.observer.batcher {
batcher.lock().pending.clear();
}
}
}

Expand All @@ -85,10 +107,22 @@ impl Stream for NativeAudioStream {

pub struct AudioTrackObserver {
frame_queue: Arc<AudioFrameQueue>,
batcher: Option<Mutex<FrameBatcher>>,
}

impl sys_at::AudioSink for AudioTrackObserver {
fn on_data(&self, data: &[i16], sample_rate: i32, nb_channels: usize, nb_frames: usize) {
if let Some(batcher) = &self.batcher {
batcher.lock().push(data, |data| {
self.frame_queue.push(AudioFrame {
samples_per_channel: (data.len() / nb_channels) as u32,
data: data.into(),
sample_rate: sample_rate as u32,
num_channels: nb_channels as u32,
})
});
return;
}
self.frame_queue.push(AudioFrame {
data: data.to_owned().into(),
sample_rate: sample_rate as u32,
Expand All @@ -98,6 +132,31 @@ impl sys_at::AudioSink for AudioTrackObserver {
}
}

/// Accumulates PCM before waking the async consumer, without changing sink cadence.
struct FrameBatcher {
pending: Vec<i16>,
target: usize,
}

impl FrameBatcher {
/// Allocates the accumulation buffer for one output frame.
fn new(target: usize) -> Self {
Self { pending: Vec::with_capacity(target), target }
}

/// Emits complete frames while retaining an incomplete tail.
fn push(&mut self, mut data: &[i16], mut emit: impl FnMut(Vec<i16>)) {
while !data.is_empty() {
let copied = data.len().min(self.target - self.pending.len());
self.pending.extend_from_slice(&data[..copied]);
data = &data[copied..];
if self.pending.len() == self.target {
emit(std::mem::replace(&mut self.pending, Vec::with_capacity(self.target)));
}
}
}
}

struct AudioFrameQueue {
kind: AudioFrameQueueKind,
closed: AtomicBool,
Expand Down Expand Up @@ -312,3 +371,46 @@ mod tests {
assert_eq!(pop_marker(&queue), None);
}
}

#[cfg(test)]
mod batching_tests {
use super::*;

#[test]
fn arbitrary_chunks_preserve_samples_and_tail() {
for target in [56, 80, 160, 312, 1200, 4800] {
let input: Vec<i16> = (0..15000).map(|i| i as i16).collect();
for chunk in [1, 80, 240, 1023, 4096] {
let mut batcher = FrameBatcher::new(target);
let mut output = Vec::new();
for part in input.chunks(chunk) {
batcher.push(part, |frame| {
assert_eq!(frame.len(), target);
output.extend(frame);
});
}
assert_eq!(output.len(), input.len() / target * target);
output.extend(batcher.pending);
assert_eq!(output, input);
}
}
}

#[test]
fn no_early_delivery_and_observer_releases_tail() {
let queue = Arc::new(AudioFrameQueue::new(None));
let observer = Arc::new(AudioTrackObserver {
frame_queue: queue.clone(),
batcher: Some(Mutex::new(FrameBatcher::new(1200))),
});
use webrtc_sys::audio_track::AudioSink;
observer.on_data(&[1; 240], 24000, 1, 240);
let mut cx = Context::from_waker(std::task::Waker::noop());
assert!(queue.poll_recv(&mut cx).is_pending());
let weak = Arc::downgrade(&observer);
drop(observer);
assert!(weak.upgrade().is_none());
queue.close();
assert!(matches!(queue.poll_recv(&mut cx), Poll::Ready(None)));
}
}
14 changes: 10 additions & 4 deletions livekit-ffi/src/room_apis/server/audio_stream.rs
Original file line number Diff line number Diff line change
Expand Up @@ -142,11 +142,12 @@ impl FfiAudioStream {
// mislabeled as the output rate, dilating it downstream.
let sink_rate =
if filter_session.is_some() { input_sample_rate } else { output_sample_rate };
let native_stream = NativeAudioStream::with_options(
let native_stream = NativeAudioStream::with_frame_size(
rtc_track,
sink_rate as i32,
num_channels as i32,
options,
if filter_session.is_none() { new_stream.frame_size_ms } else { None },
);

let stream = match filter_session {
Expand Down Expand Up @@ -345,8 +346,13 @@ impl FfiAudioStream {
} else {
output_sample_rate
};
let native_stream =
NativeAudioStream::with_options(rtc_track, sink_rate, num_channels, options);
let native_stream = NativeAudioStream::with_frame_size(
rtc_track,
sink_rate,
num_channels,
options,
if audio_filter_session.is_none() { request.frame_size_ms } else { None },
);

let stream = if let Some(session) = audio_filter_session.take() {
let stream = AudioFilterAudioStream::new(
Expand Down Expand Up @@ -447,7 +453,7 @@ impl FfiAudioStream {
}
}

if let Some(target) = target_samples {
if let Some(target) = target_samples.filter(|target| !buf.is_empty() || frame.data.len() != *target) {
buf.extend_from_slice(&frame.data);
while buf.len() >= target {
let frame_data = buf.drain(..target).collect::<Vec<_>>();
Expand Down
Loading