diff --git a/.changeset/audio_receive_aggregation.md b/.changeset/audio_receive_aggregation.md new file mode 100644 index 0000000000..ddb2ca215a --- /dev/null +++ b/.changeset/audio_receive_aggregation.md @@ -0,0 +1,6 @@ +--- +libwebrtc: patch +livekit-ffi: patch +--- + +Aggregate explicitly requested unbounded audio receive frames before waking the Rust consumer, preserving PCM, filtered streams, and bounded queue behavior. diff --git a/libwebrtc/src/audio_stream.rs b/libwebrtc/src/audio_stream.rs index 00677c421c..41a5ced693 100644 --- a/libwebrtc/src/audio_stream.rs +++ b/libwebrtc/src/audio_stream.rs @@ -86,6 +86,27 @@ pub mod native { } } + /// Aggregates unbounded sink frames before waking the consumer. + /// + /// Bounded queues and frame durations at or below 10 ms retain their existing behavior. + pub fn with_frame_size( + audio_track: RtcAudioTrack, + sample_rate: i32, + num_channels: i32, + options: NativeAudioStreamOptions, + frame_size_ms: Option, + ) -> Self { + Self { + handle: stream_imp::NativeAudioStream::new_with_frame_size( + audio_track, + sample_rate, + num_channels, + normalize_queue_size_frames(options.queue_size_frames), + frame_size_ms, + ), + } + } + pub fn track(&self) -> RtcAudioTrack { self.handle.track() } diff --git a/libwebrtc/src/native/audio_stream.rs b/libwebrtc/src/native/audio_stream.rs index deb72918f0..8b87388fb4 100644 --- a/libwebrtc/src/native/audio_stream.rs +++ b/libwebrtc/src/native/audio_stream.rs @@ -34,6 +34,7 @@ pub struct NativeAudioStream { native_sink: SharedPtr, audio_track: RtcAudioTrack, frame_queue: Arc, + observer: Arc, } impl NativeAudioStream { @@ -42,9 +43,27 @@ impl NativeAudioStream { sample_rate: i32, num_channels: i32, queue_size_frames: Option, + ) -> Self { + Self::new_with_frame_size(audio_track, sample_rate, num_channels, queue_size_frames, None) + } + + /// Creates a sink with optional aggregation for an unbounded queue. + pub fn new_with_frame_size( + audio_track: RtcAudioTrack, + sample_rate: i32, + num_channels: i32, + queue_size_frames: Option, + frame_size_ms: Option, ) -> Self { let frame_queue = Arc::new(AudioFrameQueue::new(queue_size_frames)); - let observer = Arc::new(AudioTrackObserver { frame_queue: frame_queue.clone() }); + // Bounded queues count original sink frames; keep their overflow semantics unchanged. + let target = frame_size_ms + .filter(|ms| *ms > 10 && queue_size_frames.is_none()) + .map(|ms| sample_rate as usize * ms as usize / 1000 * num_channels as usize); + let observer = Arc::new(AudioTrackObserver { + frame_queue: frame_queue.clone(), + batcher: target.map(|samples| Mutex::new(FrameBatcher::new(samples))), + }); let native_sink = sys_at::ffi::new_native_audio_sink( Box::new(sys_at::AudioSinkWrapper::new(observer.clone())), sample_rate, @@ -54,7 +73,7 @@ impl NativeAudioStream { let audio = unsafe { sys_at::ffi::media_to_audio(audio_track.sys_handle()) }; audio.add_sink(&native_sink); - Self { native_sink, audio_track, frame_queue } + Self { native_sink, audio_track, frame_queue, observer } } pub fn track(&self) -> RtcAudioTrack { @@ -66,6 +85,9 @@ impl NativeAudioStream { audio.remove_sink(&self.native_sink); self.frame_queue.close(); + if let Some(batcher) = &self.observer.batcher { + batcher.lock().pending.clear(); + } } } @@ -85,10 +107,22 @@ impl Stream for NativeAudioStream { pub struct AudioTrackObserver { frame_queue: Arc, + batcher: Option>, } impl sys_at::AudioSink for AudioTrackObserver { fn on_data(&self, data: &[i16], sample_rate: i32, nb_channels: usize, nb_frames: usize) { + if let Some(batcher) = &self.batcher { + batcher.lock().push(data, |data| { + self.frame_queue.push(AudioFrame { + samples_per_channel: (data.len() / nb_channels) as u32, + data: data.into(), + sample_rate: sample_rate as u32, + num_channels: nb_channels as u32, + }) + }); + return; + } self.frame_queue.push(AudioFrame { data: data.to_owned().into(), sample_rate: sample_rate as u32, @@ -98,6 +132,31 @@ impl sys_at::AudioSink for AudioTrackObserver { } } +/// Accumulates PCM before waking the async consumer, without changing sink cadence. +struct FrameBatcher { + pending: Vec, + target: usize, +} + +impl FrameBatcher { + /// Allocates the accumulation buffer for one output frame. + fn new(target: usize) -> Self { + Self { pending: Vec::with_capacity(target), target } + } + + /// Emits complete frames while retaining an incomplete tail. + fn push(&mut self, mut data: &[i16], mut emit: impl FnMut(Vec)) { + while !data.is_empty() { + let copied = data.len().min(self.target - self.pending.len()); + self.pending.extend_from_slice(&data[..copied]); + data = &data[copied..]; + if self.pending.len() == self.target { + emit(std::mem::replace(&mut self.pending, Vec::with_capacity(self.target))); + } + } + } +} + struct AudioFrameQueue { kind: AudioFrameQueueKind, closed: AtomicBool, @@ -312,3 +371,46 @@ mod tests { assert_eq!(pop_marker(&queue), None); } } + +#[cfg(test)] +mod batching_tests { + use super::*; + + #[test] + fn arbitrary_chunks_preserve_samples_and_tail() { + for target in [56, 80, 160, 312, 1200, 4800] { + let input: Vec = (0..15000).map(|i| i as i16).collect(); + for chunk in [1, 80, 240, 1023, 4096] { + let mut batcher = FrameBatcher::new(target); + let mut output = Vec::new(); + for part in input.chunks(chunk) { + batcher.push(part, |frame| { + assert_eq!(frame.len(), target); + output.extend(frame); + }); + } + assert_eq!(output.len(), input.len() / target * target); + output.extend(batcher.pending); + assert_eq!(output, input); + } + } + } + + #[test] + fn no_early_delivery_and_observer_releases_tail() { + let queue = Arc::new(AudioFrameQueue::new(None)); + let observer = Arc::new(AudioTrackObserver { + frame_queue: queue.clone(), + batcher: Some(Mutex::new(FrameBatcher::new(1200))), + }); + use webrtc_sys::audio_track::AudioSink; + observer.on_data(&[1; 240], 24000, 1, 240); + let mut cx = Context::from_waker(std::task::Waker::noop()); + assert!(queue.poll_recv(&mut cx).is_pending()); + let weak = Arc::downgrade(&observer); + drop(observer); + assert!(weak.upgrade().is_none()); + queue.close(); + assert!(matches!(queue.poll_recv(&mut cx), Poll::Ready(None))); + } +} diff --git a/livekit-ffi/src/room_apis/server/audio_stream.rs b/livekit-ffi/src/room_apis/server/audio_stream.rs index 1bcb21f645..3f7d0c248a 100644 --- a/livekit-ffi/src/room_apis/server/audio_stream.rs +++ b/livekit-ffi/src/room_apis/server/audio_stream.rs @@ -142,11 +142,12 @@ impl FfiAudioStream { // mislabeled as the output rate, dilating it downstream. let sink_rate = if filter_session.is_some() { input_sample_rate } else { output_sample_rate }; - let native_stream = NativeAudioStream::with_options( + let native_stream = NativeAudioStream::with_frame_size( rtc_track, sink_rate as i32, num_channels as i32, options, + if filter_session.is_none() { new_stream.frame_size_ms } else { None }, ); let stream = match filter_session { @@ -345,8 +346,13 @@ impl FfiAudioStream { } else { output_sample_rate }; - let native_stream = - NativeAudioStream::with_options(rtc_track, sink_rate, num_channels, options); + let native_stream = NativeAudioStream::with_frame_size( + rtc_track, + sink_rate, + num_channels, + options, + if audio_filter_session.is_none() { request.frame_size_ms } else { None }, + ); let stream = if let Some(session) = audio_filter_session.take() { let stream = AudioFilterAudioStream::new( @@ -447,7 +453,7 @@ impl FfiAudioStream { } } - if let Some(target) = target_samples { + if let Some(target) = target_samples.filter(|target| !buf.is_empty() || frame.data.len() != *target) { buf.extend_from_slice(&frame.data); while buf.len() >= target { let frame_data = buf.drain(..target).collect::>();