From 8c35899b107d828367f161154b5d7f543cb00ddb Mon Sep 17 00:00:00 2001 From: shijing xian Date: Fri, 6 Nov 2026 12:37:13 -0800 Subject: [PATCH] Surface dropped video frames and reuse frame conversion buffers A decoded video frame can be dropped after `framesDecoded` counts it and before the application sees it, with every WebRTC stat reading zero. Three changes make that visible and make it less likely. `VideoFrameReceived` gains `dropped_frames`, the cumulative count of frames `NativeVideoStream` discarded because the FFI forwarding task fell behind the WebRTC sink. The count already existed but was only ever written: it reached an application as a log line on the first drop and every hundredth after it. `NativeVideoStream::dropped_frames()` exposes the same counter to Rust. The libwebrtc log sink dropped each message's severity and logged everything at `Debug`, so libwebrtc warnings and errors were filtered out by default in every binding. Severity is now mapped through. Verbose and info stay noisy and move to `Trace` and `Debug`; warning and error become visible. Frame conversion drew a fresh frame-sized buffer per frame and freed it when the binding released the handle. A 1628x1920 I420 frame is 4.5 MB, so a 30 fps stream churned 134 MB/s of short-lived allocations that large, and the Windows heap sends anything over ~512 KB straight to `VirtualAlloc`, decommitting it on free. `cvt` now writes into a buffer the caller owns, and each stream keeps its own set to hand back. A buffer may only be written while nothing else holds it, and `Arc` is what knows: the FFI handle map holds a clone for as long as the binding has the frame, so a strong count of 1 means the buffer came back. `Arc::get_mut` turns "is the binding done?" into a question the borrow checker answers, rather than one the code has to get right. Nothing depends on the binding releasing promptly -- when no buffer is free the stream adds one, up to a cap, past which it allocates per frame as before. A stream therefore keeps one buffer for a binding that copies and releases immediately, and two for a zero-copy binding holding the current frame while the next arrives. Co-Authored-By: Claude Opus 5 (1M context) --- .changeset/harden_incoming_data_streams.md | 9 + .../network_quality_start_bitrate_hints.md | 6 + .../platform_audio_invalid_device_id_test.md | 5 + ...trim_uniffi_bindgen_from_uniffi_runtime.md | 6 + .../use_typed_platform_audio_device_ids.md | 10 + ..._frame_drop_visibility_and_buffer_reuse.md | 6 + .../src/native/peer_connection_factory.rs | 51 +++- libwebrtc/src/native/video_stream.rs | 42 ++++ libwebrtc/src/video_stream.rs | 11 + .../proto/video_frame_pb.d.ts | 17 ++ .../proto/video_frame_pb.js | 1 + livekit-ffi/protocol/video_frame.proto | 13 ++ .../src/room_apis/server/colorcvt/cvtimpl.rs | 199 ++++++++-------- .../src/room_apis/server/colorcvt/mod.rs | 135 ++++++++++- livekit-ffi/src/room_apis/server/mod.rs | 2 +- livekit-ffi/src/room_apis/server/requests.rs | 8 +- .../src/room_apis/server/video_stream.rs | 221 +++++++++++++++++- 17 files changed, 631 insertions(+), 111 deletions(-) create mode 100644 .changeset/harden_incoming_data_streams.md create mode 100644 .changeset/network_quality_start_bitrate_hints.md create mode 100644 .changeset/platform_audio_invalid_device_id_test.md create mode 100644 .changeset/trim_uniffi_bindgen_from_uniffi_runtime.md create mode 100644 .changeset/use_typed_platform_audio_device_ids.md create mode 100644 .changeset/video_frame_drop_visibility_and_buffer_reuse.md diff --git a/.changeset/harden_incoming_data_streams.md b/.changeset/harden_incoming_data_streams.md new file mode 100644 index 0000000000..3389354930 --- /dev/null +++ b/.changeset/harden_incoming_data_streams.md @@ -0,0 +1,9 @@ +--- +livekit-data-stream: patch +livekit-rpc: patch +livekit: patch +livekit-ffi: patch +livekit-uniffi: patch +--- + +Reject incoming stream packets from other participants, enforce decompression limits before buffering output, and prevent received files from overwriting existing files or following destination symlinks. diff --git a/.changeset/network_quality_start_bitrate_hints.md b/.changeset/network_quality_start_bitrate_hints.md new file mode 100644 index 0000000000..00feb16288 --- /dev/null +++ b/.changeset/network_quality_start_bitrate_hints.md @@ -0,0 +1,6 @@ +--- +livekit: patch +libwebrtc: patch +--- + +Add publish-time network quality signals for start bitrate hints. diff --git a/.changeset/platform_audio_invalid_device_id_test.md b/.changeset/platform_audio_invalid_device_id_test.md new file mode 100644 index 0000000000..f4e0d0be68 --- /dev/null +++ b/.changeset/platform_audio_invalid_device_id_test.md @@ -0,0 +1,5 @@ +--- +livekit: patch +--- + +test: cover `DeviceNotFound` for invalid PlatformAudio device IDs. diff --git a/.changeset/trim_uniffi_bindgen_from_uniffi_runtime.md b/.changeset/trim_uniffi_bindgen_from_uniffi_runtime.md new file mode 100644 index 0000000000..592b1be6d8 --- /dev/null +++ b/.changeset/trim_uniffi_bindgen_from_uniffi_runtime.md @@ -0,0 +1,6 @@ +--- +livekit-uniffi: patch +livekit-datatrack: patch +--- + +Stop enabling UniFFI bindgen CLI dependencies in runtime UniFFI builds. diff --git a/.changeset/use_typed_platform_audio_device_ids.md b/.changeset/use_typed_platform_audio_device_ids.md new file mode 100644 index 0000000000..33154b65d1 --- /dev/null +++ b/.changeset/use_typed_platform_audio_device_ids.md @@ -0,0 +1,10 @@ +--- +livekit: minor +livekit-ffi: patch +--- + +refactor: route platform audio device selection through typed device IDs in Rust and FFI. + +fix: narrow hidden platform audio index/count helpers to internal APIs and use `usize` for public device indices. + +fix: return `DeviceNotFound` when selecting an unavailable platform audio device ID. diff --git a/.changeset/video_frame_drop_visibility_and_buffer_reuse.md b/.changeset/video_frame_drop_visibility_and_buffer_reuse.md new file mode 100644 index 0000000000..7b32233d46 --- /dev/null +++ b/.changeset/video_frame_drop_visibility_and_buffer_reuse.md @@ -0,0 +1,6 @@ +--- +libwebrtc: patch +livekit-ffi: patch +--- + +Report dropped video frames to the application via `VideoFrameReceived.dropped_frames`, keep libwebrtc log messages at their own severity instead of flattening every one to debug, and reuse video frame conversion buffers rather than allocating one per frame. diff --git a/libwebrtc/src/native/peer_connection_factory.rs b/libwebrtc/src/native/peer_connection_factory.rs index 8f1cd4993b..eb5c493382 100644 --- a/libwebrtc/src/native/peer_connection_factory.rs +++ b/libwebrtc/src/native/peer_connection_factory.rs @@ -35,12 +35,35 @@ lazy_static! { static ref LOG_SINK: Mutex>> = Default::default(); } +/// Maps a libwebrtc severity onto the matching `log` level. +/// +/// Warning and error move up so consumers can see them. Verbose and info both +/// move down a step -- libwebrtc logs both very heavily, and the sink is +/// installed at `LS_VERBOSE`, so mapping info to `Info` would flood every +/// binding that shows it. Keeping info at `Debug` preserves today's behaviour +/// for the noisy levels while making the two that matter visible. +fn log_level(severity: sys_rtc::ffi::LoggingSeverity) -> Option { + match severity { + sys_rtc::ffi::LoggingSeverity::Verbose => Some(log::Level::Trace), + sys_rtc::ffi::LoggingSeverity::Info => Some(log::Level::Debug), + sys_rtc::ffi::LoggingSeverity::Warning => Some(log::Level::Warn), + sys_rtc::ffi::LoggingSeverity::Error => Some(log::Level::Error), + _ => None, + } +} + fn ensure_log_sink() { let mut log_sink = LOG_SINK.lock(); if log_sink.is_none() { - *log_sink = Some(sys_rtc::ffi::new_log_sink(|msg, _| { + *log_sink = Some(sys_rtc::ffi::new_log_sink(|msg, severity| { + // Every message used to arrive as Debug regardless of severity, so + // libwebrtc warnings and errors were discarded by any consumer + // filtering at Info -- which is every binding's default. + let Some(level) = log_level(severity) else { + return; + }; let msg = msg.strip_suffix("\r\n").or(msg.strip_suffix('\n')).unwrap_or(&msg); - log::debug!(target: "libwebrtc", "{}", msg); + log::log!(target: "libwebrtc", level, "{}", msg); })); } } @@ -351,6 +374,7 @@ impl PeerConnectionFactory { mod tests { use super::*; use std::sync::Mutex; + use webrtc_sys::webrtc::ffi::LoggingSeverity; static TEST_MUTEX: Mutex<()> = Mutex::new(()); @@ -373,4 +397,27 @@ mod tests { assert_eq!(factory.adm_recording_enabled(), !initial_recording); factory.set_adm_recording_enabled(initial_recording); } + + /// Warnings and errors used to arrive as `Debug` and were filtered out by + /// default in every binding; these pin the mapping that fixed that. + #[test] + fn libwebrtc_warnings_and_errors_map_to_visible_levels() { + assert_eq!(log_level(LoggingSeverity::Warning), Some(log::Level::Warn)); + assert_eq!(log_level(LoggingSeverity::Error), Some(log::Level::Error)); + } + + #[test] + fn the_noisy_levels_stay_below_info() { + // libwebrtc logs both heavily and the sink is installed at LS_VERBOSE, + // so neither may reach a level a binding shows by default. + for severity in [LoggingSeverity::Verbose, LoggingSeverity::Info] { + let level = log_level(severity).expect("still forwarded"); + assert!(level > log::Level::Info, "{severity:?} mapped to {level:?}"); + } + } + + #[test] + fn severity_none_is_dropped() { + assert_eq!(log_level(LoggingSeverity::None), None); + } } diff --git a/libwebrtc/src/native/video_stream.rs b/libwebrtc/src/native/video_stream.rs index 8bb4081284..f285a6314b 100644 --- a/libwebrtc/src/native/video_stream.rs +++ b/libwebrtc/src/native/video_stream.rs @@ -78,6 +78,12 @@ impl NativeVideoStream { self.video_track.clone() } + /// Frames this stream has dropped because the consumer did not keep up + /// with the WebRTC sink, counted since the stream was created. + pub fn dropped_frames(&self) -> u64 { + self.frame_queue.dropped_frames() + } + pub fn close(&mut self) { let video = unsafe { sys_vt::ffi::media_to_video(self.video_track.sys_handle()) }; video.remove_sink(&self.native_sink); @@ -289,6 +295,10 @@ impl VideoFrameQueue { } } + fn dropped_frames(&self) -> u64 { + self.dropped_frames.load(Ordering::Relaxed) + } + fn record_drop(&self) { let dropped_frames = self.dropped_frames.fetch_add(1, Ordering::Relaxed) + 1; if dropped_frames == 1 || dropped_frames % 100 == 0 { @@ -361,6 +371,38 @@ mod tests { assert_eq!(pop_timestamp(&queue), None); } + #[test] + fn dropped_frames_is_readable_and_cumulative() { + let queue = VideoFrameQueue::new(Some(1)); + + assert_eq!(queue.dropped_frames(), 0); + + queue.push(test_frame(1)); + assert_eq!(queue.dropped_frames(), 0, "the first frame fits"); + + queue.push(test_frame(2)); + assert_eq!(queue.dropped_frames(), 1); + + queue.push(test_frame(3)); + assert_eq!(queue.dropped_frames(), 2, "the count accumulates"); + + // Draining does not reset it: the counter reports losses over the + // stream's whole life, so a consumer can diff it between frames. + assert_eq!(pop_timestamp(&queue), Some(3)); + assert_eq!(queue.dropped_frames(), 2); + } + + #[test] + fn an_unbounded_queue_never_reports_drops() { + let queue = VideoFrameQueue::new(None); + + for timestamp_us in 1..=64 { + queue.push(test_frame(timestamp_us)); + } + + assert_eq!(queue.dropped_frames(), 0); + } + #[test] fn unbounded_queue_retains_all_frames() { let queue = VideoFrameQueue::new(None); diff --git a/libwebrtc/src/video_stream.rs b/libwebrtc/src/video_stream.rs index 16fdeefb60..e64773263b 100644 --- a/libwebrtc/src/video_stream.rs +++ b/libwebrtc/src/video_stream.rs @@ -97,6 +97,17 @@ pub mod native { self.handle.track() } + /// Frames this stream has dropped because the consumer did not keep up + /// with the WebRTC sink, counted since the stream was created. + /// + /// A stream built with `queue_size_frames: Some(0)` never drops, so + /// this stays at zero. For any bounded queue a rising count means the + /// consumer is too slow for the incoming frame rate; the frames are + /// gone and no WebRTC stat reports them. + pub fn dropped_frames(&self) -> u64 { + self.handle.dropped_frames() + } + pub fn close(&mut self) { self.handle.close(); } diff --git a/livekit-ffi-node-bindings/proto/video_frame_pb.d.ts b/livekit-ffi-node-bindings/proto/video_frame_pb.d.ts index 8b9f9f96a8..2b31514b2f 100644 --- a/livekit-ffi-node-bindings/proto/video_frame_pb.d.ts +++ b/livekit-ffi-node-bindings/proto/video_frame_pb.d.ts @@ -1065,6 +1065,23 @@ export declare class VideoFrameReceived extends Message { */ metadata?: FrameMetadata; + /** + * Frames this stream has dropped since it was created, because the FFI + * forwarding task did not keep up with the WebRTC sink. + * + * Dropped frames are decoded but never delivered. No WebRTC stat reports + * them -- `framesDecoded` counts them and `framesDropped` does not -- so + * this counter is the only way an application can observe the loss. It is + * cumulative and monotonic: compare it against the previous frame's value + * to find how many were lost in between. + * + * Always zero for a stream created with `queue_size_frames = 0`, which is + * unbounded and never drops. + * + * @generated from field: optional uint64 dropped_frames = 5; + */ + droppedFrames?: bigint; + constructor(data?: PartialMessage); static readonly runtime: typeof proto2; diff --git a/livekit-ffi-node-bindings/proto/video_frame_pb.js b/livekit-ffi-node-bindings/proto/video_frame_pb.js index d10134ea72..f6ce0fc751 100644 --- a/livekit-ffi-node-bindings/proto/video_frame_pb.js +++ b/livekit-ffi-node-bindings/proto/video_frame_pb.js @@ -420,6 +420,7 @@ const VideoFrameReceived = /*@__PURE__*/ proto2.makeMessageType( { no: 2, name: "timestamp_us", kind: "scalar", T: 3 /* ScalarType.INT64 */, req: true }, { no: 3, name: "rotation", kind: "enum", T: proto2.getEnumType(VideoRotation), req: true }, { no: 4, name: "metadata", kind: "message", T: FrameMetadata, opt: true }, + { no: 5, name: "dropped_frames", kind: "scalar", T: 4 /* ScalarType.UINT64 */, opt: true }, ], ); diff --git a/livekit-ffi/protocol/video_frame.proto b/livekit-ffi/protocol/video_frame.proto index 57ed268816..a86e34f0d0 100644 --- a/livekit-ffi/protocol/video_frame.proto +++ b/livekit-ffi/protocol/video_frame.proto @@ -234,6 +234,19 @@ message VideoFrameReceived { required int64 timestamp_us = 2; // In microseconds required VideoRotation rotation = 3; optional FrameMetadata metadata = 4; + + // Frames this stream has dropped since it was created, because the FFI + // forwarding task did not keep up with the WebRTC sink. + // + // Dropped frames are decoded but never delivered. No WebRTC stat reports + // them -- `framesDecoded` counts them and `framesDropped` does not -- so + // this counter is the only way an application can observe the loss. It is + // cumulative and monotonic: compare it against the previous frame's value + // to find how many were lost in between. + // + // Always zero for a stream created with `queue_size_frames = 0`, which is + // unbounded and never drops. + optional uint64 dropped_frames = 5; } message VideoStreamEOS {} diff --git a/livekit-ffi/src/room_apis/server/colorcvt/cvtimpl.rs b/livekit-ffi/src/room_apis/server/colorcvt/cvtimpl.rs index 95a52e26fb..b1d88cb362 100644 --- a/livekit-ffi/src/room_apis/server/colorcvt/cvtimpl.rs +++ b/livekit-ffi/src/room_apis/server/colorcvt/cvtimpl.rs @@ -17,23 +17,38 @@ use crate::proto; use crate::{FfiError, FfiResult}; use imgproc::colorcvt; +/// Sizes `dst` to hold exactly `len` bytes and hands back the slice to write. +/// +/// The buffer belongs to the caller and is reused across frames, so its +/// previous contents survive a same-size fit. Every conversion below writes +/// its destination in full -- destination strides are always tight -- which is +/// what makes that safe. +fn fit(dst: &mut Vec, len: usize) -> &mut [u8] { + if dst.len() != len { + dst.clear(); + dst.resize(len, 0); + } + dst.as_mut_slice() +} + pub unsafe fn cvt( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { match buffer.r#type() { - proto::VideoBufferType::Rgba => cvt_rgba(buffer, dst_type, flip_y), - proto::VideoBufferType::Abgr => cvt_abgr(buffer, dst_type, flip_y), - proto::VideoBufferType::Argb => cvt_argb(buffer, dst_type, flip_y), - proto::VideoBufferType::Bgra => cvt_bgra(buffer, dst_type, flip_y), - proto::VideoBufferType::Rgb24 => cvt_rgb24(buffer, dst_type, flip_y), - proto::VideoBufferType::I420 => cvt_i420(buffer, dst_type, flip_y), - proto::VideoBufferType::I420a => cvt_i420a(buffer, dst_type, flip_y), - proto::VideoBufferType::I422 => cvt_i422(buffer, dst_type, flip_y), - proto::VideoBufferType::I444 => cvt_i444(buffer, dst_type, flip_y), - proto::VideoBufferType::I010 => cvt_i010(buffer, dst_type, flip_y), - proto::VideoBufferType::Nv12 => cvt_nv12(buffer, dst_type, flip_y), + proto::VideoBufferType::Rgba => cvt_rgba(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::Abgr => cvt_abgr(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::Argb => cvt_argb(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::Bgra => cvt_bgra(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::Rgb24 => cvt_rgb24(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::I420 => cvt_i420(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::I420a => cvt_i420a(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::I422 => cvt_i422(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::I444 => cvt_i444(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::I010 => cvt_i010(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::Nv12 => cvt_nv12(buffer, dst_type, flip_y, dst), } } @@ -41,7 +56,8 @@ pub unsafe fn cvt_rgba( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::Rgba); let proto::VideoBufferInfo { stride, width, height, data_ptr, .. } = buffer; let stride = stride.unwrap_or(width * 4); @@ -52,8 +68,7 @@ pub unsafe fn cvt_rgba( proto::VideoBufferType::I420 => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); @@ -76,16 +91,16 @@ pub unsafe fn cvt_rgba( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::Bgra => { - let mut dst = vec![0u8; (width * height * 4) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height * 4) as usize); let stride = width * 4; - colorcvt::abgr_to_argb(data, stride, &mut dst, stride, width, height, flip_y); + colorcvt::abgr_to_argb(data, stride, &mut *dst, stride, width, height, flip_y); let info = rgba_info(dst.as_ptr(), dst_type, width, height); - Ok((dst, info)) + Ok(info) } _ => { Err(FfiError::InvalidRequest(format!("rgba to {:?} is not supported", dst_type).into())) @@ -97,7 +112,8 @@ pub unsafe fn cvt_abgr( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::Rgba); let proto::VideoBufferInfo { stride, width, height, data_ptr, .. } = buffer; let stride = stride.unwrap_or(width * 4); @@ -108,8 +124,7 @@ pub unsafe fn cvt_abgr( proto::VideoBufferType::I420 => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); @@ -132,7 +147,7 @@ pub unsafe fn cvt_abgr( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } _ => { Err(FfiError::InvalidRequest(format!("abgr to {:?} is not supported", dst_type).into())) @@ -144,7 +159,8 @@ pub unsafe fn cvt_argb( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::Argb); let proto::VideoBufferInfo { stride, width, height, data_ptr, .. } = buffer; let stride = stride.unwrap_or(width * 4); @@ -155,8 +171,7 @@ pub unsafe fn cvt_argb( proto::VideoBufferType::I420 => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -178,7 +193,7 @@ pub unsafe fn cvt_argb( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } _ => { Err(FfiError::InvalidRequest(format!("argb to {:?} is not supported", dst_type).into())) @@ -275,7 +290,8 @@ pub unsafe fn cvt_bgra( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::Bgra); let proto::VideoBufferInfo { stride, width, height, data_ptr, .. } = buffer; let stride = stride.unwrap_or(width * 4); @@ -286,8 +302,7 @@ pub unsafe fn cvt_bgra( proto::VideoBufferType::I420 => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -310,16 +325,16 @@ pub unsafe fn cvt_bgra( chroma_w, ); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::Rgba => { - let mut dst = vec![0u8; (width * height * 4) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height * 4) as usize); let stride = width * 4; - colorcvt::argb_to_abgr(data, stride, &mut dst, stride, width, height, flip_y); + colorcvt::argb_to_abgr(data, stride, &mut *dst, stride, width, height, flip_y); let info = rgba_info(dst.as_ptr(), dst_type, width, height); - Ok((dst, info)) + Ok(info) } _ => { Err(FfiError::InvalidRequest(format!("bgra to {:?} is not supported", dst_type).into())) @@ -331,7 +346,8 @@ pub unsafe fn cvt_rgb24( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::Rgb24); let proto::VideoBufferInfo { stride, width, height, data_ptr, .. } = buffer; let stride = stride.unwrap_or(width * 3); @@ -342,8 +358,7 @@ pub unsafe fn cvt_rgb24( proto::VideoBufferType::I420 => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -365,7 +380,7 @@ pub unsafe fn cvt_rgb24( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } _ => { return Err(FfiError::InvalidRequest( @@ -379,7 +394,8 @@ pub unsafe fn cvt_i420( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::I420); let proto::VideoBufferInfo { width, height, components, .. } = buffer; @@ -397,14 +413,14 @@ pub unsafe fn cvt_i420( | proto::VideoBufferType::Abgr | proto::VideoBufferType::Argb | proto::VideoBufferType::Bgra => { - let mut dst = vec![0u8; (width * height * 4) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height * 4) as usize); let stride = width * 4; macro_rules! cvt { ($rgba:expr, $fnc:ident) => { if dst_type == $rgba { colorcvt::$fnc( - data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut dst, + data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut *dst, stride, width, height, flip_y, ); } @@ -417,25 +433,24 @@ pub unsafe fn cvt_i420( cvt!(proto::VideoBufferType::Bgra, i420_to_argb); let info = rgba_info(dst.as_ptr(), dst_type, width, height); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::Rgb24 => { - let mut dst = vec![0u8; (width * height * 3) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height * 3) as usize); let stride = width * 3; colorcvt::i420_to_raw( - data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut dst, stride, width, + data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut *dst, stride, width, height, flip_y, ); let info = rgb_info(dst.as_ptr(), dst_type, width, height); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::I420 => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -458,7 +473,7 @@ pub unsafe fn cvt_i420( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } _ => { return Err(FfiError::InvalidRequest( @@ -472,7 +487,8 @@ pub unsafe fn cvt_i420a( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::I420a); let proto::VideoBufferInfo { width, height, components, .. } = buffer; @@ -490,8 +506,7 @@ pub unsafe fn cvt_i420a( proto::VideoBufferType::I420 => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -513,14 +528,13 @@ pub unsafe fn cvt_i420a( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::I420a => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2 + width * height) as usize] - .into_boxed_slice(); + let dst = + fit(dst, (width * height + chroma_w * chroma_h * 2 + width * height) as usize); let (dst_y, dst_u, dst_v, dst_a) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); @@ -547,7 +561,7 @@ pub unsafe fn cvt_i420a( chroma_w, width, ); - Ok((dst, info)) + Ok(info) } _ => { return Err(FfiError::InvalidRequest( @@ -561,7 +575,8 @@ pub unsafe fn cvt_i422( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::I422); let proto::VideoBufferInfo { width, height, components, .. } = buffer; @@ -578,14 +593,14 @@ pub unsafe fn cvt_i422( proto::VideoBufferType::Rgba | proto::VideoBufferType::Abgr | proto::VideoBufferType::Argb => { - let mut dst = vec![0u8; (buffer.width * buffer.height * 4) as usize].into_boxed_slice(); + let dst = fit(dst, (buffer.width * buffer.height * 4) as usize); let stride = buffer.width * 4; macro_rules! cvt { ($rgba:expr, $fnc:ident) => { if dst_type == $rgba { colorcvt::$fnc( - data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut dst, + data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut *dst, stride, width, height, flip_y, ); } @@ -597,13 +612,12 @@ pub unsafe fn cvt_i422( cvt!(proto::VideoBufferType::Argb, i422_to_bgra); let info = rgba_info(dst.as_ptr(), dst_type, width, height); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::I420 => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -626,13 +640,12 @@ pub unsafe fn cvt_i422( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::I422 => { let chroma_w = (width + 1) / 2; let chroma_h = height; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -654,7 +667,7 @@ pub unsafe fn cvt_i422( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } _ => { return Err(FfiError::InvalidRequest( @@ -668,7 +681,8 @@ pub unsafe fn cvt_i444( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::I444); let proto::VideoBufferInfo { width, height, components, .. } = buffer; @@ -683,14 +697,14 @@ pub unsafe fn cvt_i444( match dst_type { proto::VideoBufferType::Rgba | proto::VideoBufferType::Bgra => { - let mut dst = vec![0u8; (buffer.width * buffer.height * 4) as usize].into_boxed_slice(); + let dst = fit(dst, (buffer.width * buffer.height * 4) as usize); let stride = buffer.width * 4; macro_rules! cvt { ($rgba:expr, $fnc:ident) => { if dst_type == $rgba { imgproc::colorcvt::$fnc( - data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut dst, + data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut *dst, stride, width, height, flip_y, ); } @@ -701,13 +715,12 @@ pub unsafe fn cvt_i444( cvt!(proto::VideoBufferType::Bgra, i444_to_argb); let info = rgba_info(dst.as_ptr(), dst_type, width, height); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::I420 => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -730,13 +743,12 @@ pub unsafe fn cvt_i444( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::I444 => { let chroma_w = width; let chroma_h = height; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); @@ -760,7 +772,7 @@ pub unsafe fn cvt_i444( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } _ => { return Err(FfiError::InvalidRequest( @@ -774,7 +786,8 @@ pub unsafe fn cvt_i010( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::I010); let proto::VideoBufferInfo { width, height, components, .. } = buffer; @@ -789,14 +802,14 @@ pub unsafe fn cvt_i010( match dst_type { proto::VideoBufferType::Rgba | proto::VideoBufferType::Bgra => { - let mut dst = vec![0u8; (buffer.width * buffer.height * 4) as usize].into_boxed_slice(); + let dst = fit(dst, (buffer.width * buffer.height * 4) as usize); let stride = buffer.width * 4; macro_rules! cvt { ($rgba:expr, $fnc:ident) => { if dst_type == $rgba { imgproc::colorcvt::$fnc( - data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut dst, + data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut *dst, stride, width, height, flip_y, ); } @@ -807,13 +820,12 @@ pub unsafe fn cvt_i010( cvt!(proto::VideoBufferType::Bgra, i010_to_argb); let info = rgba_info(dst.as_ptr(), dst_type, width, height); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::I420 => { let chroma_w = (buffer.width + 1) / 2; let chroma_h = (buffer.height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -836,7 +848,7 @@ pub unsafe fn cvt_i010( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } _ => { return Err(FfiError::InvalidRequest( @@ -850,7 +862,8 @@ pub unsafe fn cvt_nv12( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::Nv12); let proto::VideoBufferInfo { width, height, components, .. } = buffer; @@ -864,15 +877,15 @@ pub unsafe fn cvt_nv12( match dst_type { proto::VideoBufferType::Rgba | proto::VideoBufferType::Bgra => { - let mut dst = vec![0u8; (buffer.width * buffer.height * 4) as usize].into_boxed_slice(); + let dst = fit(dst, (buffer.width * buffer.height * 4) as usize); let stride = buffer.width * 4; macro_rules! cvt { ($rgba:expr, $fnc:ident) => { if dst_type == $rgba { imgproc::colorcvt::$fnc( - data_y, c0.stride, data_uv, c1.stride, &mut dst, stride, width, height, - flip_y, + data_y, c0.stride, data_uv, c1.stride, &mut *dst, stride, width, + height, flip_y, ); } }; @@ -882,13 +895,12 @@ pub unsafe fn cvt_nv12( cvt!(proto::VideoBufferType::Bgra, nv12_to_argb); let info = rgba_info(dst.as_ptr(), dst_type, width, height); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::I420 => { let chroma_w = (buffer.width + 1) / 2; let chroma_h = (buffer.height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -911,13 +923,12 @@ pub unsafe fn cvt_nv12( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::Nv12 => { let chroma_w = (buffer.width + 1) / 2; let chroma_h = (buffer.height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_uv) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); (luma, chroma) @@ -937,7 +948,7 @@ pub unsafe fn cvt_nv12( width, chroma_w, ); - Ok((dst, info)) + Ok(info) } _ => { return Err(FfiError::InvalidRequest( diff --git a/livekit-ffi/src/room_apis/server/colorcvt/mod.rs b/livekit-ffi/src/room_apis/server/colorcvt/mod.rs index bb1ca6fc56..b34b460b23 100644 --- a/livekit-ffi/src/room_apis/server/colorcvt/mod.rs +++ b/livekit-ffi/src/room_apis/server/colorcvt/mod.rs @@ -122,11 +122,37 @@ pub unsafe fn to_libwebrtc_buffer(info: proto::VideoBufferInfo) -> BoxVideoBuffe } } +/// Converts a libwebrtc frame buffer into `dst`, tightly packed, and returns +/// the pointers describing it. +/// +/// `dst` belongs to the caller so a stream can hand the same allocation back +/// frame after frame instead of churning one frame-sized block per frame. The +/// returned info points into `dst`, so it is only valid while `dst` is. +/// +/// TODO(sxian): skip the copy entirely when no conversion was asked for. +/// +/// For a native-format request `cvt` only repacks the decoder's padded strides +/// into tight ones. We could hand out the decoder's buffer instead: the handle +/// map can already own one (`impl FfiHandle for BoxVideoBuffer`) and +/// `VideoBufferInfo.components` already carries per-plane stride. Two steps: +/// +/// 1. Strides already tight (`stride == width`): the info emitted is identical +/// to what the copy produces, so no binding needs to change. +/// 2. Padded strides: needs stride-aware consumers. The C++ SDK reads +/// `components[].stride` already; Python's `VideoFrame` and Unity's +/// `YuvToRgbConverter` both assume tightly packed data. +/// +/// That would also delete the stream's `FrameBuffers`: libwebrtc's +/// `VideoFrameBufferPool` already pools and refcounts decoder buffers, so this +/// layer is only pooling because it copies. Not avoidable when the application +/// requests a different pixel format, or for hardware decode where the source +/// is a platform texture. pub fn to_video_buffer_info( rtcbuffer: BoxVideoBuffer, dst_type: Option, _normalize_stride: bool, // always normalize stride for now.. -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { match rtcbuffer.buffer_type() { // Convert Native buffer to I420 VideoBufferType::Native => { @@ -145,7 +171,9 @@ pub fn to_video_buffer_info( stride_u, stride_v, ); - unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I420), false) } + unsafe { + cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I420), false, dst) + } } VideoBufferType::I420 => { let i420 = rtcbuffer.as_i420().unwrap(); @@ -163,7 +191,9 @@ pub fn to_video_buffer_info( stride_u, stride_v, ); - unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I420), false) } + unsafe { + cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I420), false, dst) + } } VideoBufferType::I420A => { let i420 = rtcbuffer.as_i420a().unwrap(); @@ -183,7 +213,9 @@ pub fn to_video_buffer_info( stride_v, stride_a, ); - unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I420a), false) } + unsafe { + cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I420a), false, dst) + } } VideoBufferType::I422 => { let i422 = rtcbuffer.as_i422().unwrap(); @@ -201,7 +233,9 @@ pub fn to_video_buffer_info( stride_u, stride_v, ); - unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I422), false) } + unsafe { + cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I422), false, dst) + } } VideoBufferType::I444 => { let i444 = rtcbuffer.as_i444().unwrap(); @@ -219,7 +253,9 @@ pub fn to_video_buffer_info( stride_u, stride_v, ); - unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I444), false) } + unsafe { + cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I444), false, dst) + } } VideoBufferType::I010 => { let i010 = rtcbuffer.as_i010().unwrap(); @@ -237,7 +273,9 @@ pub fn to_video_buffer_info( stride_u, stride_v, ); - unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I010), false) } + unsafe { + cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I010), false, dst) + } } VideoBufferType::NV12 => { let nv12 = rtcbuffer.as_nv12().unwrap(); @@ -253,7 +291,9 @@ pub fn to_video_buffer_info( stride_y, stride_uv, ); - unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::Nv12), false) } + unsafe { + cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::Nv12), false, dst) + } } _ => todo!(), } @@ -544,3 +584,82 @@ pub fn rgb_info( stride: Some(width * 3), } } + +#[cfg(test)] +mod tests { + use livekit::webrtc::video_frame::I420Buffer; + + use super::*; + + /// A realistic shape: 640x480 I420 is 460,800 bytes. + const WIDTH: u32 = 640; + const HEIGHT: u32 = 480; + + fn i420_frame(width: u32, height: u32) -> BoxVideoBuffer { + Box::new(I420Buffer::new(width, height)) as BoxVideoBuffer + } + + fn convert_into(dst: &mut Vec, width: u32, height: u32) -> proto::VideoBufferInfo { + to_video_buffer_info( + i420_frame(width, height), + Some(proto::VideoBufferType::I420), + true, + dst, + ) + .expect("i420 to i420 is supported") + } + + #[test] + fn conversion_writes_into_the_buffer_it_is_given() { + let mut dst = Vec::new(); + + let info = convert_into(&mut dst, WIDTH, HEIGHT); + + let chroma_w = (WIDTH + 1) / 2; + let chroma_h = (HEIGHT + 1) / 2; + assert_eq!(dst.len(), (WIDTH * HEIGHT + chroma_w * chroma_h * 2) as usize); + assert_eq!(info.data_ptr, dst.as_ptr() as u64, "the info points into the caller's buffer"); + } + + #[test] + fn converting_again_into_the_same_buffer_does_not_reallocate() { + // This is what lets a stream hand the same allocation back frame after + // frame instead of churning one per frame. + let mut dst = Vec::new(); + convert_into(&mut dst, WIDTH, HEIGHT); + let address = dst.as_ptr(); + + let info = convert_into(&mut dst, WIDTH, HEIGHT); + + assert_eq!(dst.as_ptr(), address, "the second frame reused the allocation"); + assert_eq!(info.data_ptr, address as u64); + } + + #[test] + fn a_different_frame_size_resizes_the_buffer() { + // A simulcast layer switch changes the frame size mid-stream. + let mut dst = Vec::new(); + convert_into(&mut dst, WIDTH, HEIGHT); + + let info = convert_into(&mut dst, WIDTH / 2, HEIGHT / 2); + + let chroma_w = (WIDTH / 2 + 1) / 2; + let chroma_h = (HEIGHT / 2 + 1) / 2; + assert_eq!(dst.len(), ((WIDTH / 2) * (HEIGHT / 2) + chroma_w * chroma_h * 2) as usize); + assert_eq!(info.width, WIDTH / 2); + assert_eq!(info.data_ptr, dst.as_ptr() as u64); + } + + #[test] + fn conversion_output_is_tightly_packed() { + let mut dst = Vec::new(); + + let info = convert_into(&mut dst, WIDTH, HEIGHT); + + let chroma_w = (WIDTH + 1) / 2; + assert_eq!(info.width, WIDTH); + assert_eq!(info.height, HEIGHT); + let [y, u, v] = &info.components[..] else { panic!("expected three planes") }; + assert_eq!((y.stride, u.stride, v.stride), (WIDTH, chroma_w, chroma_w)); + } +} diff --git a/livekit-ffi/src/room_apis/server/mod.rs b/livekit-ffi/src/room_apis/server/mod.rs index 43a2dcd302..9d7ccdee76 100644 --- a/livekit-ffi/src/room_apis/server/mod.rs +++ b/livekit-ffi/src/room_apis/server/mod.rs @@ -82,7 +82,7 @@ impl FfiHandle for Arc> {} impl FfiHandle for Arc> {} impl FfiHandle for AudioFrame<'static> {} impl FfiHandle for BoxVideoBuffer {} -impl FfiHandle for Box<[u8]> {} +impl FfiHandle for Arc> {} impl FfiHandle for () {} pub struct FfiServer { diff --git a/livekit-ffi/src/room_apis/server/requests.rs b/livekit-ffi/src/room_apis/server/requests.rs index d1a733f58f..55f2dba2b8 100644 --- a/livekit-ffi/src/room_apis/server/requests.rs +++ b/livekit-ffi/src/room_apis/server/requests.rs @@ -579,10 +579,12 @@ unsafe fn on_video_convert( let ref buffer = video_convert.buffer; let flip_y = video_convert.flip_y; let dst_type = video_convert.dst_type(); - match cvtimpl::cvt(buffer.clone(), dst_type, flip_y.unwrap_or(false)) { - Ok((buffer, info)) => { + // One-off conversion, so there is no stream to reuse a buffer from. + let mut dst = Vec::new(); + match cvtimpl::cvt(buffer.clone(), dst_type, flip_y.unwrap_or(false), &mut dst) { + Ok(info) => { let id = server.next_id(); - server.store_handle(id, buffer); + server.store_handle(id, std::sync::Arc::new(dst)); let owned_info = proto::OwnedVideoBuffer { handle: proto::FfiOwnedHandle { id }, info }; Ok(proto::VideoConvertResponse { message: Some(proto::video_convert_response::Message::Buffer(owned_info)), diff --git a/livekit-ffi/src/room_apis/server/video_stream.rs b/livekit-ffi/src/room_apis/server/video_stream.rs index 2cc69ae7f6..7fca250dd7 100644 --- a/livekit-ffi/src/room_apis/server/video_stream.rs +++ b/livekit-ffi/src/room_apis/server/video_stream.rs @@ -12,6 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::{ + atomic::{AtomicU64, Ordering}, + Arc, +}; + use futures_util::StreamExt; use livekit::{ prelude::Track, @@ -37,6 +42,82 @@ pub struct FfiVideoStream { impl FfiHandle for FfiVideoStream {} +/// Frames an FFI video stream has lost, totalled across every track it +/// forwards. +/// +/// A participant stream keeps one handle while the participant swaps tracks, +/// and each track gets a fresh `NativeVideoStream` whose own counter starts at +/// zero. Carrying finished tracks' losses here is what keeps the number the +/// application sees cumulative across a replacement. +#[derive(Clone, Default)] +struct DropTotal(Arc); + +impl DropTotal { + /// Losses of earlier tracks plus `current`, the live track's own count. + fn with_current(&self, current: u64) -> u64 { + self.0.load(Ordering::Relaxed) + current + } + + /// Folds a finished track's losses in, so the next track counts on top. + fn finish_track(&self, current: u64) { + self.0.fetch_add(current, Ordering::Relaxed); + } +} + +/// Upper bound on the frame buffers one stream keeps alive. +/// +/// Sized for how many frames a consumer holds at once in steady state: one for +/// a binding that copies and releases immediately, two for a renderer holding +/// the frame it is displaying while the next arrives, and one of slack for +/// jitter. Going beyond that means a sustained backlog, where the consumer has +/// already pinned the memory itself and the allocation is not what is hurting. +/// +/// Kept small because the set never shrinks, so a burst would otherwise pin its +/// high-water mark for the life of the stream -- 12 MB per 4K frame. Past the +/// cap the stream allocates per frame, exactly as it did before. +const MAX_FRAME_BUFFERS: usize = 3; + +/// The destination buffers one video stream converts into. +/// +/// Converting a frame used to allocate a fresh buffer the size of the frame and +/// free it when the binding released the handle -- 4.5 MB per frame at +/// 1628x1920 I420, 134 MB/s at 30 fps. These are kept and handed out again +/// instead. +/// +/// A buffer may only be written while nothing else holds it. `Arc` tracks +/// that: the FFI handle map holds a clone for as long as the binding has the +/// frame, so a strong count of 1 means the buffer came back and is safe to +/// reuse. Nothing relies on the binding releasing frames promptly -- if none +/// of the buffers is free, [`acquire`](Self::acquire) adds one. +#[derive(Default)] +struct FrameBuffers { + buffers: Vec>>, +} + +impl FrameBuffers { + /// Borrows a buffer no one else holds, reusing one when possible. + /// + /// The slot always has a strong count of 1, so the caller can take + /// `Arc::get_mut` on it. Returning the slot rather than a clone is what + /// makes that true -- a clone would be the second owner itself. + fn acquire(&mut self) -> &mut Arc> { + if let Some(index) = self.buffers.iter().position(|buffer| Arc::strong_count(buffer) == 1) { + return &mut self.buffers[index]; + } + + if self.buffers.len() < MAX_FRAME_BUFFERS { + self.buffers.push(Arc::new(Vec::new())); + return self.buffers.last_mut().unwrap(); + } + + // Every buffer is still out and we are at the cap. Give up this slot: + // the binding owns that buffer now and frees it on release, and we + // start a fresh one here rather than block or overwrite it. + self.buffers[0] = Arc::new(Vec::new()); + &mut self.buffers[0] + } +} + fn frame_metadata_to_proto(metadata: Option) -> Option { metadata.map(|metadata| proto::FrameMetadata { user_timestamp: metadata.user_timestamp, @@ -86,6 +167,7 @@ impl FfiVideoStream { self_dropped_rx, server.watch_handle_dropped(new_stream.track_handle), true, + DropTotal::default(), )); server.watch_panic(handle); Ok::(video_stream) @@ -139,7 +221,10 @@ impl FfiVideoStream { mut self_dropped_rx: oneshot::Receiver<()>, mut handle_dropped_rx: oneshot::Receiver<()>, send_eos: bool, + dropped_before: DropTotal, ) { + let mut buffers = FrameBuffers::default(); + loop { tokio::select! { _ = &mut self_dropped_rx => { @@ -156,14 +241,29 @@ impl FfiVideoStream { let metadata = frame_metadata_to_proto(frame.frame_metadata); let timestamp_us = frame.timestamp_us; let rotation = proto::VideoRotation::from(frame.rotation).into(); - let Ok((buffer, info)) = colorcvt::to_video_buffer_info(frame.buffer, dst_type, normalize_stride) else { + let slot = buffers.acquire(); + let Ok(info) = colorcvt::to_video_buffer_info( + frame.buffer, + dst_type, + normalize_stride, + Arc::get_mut(slot).expect("acquire yields an unshared buffer"), + ) else { log::error!("video stream failed to convert video frame to {:?}", dst_type); continue; }; let handle_id = server.next_id(); - server.store_handle(handle_id, buffer); + // The handle co-owns the buffer. That second reference is + // what stops the buffer being written again before the + // binding is done reading it. + server.store_handle(handle_id, slot.clone()); + // Read after the frame is popped, so the count covers every + // drop that happened before this frame was delivered. The + // base carries the losses of tracks this FFI stream + // forwarded earlier, which keeps the total cumulative + // across a track replacement. + let dropped_frames = dropped_before.with_current(native_stream.dropped_frames()); if let Err(err) = server.send_event( proto::VideoStreamEvent { @@ -179,6 +279,7 @@ impl FfiVideoStream { info, }, metadata, + dropped_frames: Some(dropped_frames), } )), }.into() @@ -190,6 +291,9 @@ impl FfiVideoStream { } } + // This track is done; its losses belong to the stream's running total. + dropped_before.finish_track(native_stream.dropped_frames()); + if send_eos { if let Err(err) = server.send_event( proto::VideoStreamEvent { @@ -225,6 +329,9 @@ impl FfiVideoStream { let track_source = request.track_source(); let normalize_stride = request.normalize_stride.unwrap_or(true); let queue_size_frames = request.queue_size_frames.map(|capacity| capacity as usize); + // One FFI stream handle spans every track this participant publishes, + // so the drop total has to survive a replacement. + let dropped_before = DropTotal::default(); let (track_tx, mut track_rx) = mpsc::channel::(1); let (track_finished_tx, track_finished_rx) = broadcast::channel::(1); server.async_runtime.spawn(utils::track_changed_trigger( @@ -247,6 +354,7 @@ impl FfiVideoStream { let (done_tx, mut done_rx) = oneshot::channel::<()>(); let options = NativeVideoStreamOptions { queue_size_frames }; + let dropped_before = dropped_before.clone(); let mut track_finished_rx = track_finished_tx.subscribe(); server.async_runtime.spawn(async move { tokio::select! { @@ -272,6 +380,7 @@ impl FfiVideoStream { c_rx, handle_dropped_rx, false, + dropped_before, ) .await; let _ = done_tx.send(()); @@ -304,9 +413,10 @@ impl FfiVideoStream { #[cfg(test)] mod tests { - use super::frame_metadata_to_proto; use livekit::webrtc::video_frame::FrameMetadata; + use super::*; + #[test] fn missing_frame_metadata_stays_missing() { assert!(frame_metadata_to_proto(None).is_none()); @@ -325,4 +435,109 @@ mod tests { assert_eq!(metadata.frame_id, None); assert_eq!(metadata.user_data, Some(vec![1, 2, 3])); } + + /// Stands in for the FFI handle map: holding this clone is what marks a + /// buffer as still out with the binding. + fn handed_to_binding(slot: &Arc>) -> Arc> { + slot.clone() + } + + #[test] + fn a_released_buffer_is_handed_out_again() { + let mut buffers = FrameBuffers::default(); + + let first = buffers.acquire(); + Arc::get_mut(first).expect("unshared").extend_from_slice(&[1, 2, 3]); + let allocation = Arc::as_ptr(first); + let held = handed_to_binding(first); + drop(held); // the binding released the frame + + let second = buffers.acquire(); + + assert_eq!(Arc::as_ptr(second), allocation, "the same allocation came back"); + assert_eq!(second.as_slice(), &[1, 2, 3], "and it is the same storage"); + assert_eq!(buffers.buffers.len(), 1, "no second buffer was needed"); + } + + #[test] + fn a_buffer_still_held_by_the_binding_is_never_handed_out() { + let mut buffers = FrameBuffers::default(); + + let first = buffers.acquire(); + // Identity by allocation, not by data pointer: an empty Vec's data + // pointer is a shared dangling sentinel and compares equal. + let first_allocation = Arc::as_ptr(first); + let _held = handed_to_binding(first); // binding still has frame 1 + + let second = buffers.acquire(); + + assert_ne!(Arc::as_ptr(second), first_allocation, "writing here would corrupt frame 1"); + assert!(Arc::get_mut(second).is_some(), "the caller can write to it"); + assert_eq!(buffers.buffers.len(), 2); + } + + #[test] + fn two_buffers_suffice_when_one_frame_is_held_at_a_time() { + // What a zero-copy binding does: hold the current frame until the next + // one arrives, then release it. + let mut buffers = FrameBuffers::default(); + let mut previous = None; + + for _ in 0..16 { + let slot = buffers.acquire(); + assert!(Arc::get_mut(slot).is_some()); + let held = handed_to_binding(slot); + previous = Some(held); // releases the frame before it, as Unity does + } + drop(previous); + + assert_eq!(buffers.buffers.len(), 2, "alternating between two is enough"); + } + + #[test] + fn growth_stops_at_the_cap() { + let mut buffers = FrameBuffers::default(); + + // Never release, so every buffer stays out. + let held: Vec<_> = + (0..MAX_FRAME_BUFFERS + 4).map(|_| handed_to_binding(buffers.acquire())).collect(); + + assert_eq!(buffers.buffers.len(), MAX_FRAME_BUFFERS); + assert_eq!(held.len(), MAX_FRAME_BUFFERS + 4, "every frame still got a buffer"); + // Past the cap each frame gets its own, so no two frames share memory. + let mut allocations: Vec<_> = held.iter().map(Arc::as_ptr).collect(); + allocations.sort_unstable(); + let total = allocations.len(); + allocations.dedup(); + assert_eq!(allocations.len(), total, "buffers handed out at once must be distinct"); + } + + #[test] + fn the_drop_total_carries_across_a_track_replacement() { + let total = DropTotal::default(); + + assert_eq!(total.with_current(12), 12, "first track, 12 of its own"); + total.finish_track(12); // the participant swaps tracks here + + // The next track's NativeVideoStream starts its own counter at zero. + assert_eq!(total.with_current(0), 12, "the swap must not reset the total"); + assert_eq!(total.with_current(3), 15, "and the new track counts on top"); + } + + #[test] + fn a_resized_frame_reuses_the_same_buffer() { + // A simulcast layer switch changes the frame size; the allocation + // should still be reused rather than replaced. + let mut buffers = FrameBuffers::default(); + + let first = buffers.acquire(); + Arc::get_mut(first).expect("unshared").resize(1024, 0); + drop(handed_to_binding(first)); + + let second = buffers.acquire(); + Arc::get_mut(second).expect("unshared").resize(2048, 0); + + assert_eq!(buffers.buffers.len(), 1); + assert_eq!(buffers.buffers[0].len(), 2048); + } }