From 4f55a4011e63c403e1831f3cc57ba8d08d04542d Mon Sep 17 00:00:00 2001 From: shijing xian Date: Fri, 6 Nov 2026 17:34:59 -0800 Subject: [PATCH 1/2] Reuse video frame conversion buffers Converting a frame allocated a fresh buffer the size of the frame and freed it when the binding released the handle. A 1628x1920 I420 frame is 4.5 MB, so a 30 fps stream churned 134 MB/s of short-lived allocations that large, and the Windows heap sends anything over ~512 KB straight to `VirtualAlloc`, decommitting it on free -- which is one reason the same workload can lose frames there and not on Linux. `cvt` now writes into a buffer the caller owns, and each stream keeps a small set to hand back. A buffer may only be written while nothing else holds it, and `Arc` is what knows: the FFI handle map holds a clone for as long as the binding has the frame, so a strong count of 1 means the buffer came back. `Arc::get_mut` turns "is the binding done?" into a question the borrow checker answers rather than one the code has to get right -- the alternative, writing into a buffer the binding is still reading, tears frames rather than dropping them. Nothing depends on the binding releasing promptly: when no buffer is free the stream adds one, up to a small cap, past which it allocates per frame as before. A stream settles at one buffer for a binding that copies and releases immediately, and two for a zero-copy binding holding the current frame while the next arrives. Co-Authored-By: Claude Opus 5 (1M context) --- .changeset/reuse_video_conversion_buffers.md | 5 + .../src/room_apis/server/colorcvt/cvtimpl.rs | 199 +++++++++--------- .../src/room_apis/server/colorcvt/mod.rs | 135 +++++++++++- livekit-ffi/src/room_apis/server/mod.rs | 2 +- livekit-ffi/src/room_apis/server/requests.rs | 8 +- .../src/room_apis/server/video_stream.rs | 163 +++++++++++++- 6 files changed, 403 insertions(+), 109 deletions(-) create mode 100644 .changeset/reuse_video_conversion_buffers.md diff --git a/.changeset/reuse_video_conversion_buffers.md b/.changeset/reuse_video_conversion_buffers.md new file mode 100644 index 0000000000..b51e008f60 --- /dev/null +++ b/.changeset/reuse_video_conversion_buffers.md @@ -0,0 +1,5 @@ +--- +livekit-ffi: patch +--- + +Reuse video frame conversion buffers across frames instead of allocating and freeing a frame-sized block for every frame. diff --git a/livekit-ffi/src/room_apis/server/colorcvt/cvtimpl.rs b/livekit-ffi/src/room_apis/server/colorcvt/cvtimpl.rs index 95a52e26fb..b1d88cb362 100644 --- a/livekit-ffi/src/room_apis/server/colorcvt/cvtimpl.rs +++ b/livekit-ffi/src/room_apis/server/colorcvt/cvtimpl.rs @@ -17,23 +17,38 @@ use crate::proto; use crate::{FfiError, FfiResult}; use imgproc::colorcvt; +/// Sizes `dst` to hold exactly `len` bytes and hands back the slice to write. +/// +/// The buffer belongs to the caller and is reused across frames, so its +/// previous contents survive a same-size fit. Every conversion below writes +/// its destination in full -- destination strides are always tight -- which is +/// what makes that safe. +fn fit(dst: &mut Vec, len: usize) -> &mut [u8] { + if dst.len() != len { + dst.clear(); + dst.resize(len, 0); + } + dst.as_mut_slice() +} + pub unsafe fn cvt( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { match buffer.r#type() { - proto::VideoBufferType::Rgba => cvt_rgba(buffer, dst_type, flip_y), - proto::VideoBufferType::Abgr => cvt_abgr(buffer, dst_type, flip_y), - proto::VideoBufferType::Argb => cvt_argb(buffer, dst_type, flip_y), - proto::VideoBufferType::Bgra => cvt_bgra(buffer, dst_type, flip_y), - proto::VideoBufferType::Rgb24 => cvt_rgb24(buffer, dst_type, flip_y), - proto::VideoBufferType::I420 => cvt_i420(buffer, dst_type, flip_y), - proto::VideoBufferType::I420a => cvt_i420a(buffer, dst_type, flip_y), - proto::VideoBufferType::I422 => cvt_i422(buffer, dst_type, flip_y), - proto::VideoBufferType::I444 => cvt_i444(buffer, dst_type, flip_y), - proto::VideoBufferType::I010 => cvt_i010(buffer, dst_type, flip_y), - proto::VideoBufferType::Nv12 => cvt_nv12(buffer, dst_type, flip_y), + proto::VideoBufferType::Rgba => cvt_rgba(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::Abgr => cvt_abgr(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::Argb => cvt_argb(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::Bgra => cvt_bgra(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::Rgb24 => cvt_rgb24(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::I420 => cvt_i420(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::I420a => cvt_i420a(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::I422 => cvt_i422(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::I444 => cvt_i444(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::I010 => cvt_i010(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::Nv12 => cvt_nv12(buffer, dst_type, flip_y, dst), } } @@ -41,7 +56,8 @@ pub unsafe fn cvt_rgba( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::Rgba); let proto::VideoBufferInfo { stride, width, height, data_ptr, .. } = buffer; let stride = stride.unwrap_or(width * 4); @@ -52,8 +68,7 @@ pub unsafe fn cvt_rgba( proto::VideoBufferType::I420 => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); @@ -76,16 +91,16 @@ pub unsafe fn cvt_rgba( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::Bgra => { - let mut dst = vec![0u8; (width * height * 4) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height * 4) as usize); let stride = width * 4; - colorcvt::abgr_to_argb(data, stride, &mut dst, stride, width, height, flip_y); + colorcvt::abgr_to_argb(data, stride, &mut *dst, stride, width, height, flip_y); let info = rgba_info(dst.as_ptr(), dst_type, width, height); - Ok((dst, info)) + Ok(info) } _ => { Err(FfiError::InvalidRequest(format!("rgba to {:?} is not supported", dst_type).into())) @@ -97,7 +112,8 @@ pub unsafe fn cvt_abgr( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::Rgba); let proto::VideoBufferInfo { stride, width, height, data_ptr, .. } = buffer; let stride = stride.unwrap_or(width * 4); @@ -108,8 +124,7 @@ pub unsafe fn cvt_abgr( proto::VideoBufferType::I420 => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); @@ -132,7 +147,7 @@ pub unsafe fn cvt_abgr( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } _ => { Err(FfiError::InvalidRequest(format!("abgr to {:?} is not supported", dst_type).into())) @@ -144,7 +159,8 @@ pub unsafe fn cvt_argb( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::Argb); let proto::VideoBufferInfo { stride, width, height, data_ptr, .. } = buffer; let stride = stride.unwrap_or(width * 4); @@ -155,8 +171,7 @@ pub unsafe fn cvt_argb( proto::VideoBufferType::I420 => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -178,7 +193,7 @@ pub unsafe fn cvt_argb( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } _ => { Err(FfiError::InvalidRequest(format!("argb to {:?} is not supported", dst_type).into())) @@ -275,7 +290,8 @@ pub unsafe fn cvt_bgra( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::Bgra); let proto::VideoBufferInfo { stride, width, height, data_ptr, .. } = buffer; let stride = stride.unwrap_or(width * 4); @@ -286,8 +302,7 @@ pub unsafe fn cvt_bgra( proto::VideoBufferType::I420 => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -310,16 +325,16 @@ pub unsafe fn cvt_bgra( chroma_w, ); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::Rgba => { - let mut dst = vec![0u8; (width * height * 4) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height * 4) as usize); let stride = width * 4; - colorcvt::argb_to_abgr(data, stride, &mut dst, stride, width, height, flip_y); + colorcvt::argb_to_abgr(data, stride, &mut *dst, stride, width, height, flip_y); let info = rgba_info(dst.as_ptr(), dst_type, width, height); - Ok((dst, info)) + Ok(info) } _ => { Err(FfiError::InvalidRequest(format!("bgra to {:?} is not supported", dst_type).into())) @@ -331,7 +346,8 @@ pub unsafe fn cvt_rgb24( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::Rgb24); let proto::VideoBufferInfo { stride, width, height, data_ptr, .. } = buffer; let stride = stride.unwrap_or(width * 3); @@ -342,8 +358,7 @@ pub unsafe fn cvt_rgb24( proto::VideoBufferType::I420 => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -365,7 +380,7 @@ pub unsafe fn cvt_rgb24( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } _ => { return Err(FfiError::InvalidRequest( @@ -379,7 +394,8 @@ pub unsafe fn cvt_i420( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::I420); let proto::VideoBufferInfo { width, height, components, .. } = buffer; @@ -397,14 +413,14 @@ pub unsafe fn cvt_i420( | proto::VideoBufferType::Abgr | proto::VideoBufferType::Argb | proto::VideoBufferType::Bgra => { - let mut dst = vec![0u8; (width * height * 4) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height * 4) as usize); let stride = width * 4; macro_rules! cvt { ($rgba:expr, $fnc:ident) => { if dst_type == $rgba { colorcvt::$fnc( - data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut dst, + data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut *dst, stride, width, height, flip_y, ); } @@ -417,25 +433,24 @@ pub unsafe fn cvt_i420( cvt!(proto::VideoBufferType::Bgra, i420_to_argb); let info = rgba_info(dst.as_ptr(), dst_type, width, height); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::Rgb24 => { - let mut dst = vec![0u8; (width * height * 3) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height * 3) as usize); let stride = width * 3; colorcvt::i420_to_raw( - data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut dst, stride, width, + data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut *dst, stride, width, height, flip_y, ); let info = rgb_info(dst.as_ptr(), dst_type, width, height); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::I420 => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -458,7 +473,7 @@ pub unsafe fn cvt_i420( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } _ => { return Err(FfiError::InvalidRequest( @@ -472,7 +487,8 @@ pub unsafe fn cvt_i420a( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::I420a); let proto::VideoBufferInfo { width, height, components, .. } = buffer; @@ -490,8 +506,7 @@ pub unsafe fn cvt_i420a( proto::VideoBufferType::I420 => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -513,14 +528,13 @@ pub unsafe fn cvt_i420a( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::I420a => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2 + width * height) as usize] - .into_boxed_slice(); + let dst = + fit(dst, (width * height + chroma_w * chroma_h * 2 + width * height) as usize); let (dst_y, dst_u, dst_v, dst_a) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); @@ -547,7 +561,7 @@ pub unsafe fn cvt_i420a( chroma_w, width, ); - Ok((dst, info)) + Ok(info) } _ => { return Err(FfiError::InvalidRequest( @@ -561,7 +575,8 @@ pub unsafe fn cvt_i422( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::I422); let proto::VideoBufferInfo { width, height, components, .. } = buffer; @@ -578,14 +593,14 @@ pub unsafe fn cvt_i422( proto::VideoBufferType::Rgba | proto::VideoBufferType::Abgr | proto::VideoBufferType::Argb => { - let mut dst = vec![0u8; (buffer.width * buffer.height * 4) as usize].into_boxed_slice(); + let dst = fit(dst, (buffer.width * buffer.height * 4) as usize); let stride = buffer.width * 4; macro_rules! cvt { ($rgba:expr, $fnc:ident) => { if dst_type == $rgba { colorcvt::$fnc( - data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut dst, + data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut *dst, stride, width, height, flip_y, ); } @@ -597,13 +612,12 @@ pub unsafe fn cvt_i422( cvt!(proto::VideoBufferType::Argb, i422_to_bgra); let info = rgba_info(dst.as_ptr(), dst_type, width, height); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::I420 => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -626,13 +640,12 @@ pub unsafe fn cvt_i422( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::I422 => { let chroma_w = (width + 1) / 2; let chroma_h = height; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -654,7 +667,7 @@ pub unsafe fn cvt_i422( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } _ => { return Err(FfiError::InvalidRequest( @@ -668,7 +681,8 @@ pub unsafe fn cvt_i444( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::I444); let proto::VideoBufferInfo { width, height, components, .. } = buffer; @@ -683,14 +697,14 @@ pub unsafe fn cvt_i444( match dst_type { proto::VideoBufferType::Rgba | proto::VideoBufferType::Bgra => { - let mut dst = vec![0u8; (buffer.width * buffer.height * 4) as usize].into_boxed_slice(); + let dst = fit(dst, (buffer.width * buffer.height * 4) as usize); let stride = buffer.width * 4; macro_rules! cvt { ($rgba:expr, $fnc:ident) => { if dst_type == $rgba { imgproc::colorcvt::$fnc( - data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut dst, + data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut *dst, stride, width, height, flip_y, ); } @@ -701,13 +715,12 @@ pub unsafe fn cvt_i444( cvt!(proto::VideoBufferType::Bgra, i444_to_argb); let info = rgba_info(dst.as_ptr(), dst_type, width, height); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::I420 => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -730,13 +743,12 @@ pub unsafe fn cvt_i444( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::I444 => { let chroma_w = width; let chroma_h = height; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); @@ -760,7 +772,7 @@ pub unsafe fn cvt_i444( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } _ => { return Err(FfiError::InvalidRequest( @@ -774,7 +786,8 @@ pub unsafe fn cvt_i010( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::I010); let proto::VideoBufferInfo { width, height, components, .. } = buffer; @@ -789,14 +802,14 @@ pub unsafe fn cvt_i010( match dst_type { proto::VideoBufferType::Rgba | proto::VideoBufferType::Bgra => { - let mut dst = vec![0u8; (buffer.width * buffer.height * 4) as usize].into_boxed_slice(); + let dst = fit(dst, (buffer.width * buffer.height * 4) as usize); let stride = buffer.width * 4; macro_rules! cvt { ($rgba:expr, $fnc:ident) => { if dst_type == $rgba { imgproc::colorcvt::$fnc( - data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut dst, + data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut *dst, stride, width, height, flip_y, ); } @@ -807,13 +820,12 @@ pub unsafe fn cvt_i010( cvt!(proto::VideoBufferType::Bgra, i010_to_argb); let info = rgba_info(dst.as_ptr(), dst_type, width, height); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::I420 => { let chroma_w = (buffer.width + 1) / 2; let chroma_h = (buffer.height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -836,7 +848,7 @@ pub unsafe fn cvt_i010( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } _ => { return Err(FfiError::InvalidRequest( @@ -850,7 +862,8 @@ pub unsafe fn cvt_nv12( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::Nv12); let proto::VideoBufferInfo { width, height, components, .. } = buffer; @@ -864,15 +877,15 @@ pub unsafe fn cvt_nv12( match dst_type { proto::VideoBufferType::Rgba | proto::VideoBufferType::Bgra => { - let mut dst = vec![0u8; (buffer.width * buffer.height * 4) as usize].into_boxed_slice(); + let dst = fit(dst, (buffer.width * buffer.height * 4) as usize); let stride = buffer.width * 4; macro_rules! cvt { ($rgba:expr, $fnc:ident) => { if dst_type == $rgba { imgproc::colorcvt::$fnc( - data_y, c0.stride, data_uv, c1.stride, &mut dst, stride, width, height, - flip_y, + data_y, c0.stride, data_uv, c1.stride, &mut *dst, stride, width, + height, flip_y, ); } }; @@ -882,13 +895,12 @@ pub unsafe fn cvt_nv12( cvt!(proto::VideoBufferType::Bgra, nv12_to_argb); let info = rgba_info(dst.as_ptr(), dst_type, width, height); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::I420 => { let chroma_w = (buffer.width + 1) / 2; let chroma_h = (buffer.height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -911,13 +923,12 @@ pub unsafe fn cvt_nv12( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::Nv12 => { let chroma_w = (buffer.width + 1) / 2; let chroma_h = (buffer.height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_uv) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); (luma, chroma) @@ -937,7 +948,7 @@ pub unsafe fn cvt_nv12( width, chroma_w, ); - Ok((dst, info)) + Ok(info) } _ => { return Err(FfiError::InvalidRequest( diff --git a/livekit-ffi/src/room_apis/server/colorcvt/mod.rs b/livekit-ffi/src/room_apis/server/colorcvt/mod.rs index bb1ca6fc56..b34b460b23 100644 --- a/livekit-ffi/src/room_apis/server/colorcvt/mod.rs +++ b/livekit-ffi/src/room_apis/server/colorcvt/mod.rs @@ -122,11 +122,37 @@ pub unsafe fn to_libwebrtc_buffer(info: proto::VideoBufferInfo) -> BoxVideoBuffe } } +/// Converts a libwebrtc frame buffer into `dst`, tightly packed, and returns +/// the pointers describing it. +/// +/// `dst` belongs to the caller so a stream can hand the same allocation back +/// frame after frame instead of churning one frame-sized block per frame. The +/// returned info points into `dst`, so it is only valid while `dst` is. +/// +/// TODO(sxian): skip the copy entirely when no conversion was asked for. +/// +/// For a native-format request `cvt` only repacks the decoder's padded strides +/// into tight ones. We could hand out the decoder's buffer instead: the handle +/// map can already own one (`impl FfiHandle for BoxVideoBuffer`) and +/// `VideoBufferInfo.components` already carries per-plane stride. Two steps: +/// +/// 1. Strides already tight (`stride == width`): the info emitted is identical +/// to what the copy produces, so no binding needs to change. +/// 2. Padded strides: needs stride-aware consumers. The C++ SDK reads +/// `components[].stride` already; Python's `VideoFrame` and Unity's +/// `YuvToRgbConverter` both assume tightly packed data. +/// +/// That would also delete the stream's `FrameBuffers`: libwebrtc's +/// `VideoFrameBufferPool` already pools and refcounts decoder buffers, so this +/// layer is only pooling because it copies. Not avoidable when the application +/// requests a different pixel format, or for hardware decode where the source +/// is a platform texture. pub fn to_video_buffer_info( rtcbuffer: BoxVideoBuffer, dst_type: Option, _normalize_stride: bool, // always normalize stride for now.. -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { match rtcbuffer.buffer_type() { // Convert Native buffer to I420 VideoBufferType::Native => { @@ -145,7 +171,9 @@ pub fn to_video_buffer_info( stride_u, stride_v, ); - unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I420), false) } + unsafe { + cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I420), false, dst) + } } VideoBufferType::I420 => { let i420 = rtcbuffer.as_i420().unwrap(); @@ -163,7 +191,9 @@ pub fn to_video_buffer_info( stride_u, stride_v, ); - unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I420), false) } + unsafe { + cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I420), false, dst) + } } VideoBufferType::I420A => { let i420 = rtcbuffer.as_i420a().unwrap(); @@ -183,7 +213,9 @@ pub fn to_video_buffer_info( stride_v, stride_a, ); - unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I420a), false) } + unsafe { + cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I420a), false, dst) + } } VideoBufferType::I422 => { let i422 = rtcbuffer.as_i422().unwrap(); @@ -201,7 +233,9 @@ pub fn to_video_buffer_info( stride_u, stride_v, ); - unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I422), false) } + unsafe { + cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I422), false, dst) + } } VideoBufferType::I444 => { let i444 = rtcbuffer.as_i444().unwrap(); @@ -219,7 +253,9 @@ pub fn to_video_buffer_info( stride_u, stride_v, ); - unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I444), false) } + unsafe { + cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I444), false, dst) + } } VideoBufferType::I010 => { let i010 = rtcbuffer.as_i010().unwrap(); @@ -237,7 +273,9 @@ pub fn to_video_buffer_info( stride_u, stride_v, ); - unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I010), false) } + unsafe { + cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I010), false, dst) + } } VideoBufferType::NV12 => { let nv12 = rtcbuffer.as_nv12().unwrap(); @@ -253,7 +291,9 @@ pub fn to_video_buffer_info( stride_y, stride_uv, ); - unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::Nv12), false) } + unsafe { + cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::Nv12), false, dst) + } } _ => todo!(), } @@ -544,3 +584,82 @@ pub fn rgb_info( stride: Some(width * 3), } } + +#[cfg(test)] +mod tests { + use livekit::webrtc::video_frame::I420Buffer; + + use super::*; + + /// A realistic shape: 640x480 I420 is 460,800 bytes. + const WIDTH: u32 = 640; + const HEIGHT: u32 = 480; + + fn i420_frame(width: u32, height: u32) -> BoxVideoBuffer { + Box::new(I420Buffer::new(width, height)) as BoxVideoBuffer + } + + fn convert_into(dst: &mut Vec, width: u32, height: u32) -> proto::VideoBufferInfo { + to_video_buffer_info( + i420_frame(width, height), + Some(proto::VideoBufferType::I420), + true, + dst, + ) + .expect("i420 to i420 is supported") + } + + #[test] + fn conversion_writes_into_the_buffer_it_is_given() { + let mut dst = Vec::new(); + + let info = convert_into(&mut dst, WIDTH, HEIGHT); + + let chroma_w = (WIDTH + 1) / 2; + let chroma_h = (HEIGHT + 1) / 2; + assert_eq!(dst.len(), (WIDTH * HEIGHT + chroma_w * chroma_h * 2) as usize); + assert_eq!(info.data_ptr, dst.as_ptr() as u64, "the info points into the caller's buffer"); + } + + #[test] + fn converting_again_into_the_same_buffer_does_not_reallocate() { + // This is what lets a stream hand the same allocation back frame after + // frame instead of churning one per frame. + let mut dst = Vec::new(); + convert_into(&mut dst, WIDTH, HEIGHT); + let address = dst.as_ptr(); + + let info = convert_into(&mut dst, WIDTH, HEIGHT); + + assert_eq!(dst.as_ptr(), address, "the second frame reused the allocation"); + assert_eq!(info.data_ptr, address as u64); + } + + #[test] + fn a_different_frame_size_resizes_the_buffer() { + // A simulcast layer switch changes the frame size mid-stream. + let mut dst = Vec::new(); + convert_into(&mut dst, WIDTH, HEIGHT); + + let info = convert_into(&mut dst, WIDTH / 2, HEIGHT / 2); + + let chroma_w = (WIDTH / 2 + 1) / 2; + let chroma_h = (HEIGHT / 2 + 1) / 2; + assert_eq!(dst.len(), ((WIDTH / 2) * (HEIGHT / 2) + chroma_w * chroma_h * 2) as usize); + assert_eq!(info.width, WIDTH / 2); + assert_eq!(info.data_ptr, dst.as_ptr() as u64); + } + + #[test] + fn conversion_output_is_tightly_packed() { + let mut dst = Vec::new(); + + let info = convert_into(&mut dst, WIDTH, HEIGHT); + + let chroma_w = (WIDTH + 1) / 2; + assert_eq!(info.width, WIDTH); + assert_eq!(info.height, HEIGHT); + let [y, u, v] = &info.components[..] else { panic!("expected three planes") }; + assert_eq!((y.stride, u.stride, v.stride), (WIDTH, chroma_w, chroma_w)); + } +} diff --git a/livekit-ffi/src/room_apis/server/mod.rs b/livekit-ffi/src/room_apis/server/mod.rs index 43a2dcd302..9d7ccdee76 100644 --- a/livekit-ffi/src/room_apis/server/mod.rs +++ b/livekit-ffi/src/room_apis/server/mod.rs @@ -82,7 +82,7 @@ impl FfiHandle for Arc> {} impl FfiHandle for Arc> {} impl FfiHandle for AudioFrame<'static> {} impl FfiHandle for BoxVideoBuffer {} -impl FfiHandle for Box<[u8]> {} +impl FfiHandle for Arc> {} impl FfiHandle for () {} pub struct FfiServer { diff --git a/livekit-ffi/src/room_apis/server/requests.rs b/livekit-ffi/src/room_apis/server/requests.rs index d1a733f58f..55f2dba2b8 100644 --- a/livekit-ffi/src/room_apis/server/requests.rs +++ b/livekit-ffi/src/room_apis/server/requests.rs @@ -579,10 +579,12 @@ unsafe fn on_video_convert( let ref buffer = video_convert.buffer; let flip_y = video_convert.flip_y; let dst_type = video_convert.dst_type(); - match cvtimpl::cvt(buffer.clone(), dst_type, flip_y.unwrap_or(false)) { - Ok((buffer, info)) => { + // One-off conversion, so there is no stream to reuse a buffer from. + let mut dst = Vec::new(); + match cvtimpl::cvt(buffer.clone(), dst_type, flip_y.unwrap_or(false), &mut dst) { + Ok(info) => { let id = server.next_id(); - server.store_handle(id, buffer); + server.store_handle(id, std::sync::Arc::new(dst)); let owned_info = proto::OwnedVideoBuffer { handle: proto::FfiOwnedHandle { id }, info }; Ok(proto::VideoConvertResponse { message: Some(proto::video_convert_response::Message::Buffer(owned_info)), diff --git a/livekit-ffi/src/room_apis/server/video_stream.rs b/livekit-ffi/src/room_apis/server/video_stream.rs index a4fe1d7b51..460e43d3fd 100644 --- a/livekit-ffi/src/room_apis/server/video_stream.rs +++ b/livekit-ffi/src/room_apis/server/video_stream.rs @@ -70,6 +70,60 @@ impl DropTotal { } } +/// Upper bound on the frame buffers one stream keeps alive. +/// +/// Sized for how many frames a consumer holds at once in steady state: one for +/// a binding that copies and releases immediately, two for a renderer holding +/// the frame it is displaying while the next arrives, and one of slack for +/// jitter. Going beyond that means a sustained backlog, where the consumer has +/// already pinned the memory itself and the allocation is not what is hurting. +/// +/// Kept small because the set never shrinks, so a burst would otherwise pin its +/// high-water mark for the life of the stream -- 12 MB per 4K frame. Past the +/// cap the stream allocates per frame, exactly as it did before. +const MAX_FRAME_BUFFERS: usize = 3; + +/// The destination buffers one video stream converts into. +/// +/// Converting a frame used to allocate a fresh buffer the size of the frame and +/// free it when the binding released the handle -- 4.5 MB per frame at +/// 1628x1920 I420, 134 MB/s at 30 fps. These are kept and handed out again +/// instead. +/// +/// A buffer may only be written while nothing else holds it. `Arc` tracks +/// that: the FFI handle map holds a clone for as long as the binding has the +/// frame, so a strong count of 1 means the buffer came back and is safe to +/// reuse. Nothing relies on the binding releasing frames promptly -- if none +/// of the buffers is free, [`acquire`](Self::acquire) adds one. +#[derive(Default)] +struct FrameBuffers { + buffers: Vec>>, +} + +impl FrameBuffers { + /// Borrows a buffer no one else holds, reusing one when possible. + /// + /// The slot always has a strong count of 1, so the caller can take + /// `Arc::get_mut` on it. Returning the slot rather than a clone is what + /// makes that true -- a clone would be the second owner itself. + fn acquire(&mut self) -> &mut Arc> { + if let Some(index) = self.buffers.iter().position(|buffer| Arc::strong_count(buffer) == 1) { + return &mut self.buffers[index]; + } + + if self.buffers.len() < MAX_FRAME_BUFFERS { + self.buffers.push(Arc::new(Vec::new())); + return self.buffers.last_mut().unwrap(); + } + + // Every buffer is still out and we are at the cap. Give up this slot: + // the binding owns that buffer now and frees it on release, and we + // start a fresh one here rather than block or overwrite it. + self.buffers[0] = Arc::new(Vec::new()); + &mut self.buffers[0] + } +} + fn frame_metadata_to_proto(metadata: Option) -> Option { metadata.map(|metadata| proto::FrameMetadata { user_timestamp: metadata.user_timestamp, @@ -175,6 +229,8 @@ impl FfiVideoStream { send_eos: bool, dropped_before: DropTotal, ) { + let mut buffers = FrameBuffers::default(); + loop { tokio::select! { _ = &mut self_dropped_rx => { @@ -191,13 +247,22 @@ impl FfiVideoStream { let metadata = frame_metadata_to_proto(frame.frame_metadata); let timestamp_us = frame.timestamp_us; let rotation = proto::VideoRotation::from(frame.rotation).into(); - let Ok((buffer, info)) = colorcvt::to_video_buffer_info(frame.buffer, dst_type, normalize_stride) else { + let slot = buffers.acquire(); + let Ok(info) = colorcvt::to_video_buffer_info( + frame.buffer, + dst_type, + normalize_stride, + Arc::get_mut(slot).expect("acquire yields an unshared buffer"), + ) else { log::error!("video stream failed to convert video frame to {:?}", dst_type); continue; }; let handle_id = server.next_id(); - server.store_handle(handle_id, buffer); + // The handle co-owns the buffer. That second reference is + // what stops the buffer being written again before the + // binding is done reading it. + server.store_handle(handle_id, slot.clone()); // Read after the frame is popped, so the count covers every // drop that happened before this frame was delivered. The @@ -206,7 +271,6 @@ impl FfiVideoStream { // across a track replacement. let dropped_frames = dropped_before.with_current(native_stream.dropped_frames()); - if let Err(err) = server.send_event( proto::VideoStreamEvent { stream_handle, @@ -380,6 +444,82 @@ mod tests { assert_eq!(metadata.user_data, Some(vec![1, 2, 3])); } + /// Stands in for the FFI handle map: holding this clone is what marks a + /// buffer as still out with the binding. + fn handed_to_binding(slot: &Arc>) -> Arc> { + slot.clone() + } + + #[test] + fn a_released_buffer_is_handed_out_again() { + let mut buffers = FrameBuffers::default(); + + let first = buffers.acquire(); + Arc::get_mut(first).expect("unshared").extend_from_slice(&[1, 2, 3]); + let allocation = Arc::as_ptr(first); + let held = handed_to_binding(first); + drop(held); // the binding released the frame + + let second = buffers.acquire(); + + assert_eq!(Arc::as_ptr(second), allocation, "the same allocation came back"); + assert_eq!(second.as_slice(), &[1, 2, 3], "and it is the same storage"); + assert_eq!(buffers.buffers.len(), 1, "no second buffer was needed"); + } + + #[test] + fn a_buffer_still_held_by_the_binding_is_never_handed_out() { + let mut buffers = FrameBuffers::default(); + + let first = buffers.acquire(); + // Identity by allocation, not by data pointer: an empty Vec's data + // pointer is a shared dangling sentinel and compares equal. + let first_allocation = Arc::as_ptr(first); + let _held = handed_to_binding(first); // binding still has frame 1 + + let second = buffers.acquire(); + + assert_ne!(Arc::as_ptr(second), first_allocation, "writing here would corrupt frame 1"); + assert!(Arc::get_mut(second).is_some(), "the caller can write to it"); + assert_eq!(buffers.buffers.len(), 2); + } + + #[test] + fn two_buffers_suffice_when_one_frame_is_held_at_a_time() { + // What a zero-copy binding does: hold the current frame until the next + // one arrives, then release it. + let mut buffers = FrameBuffers::default(); + let mut previous = None; + + for _ in 0..16 { + let slot = buffers.acquire(); + assert!(Arc::get_mut(slot).is_some()); + let held = handed_to_binding(slot); + previous = Some(held); // releases the frame before it, as Unity does + } + drop(previous); + + assert_eq!(buffers.buffers.len(), 2, "alternating between two is enough"); + } + + #[test] + fn growth_stops_at_the_cap() { + let mut buffers = FrameBuffers::default(); + + // Never release, so every buffer stays out. + let held: Vec<_> = + (0..MAX_FRAME_BUFFERS + 4).map(|_| handed_to_binding(buffers.acquire())).collect(); + + assert_eq!(buffers.buffers.len(), MAX_FRAME_BUFFERS); + assert_eq!(held.len(), MAX_FRAME_BUFFERS + 4, "every frame still got a buffer"); + // Past the cap each frame gets its own, so no two frames share memory. + let mut allocations: Vec<_> = held.iter().map(Arc::as_ptr).collect(); + allocations.sort_unstable(); + let total = allocations.len(); + allocations.dedup(); + assert_eq!(allocations.len(), total, "buffers handed out at once must be distinct"); + } + #[test] fn the_drop_total_carries_across_a_track_replacement() { let total = DropTotal::default(); @@ -403,4 +543,21 @@ mod tests { assert_eq!(total.total(), 4, "EOS reports what no frame event could"); } + + #[test] + fn a_resized_frame_reuses_the_same_buffer() { + // A simulcast layer switch changes the frame size; the allocation + // should still be reused rather than replaced. + let mut buffers = FrameBuffers::default(); + + let first = buffers.acquire(); + Arc::get_mut(first).expect("unshared").resize(1024, 0); + drop(handed_to_binding(first)); + + let second = buffers.acquire(); + Arc::get_mut(second).expect("unshared").resize(2048, 0); + + assert_eq!(buffers.buffers.len(), 1); + assert_eq!(buffers.buffers[0].len(), 2048); + } } From c4c9e9527e351a91534b2d3a957c0445c52f2c7e Mon Sep 17 00:00:00 2001 From: shijing xian Date: Fri, 6 Nov 2026 19:14:04 -0800 Subject: [PATCH 2/2] Document the conversion safety invariant, and cover the real handle map Two review points. Each arm of to_video_buffer_info builds a VideoBufferInfo of raw plane pointers and calls cvt inside an unsafe block, with no note of what makes that sound. The arms are not alike: the Native arm's pointers address an I420Buffer that to_i420 returned by value, so the call has to stay inside the arm or the buffer is dropped first, while every other arm borrows from rtcbuffer. That asymmetry is now written down, which also rules out the tempting refactor of hoisting the single call out of the match. The FrameBuffers tests stood in for the FFI handle map with a local clone, so nothing covered the real thing. The added test stores a buffer in the handle map, checks the stream will not hand it out while it is there, and checks dropping the handle returns it. --- .../src/room_apis/server/colorcvt/mod.rs | 15 ++++++++ .../src/room_apis/server/video_stream.rs | 38 +++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/livekit-ffi/src/room_apis/server/colorcvt/mod.rs b/livekit-ffi/src/room_apis/server/colorcvt/mod.rs index b34b460b23..6b12893a21 100644 --- a/livekit-ffi/src/room_apis/server/colorcvt/mod.rs +++ b/livekit-ffi/src/room_apis/server/colorcvt/mod.rs @@ -171,6 +171,9 @@ pub fn to_video_buffer_info( stride_u, stride_v, ); + // SAFETY: `info` holds raw pointers into `i420`, which `to_i420` returned + // by value. It lives until the end of this arm, so the conversion has to + // happen here rather than after the match. unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I420), false, dst) } @@ -191,6 +194,8 @@ pub fn to_video_buffer_info( stride_u, stride_v, ); + // SAFETY: `info` holds raw pointers into `i420`, borrowed from + // `rtcbuffer`, which outlives this call. unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I420), false, dst) } @@ -213,6 +218,8 @@ pub fn to_video_buffer_info( stride_v, stride_a, ); + // SAFETY: `info` holds raw pointers into `i420`, borrowed from + // `rtcbuffer`, which outlives this call. unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I420a), false, dst) } @@ -233,6 +240,8 @@ pub fn to_video_buffer_info( stride_u, stride_v, ); + // SAFETY: `info` holds raw pointers into `i422`, borrowed from + // `rtcbuffer`, which outlives this call. unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I422), false, dst) } @@ -253,6 +262,8 @@ pub fn to_video_buffer_info( stride_u, stride_v, ); + // SAFETY: `info` holds raw pointers into `i444`, borrowed from + // `rtcbuffer`, which outlives this call. unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I444), false, dst) } @@ -273,6 +284,8 @@ pub fn to_video_buffer_info( stride_u, stride_v, ); + // SAFETY: `info` holds raw pointers into `i010`, borrowed from + // `rtcbuffer`, which outlives this call. unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I010), false, dst) } @@ -291,6 +304,8 @@ pub fn to_video_buffer_info( stride_y, stride_uv, ); + // SAFETY: `info` holds raw pointers into `nv12`, borrowed from + // `rtcbuffer`, which outlives this call. unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::Nv12), false, dst) } diff --git a/livekit-ffi/src/room_apis/server/video_stream.rs b/livekit-ffi/src/room_apis/server/video_stream.rs index 460e43d3fd..befde39675 100644 --- a/livekit-ffi/src/room_apis/server/video_stream.rs +++ b/livekit-ffi/src/room_apis/server/video_stream.rs @@ -544,6 +544,44 @@ mod tests { assert_eq!(total.total(), 4, "EOS reports what no frame event could"); } + /// The other tests stand in for the FFI handle map with a local clone. + /// This one uses the real map, so a change to how handles are stored or + /// released shows up as a buffer being reused while a binding still has + /// the frame -- which would tear it. + /// + /// `parallel` keeps it out of `dispose_cleans_up_resources`'s window; see + /// the note on that test. + #[serial_test::parallel] + #[test] + fn the_handle_map_is_what_holds_a_buffer_out_of_rotation() { + use crate::FFI_SERVER; + + let mut buffers = FrameBuffers::default(); + let handle_id = FFI_SERVER.next_id(); + + let in_flight = { + let slot = buffers.acquire(); + FFI_SERVER.store_handle(handle_id, slot.clone()); + Arc::as_ptr(slot) + }; + + // While the handle map holds it, the buffer must not come back. + assert_ne!( + Arc::as_ptr(buffers.acquire()), + in_flight, + "reusing it here would overwrite a frame the binding still has" + ); + + assert!(FFI_SERVER.drop_handle(handle_id), "the handle was in the map"); + + // Released by the binding, so it is available again. + let reused = buffers + .buffers + .iter() + .any(|buffer| Arc::as_ptr(buffer) == in_flight && Arc::strong_count(buffer) == 1); + assert!(reused, "dropping the handle returns the buffer to the stream"); + } + #[test] fn a_resized_frame_reuses_the_same_buffer() { // A simulcast layer switch changes the frame size; the allocation