diff --git a/.changeset/reuse_video_conversion_buffers.md b/.changeset/reuse_video_conversion_buffers.md new file mode 100644 index 0000000000..b51e008f60 --- /dev/null +++ b/.changeset/reuse_video_conversion_buffers.md @@ -0,0 +1,5 @@ +--- +livekit-ffi: patch +--- + +Reuse video frame conversion buffers across frames instead of allocating and freeing a frame-sized block for every frame. diff --git a/livekit-ffi/src/room_apis/server/colorcvt/cvtimpl.rs b/livekit-ffi/src/room_apis/server/colorcvt/cvtimpl.rs index 95a52e26fb..b1d88cb362 100644 --- a/livekit-ffi/src/room_apis/server/colorcvt/cvtimpl.rs +++ b/livekit-ffi/src/room_apis/server/colorcvt/cvtimpl.rs @@ -17,23 +17,38 @@ use crate::proto; use crate::{FfiError, FfiResult}; use imgproc::colorcvt; +/// Sizes `dst` to hold exactly `len` bytes and hands back the slice to write. +/// +/// The buffer belongs to the caller and is reused across frames, so its +/// previous contents survive a same-size fit. Every conversion below writes +/// its destination in full -- destination strides are always tight -- which is +/// what makes that safe. +fn fit(dst: &mut Vec, len: usize) -> &mut [u8] { + if dst.len() != len { + dst.clear(); + dst.resize(len, 0); + } + dst.as_mut_slice() +} + pub unsafe fn cvt( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { match buffer.r#type() { - proto::VideoBufferType::Rgba => cvt_rgba(buffer, dst_type, flip_y), - proto::VideoBufferType::Abgr => cvt_abgr(buffer, dst_type, flip_y), - proto::VideoBufferType::Argb => cvt_argb(buffer, dst_type, flip_y), - proto::VideoBufferType::Bgra => cvt_bgra(buffer, dst_type, flip_y), - proto::VideoBufferType::Rgb24 => cvt_rgb24(buffer, dst_type, flip_y), - proto::VideoBufferType::I420 => cvt_i420(buffer, dst_type, flip_y), - proto::VideoBufferType::I420a => cvt_i420a(buffer, dst_type, flip_y), - proto::VideoBufferType::I422 => cvt_i422(buffer, dst_type, flip_y), - proto::VideoBufferType::I444 => cvt_i444(buffer, dst_type, flip_y), - proto::VideoBufferType::I010 => cvt_i010(buffer, dst_type, flip_y), - proto::VideoBufferType::Nv12 => cvt_nv12(buffer, dst_type, flip_y), + proto::VideoBufferType::Rgba => cvt_rgba(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::Abgr => cvt_abgr(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::Argb => cvt_argb(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::Bgra => cvt_bgra(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::Rgb24 => cvt_rgb24(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::I420 => cvt_i420(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::I420a => cvt_i420a(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::I422 => cvt_i422(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::I444 => cvt_i444(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::I010 => cvt_i010(buffer, dst_type, flip_y, dst), + proto::VideoBufferType::Nv12 => cvt_nv12(buffer, dst_type, flip_y, dst), } } @@ -41,7 +56,8 @@ pub unsafe fn cvt_rgba( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::Rgba); let proto::VideoBufferInfo { stride, width, height, data_ptr, .. } = buffer; let stride = stride.unwrap_or(width * 4); @@ -52,8 +68,7 @@ pub unsafe fn cvt_rgba( proto::VideoBufferType::I420 => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); @@ -76,16 +91,16 @@ pub unsafe fn cvt_rgba( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::Bgra => { - let mut dst = vec![0u8; (width * height * 4) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height * 4) as usize); let stride = width * 4; - colorcvt::abgr_to_argb(data, stride, &mut dst, stride, width, height, flip_y); + colorcvt::abgr_to_argb(data, stride, &mut *dst, stride, width, height, flip_y); let info = rgba_info(dst.as_ptr(), dst_type, width, height); - Ok((dst, info)) + Ok(info) } _ => { Err(FfiError::InvalidRequest(format!("rgba to {:?} is not supported", dst_type).into())) @@ -97,7 +112,8 @@ pub unsafe fn cvt_abgr( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::Rgba); let proto::VideoBufferInfo { stride, width, height, data_ptr, .. } = buffer; let stride = stride.unwrap_or(width * 4); @@ -108,8 +124,7 @@ pub unsafe fn cvt_abgr( proto::VideoBufferType::I420 => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); @@ -132,7 +147,7 @@ pub unsafe fn cvt_abgr( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } _ => { Err(FfiError::InvalidRequest(format!("abgr to {:?} is not supported", dst_type).into())) @@ -144,7 +159,8 @@ pub unsafe fn cvt_argb( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::Argb); let proto::VideoBufferInfo { stride, width, height, data_ptr, .. } = buffer; let stride = stride.unwrap_or(width * 4); @@ -155,8 +171,7 @@ pub unsafe fn cvt_argb( proto::VideoBufferType::I420 => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -178,7 +193,7 @@ pub unsafe fn cvt_argb( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } _ => { Err(FfiError::InvalidRequest(format!("argb to {:?} is not supported", dst_type).into())) @@ -275,7 +290,8 @@ pub unsafe fn cvt_bgra( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::Bgra); let proto::VideoBufferInfo { stride, width, height, data_ptr, .. } = buffer; let stride = stride.unwrap_or(width * 4); @@ -286,8 +302,7 @@ pub unsafe fn cvt_bgra( proto::VideoBufferType::I420 => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -310,16 +325,16 @@ pub unsafe fn cvt_bgra( chroma_w, ); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::Rgba => { - let mut dst = vec![0u8; (width * height * 4) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height * 4) as usize); let stride = width * 4; - colorcvt::argb_to_abgr(data, stride, &mut dst, stride, width, height, flip_y); + colorcvt::argb_to_abgr(data, stride, &mut *dst, stride, width, height, flip_y); let info = rgba_info(dst.as_ptr(), dst_type, width, height); - Ok((dst, info)) + Ok(info) } _ => { Err(FfiError::InvalidRequest(format!("bgra to {:?} is not supported", dst_type).into())) @@ -331,7 +346,8 @@ pub unsafe fn cvt_rgb24( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::Rgb24); let proto::VideoBufferInfo { stride, width, height, data_ptr, .. } = buffer; let stride = stride.unwrap_or(width * 3); @@ -342,8 +358,7 @@ pub unsafe fn cvt_rgb24( proto::VideoBufferType::I420 => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -365,7 +380,7 @@ pub unsafe fn cvt_rgb24( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } _ => { return Err(FfiError::InvalidRequest( @@ -379,7 +394,8 @@ pub unsafe fn cvt_i420( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::I420); let proto::VideoBufferInfo { width, height, components, .. } = buffer; @@ -397,14 +413,14 @@ pub unsafe fn cvt_i420( | proto::VideoBufferType::Abgr | proto::VideoBufferType::Argb | proto::VideoBufferType::Bgra => { - let mut dst = vec![0u8; (width * height * 4) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height * 4) as usize); let stride = width * 4; macro_rules! cvt { ($rgba:expr, $fnc:ident) => { if dst_type == $rgba { colorcvt::$fnc( - data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut dst, + data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut *dst, stride, width, height, flip_y, ); } @@ -417,25 +433,24 @@ pub unsafe fn cvt_i420( cvt!(proto::VideoBufferType::Bgra, i420_to_argb); let info = rgba_info(dst.as_ptr(), dst_type, width, height); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::Rgb24 => { - let mut dst = vec![0u8; (width * height * 3) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height * 3) as usize); let stride = width * 3; colorcvt::i420_to_raw( - data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut dst, stride, width, + data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut *dst, stride, width, height, flip_y, ); let info = rgb_info(dst.as_ptr(), dst_type, width, height); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::I420 => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -458,7 +473,7 @@ pub unsafe fn cvt_i420( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } _ => { return Err(FfiError::InvalidRequest( @@ -472,7 +487,8 @@ pub unsafe fn cvt_i420a( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::I420a); let proto::VideoBufferInfo { width, height, components, .. } = buffer; @@ -490,8 +506,7 @@ pub unsafe fn cvt_i420a( proto::VideoBufferType::I420 => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -513,14 +528,13 @@ pub unsafe fn cvt_i420a( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::I420a => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2 + width * height) as usize] - .into_boxed_slice(); + let dst = + fit(dst, (width * height + chroma_w * chroma_h * 2 + width * height) as usize); let (dst_y, dst_u, dst_v, dst_a) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); @@ -547,7 +561,7 @@ pub unsafe fn cvt_i420a( chroma_w, width, ); - Ok((dst, info)) + Ok(info) } _ => { return Err(FfiError::InvalidRequest( @@ -561,7 +575,8 @@ pub unsafe fn cvt_i422( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::I422); let proto::VideoBufferInfo { width, height, components, .. } = buffer; @@ -578,14 +593,14 @@ pub unsafe fn cvt_i422( proto::VideoBufferType::Rgba | proto::VideoBufferType::Abgr | proto::VideoBufferType::Argb => { - let mut dst = vec![0u8; (buffer.width * buffer.height * 4) as usize].into_boxed_slice(); + let dst = fit(dst, (buffer.width * buffer.height * 4) as usize); let stride = buffer.width * 4; macro_rules! cvt { ($rgba:expr, $fnc:ident) => { if dst_type == $rgba { colorcvt::$fnc( - data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut dst, + data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut *dst, stride, width, height, flip_y, ); } @@ -597,13 +612,12 @@ pub unsafe fn cvt_i422( cvt!(proto::VideoBufferType::Argb, i422_to_bgra); let info = rgba_info(dst.as_ptr(), dst_type, width, height); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::I420 => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -626,13 +640,12 @@ pub unsafe fn cvt_i422( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::I422 => { let chroma_w = (width + 1) / 2; let chroma_h = height; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -654,7 +667,7 @@ pub unsafe fn cvt_i422( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } _ => { return Err(FfiError::InvalidRequest( @@ -668,7 +681,8 @@ pub unsafe fn cvt_i444( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::I444); let proto::VideoBufferInfo { width, height, components, .. } = buffer; @@ -683,14 +697,14 @@ pub unsafe fn cvt_i444( match dst_type { proto::VideoBufferType::Rgba | proto::VideoBufferType::Bgra => { - let mut dst = vec![0u8; (buffer.width * buffer.height * 4) as usize].into_boxed_slice(); + let dst = fit(dst, (buffer.width * buffer.height * 4) as usize); let stride = buffer.width * 4; macro_rules! cvt { ($rgba:expr, $fnc:ident) => { if dst_type == $rgba { imgproc::colorcvt::$fnc( - data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut dst, + data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut *dst, stride, width, height, flip_y, ); } @@ -701,13 +715,12 @@ pub unsafe fn cvt_i444( cvt!(proto::VideoBufferType::Bgra, i444_to_argb); let info = rgba_info(dst.as_ptr(), dst_type, width, height); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::I420 => { let chroma_w = (width + 1) / 2; let chroma_h = (height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -730,13 +743,12 @@ pub unsafe fn cvt_i444( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::I444 => { let chroma_w = width; let chroma_h = height; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); @@ -760,7 +772,7 @@ pub unsafe fn cvt_i444( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } _ => { return Err(FfiError::InvalidRequest( @@ -774,7 +786,8 @@ pub unsafe fn cvt_i010( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::I010); let proto::VideoBufferInfo { width, height, components, .. } = buffer; @@ -789,14 +802,14 @@ pub unsafe fn cvt_i010( match dst_type { proto::VideoBufferType::Rgba | proto::VideoBufferType::Bgra => { - let mut dst = vec![0u8; (buffer.width * buffer.height * 4) as usize].into_boxed_slice(); + let dst = fit(dst, (buffer.width * buffer.height * 4) as usize); let stride = buffer.width * 4; macro_rules! cvt { ($rgba:expr, $fnc:ident) => { if dst_type == $rgba { imgproc::colorcvt::$fnc( - data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut dst, + data_y, c0.stride, data_u, c1.stride, data_v, c2.stride, &mut *dst, stride, width, height, flip_y, ); } @@ -807,13 +820,12 @@ pub unsafe fn cvt_i010( cvt!(proto::VideoBufferType::Bgra, i010_to_argb); let info = rgba_info(dst.as_ptr(), dst_type, width, height); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::I420 => { let chroma_w = (buffer.width + 1) / 2; let chroma_h = (buffer.height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -836,7 +848,7 @@ pub unsafe fn cvt_i010( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } _ => { return Err(FfiError::InvalidRequest( @@ -850,7 +862,8 @@ pub unsafe fn cvt_nv12( buffer: proto::VideoBufferInfo, dst_type: proto::VideoBufferType, flip_y: bool, -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { assert_eq!(buffer.r#type(), proto::VideoBufferType::Nv12); let proto::VideoBufferInfo { width, height, components, .. } = buffer; @@ -864,15 +877,15 @@ pub unsafe fn cvt_nv12( match dst_type { proto::VideoBufferType::Rgba | proto::VideoBufferType::Bgra => { - let mut dst = vec![0u8; (buffer.width * buffer.height * 4) as usize].into_boxed_slice(); + let dst = fit(dst, (buffer.width * buffer.height * 4) as usize); let stride = buffer.width * 4; macro_rules! cvt { ($rgba:expr, $fnc:ident) => { if dst_type == $rgba { imgproc::colorcvt::$fnc( - data_y, c0.stride, data_uv, c1.stride, &mut dst, stride, width, height, - flip_y, + data_y, c0.stride, data_uv, c1.stride, &mut *dst, stride, width, + height, flip_y, ); } }; @@ -882,13 +895,12 @@ pub unsafe fn cvt_nv12( cvt!(proto::VideoBufferType::Bgra, nv12_to_argb); let info = rgba_info(dst.as_ptr(), dst_type, width, height); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::I420 => { let chroma_w = (buffer.width + 1) / 2; let chroma_h = (buffer.height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_u, dst_v) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); let (u, v) = chroma.split_at_mut((chroma_w * chroma_h) as usize); @@ -911,13 +923,12 @@ pub unsafe fn cvt_nv12( chroma_w, chroma_w, ); - Ok((dst, info)) + Ok(info) } proto::VideoBufferType::Nv12 => { let chroma_w = (buffer.width + 1) / 2; let chroma_h = (buffer.height + 1) / 2; - let mut dst = - vec![0u8; (width * height + chroma_w * chroma_h * 2) as usize].into_boxed_slice(); + let dst = fit(dst, (width * height + chroma_w * chroma_h * 2) as usize); let (dst_y, dst_uv) = { let (luma, chroma) = dst.split_at_mut((width * height) as usize); (luma, chroma) @@ -937,7 +948,7 @@ pub unsafe fn cvt_nv12( width, chroma_w, ); - Ok((dst, info)) + Ok(info) } _ => { return Err(FfiError::InvalidRequest( diff --git a/livekit-ffi/src/room_apis/server/colorcvt/mod.rs b/livekit-ffi/src/room_apis/server/colorcvt/mod.rs index bb1ca6fc56..6b12893a21 100644 --- a/livekit-ffi/src/room_apis/server/colorcvt/mod.rs +++ b/livekit-ffi/src/room_apis/server/colorcvt/mod.rs @@ -122,11 +122,37 @@ pub unsafe fn to_libwebrtc_buffer(info: proto::VideoBufferInfo) -> BoxVideoBuffe } } +/// Converts a libwebrtc frame buffer into `dst`, tightly packed, and returns +/// the pointers describing it. +/// +/// `dst` belongs to the caller so a stream can hand the same allocation back +/// frame after frame instead of churning one frame-sized block per frame. The +/// returned info points into `dst`, so it is only valid while `dst` is. +/// +/// TODO(sxian): skip the copy entirely when no conversion was asked for. +/// +/// For a native-format request `cvt` only repacks the decoder's padded strides +/// into tight ones. We could hand out the decoder's buffer instead: the handle +/// map can already own one (`impl FfiHandle for BoxVideoBuffer`) and +/// `VideoBufferInfo.components` already carries per-plane stride. Two steps: +/// +/// 1. Strides already tight (`stride == width`): the info emitted is identical +/// to what the copy produces, so no binding needs to change. +/// 2. Padded strides: needs stride-aware consumers. The C++ SDK reads +/// `components[].stride` already; Python's `VideoFrame` and Unity's +/// `YuvToRgbConverter` both assume tightly packed data. +/// +/// That would also delete the stream's `FrameBuffers`: libwebrtc's +/// `VideoFrameBufferPool` already pools and refcounts decoder buffers, so this +/// layer is only pooling because it copies. Not avoidable when the application +/// requests a different pixel format, or for hardware decode where the source +/// is a platform texture. pub fn to_video_buffer_info( rtcbuffer: BoxVideoBuffer, dst_type: Option, _normalize_stride: bool, // always normalize stride for now.. -) -> FfiResult<(Box<[u8]>, proto::VideoBufferInfo)> { + dst: &mut Vec, +) -> FfiResult { match rtcbuffer.buffer_type() { // Convert Native buffer to I420 VideoBufferType::Native => { @@ -145,7 +171,12 @@ pub fn to_video_buffer_info( stride_u, stride_v, ); - unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I420), false) } + // SAFETY: `info` holds raw pointers into `i420`, which `to_i420` returned + // by value. It lives until the end of this arm, so the conversion has to + // happen here rather than after the match. + unsafe { + cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I420), false, dst) + } } VideoBufferType::I420 => { let i420 = rtcbuffer.as_i420().unwrap(); @@ -163,7 +194,11 @@ pub fn to_video_buffer_info( stride_u, stride_v, ); - unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I420), false) } + // SAFETY: `info` holds raw pointers into `i420`, borrowed from + // `rtcbuffer`, which outlives this call. + unsafe { + cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I420), false, dst) + } } VideoBufferType::I420A => { let i420 = rtcbuffer.as_i420a().unwrap(); @@ -183,7 +218,11 @@ pub fn to_video_buffer_info( stride_v, stride_a, ); - unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I420a), false) } + // SAFETY: `info` holds raw pointers into `i420`, borrowed from + // `rtcbuffer`, which outlives this call. + unsafe { + cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I420a), false, dst) + } } VideoBufferType::I422 => { let i422 = rtcbuffer.as_i422().unwrap(); @@ -201,7 +240,11 @@ pub fn to_video_buffer_info( stride_u, stride_v, ); - unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I422), false) } + // SAFETY: `info` holds raw pointers into `i422`, borrowed from + // `rtcbuffer`, which outlives this call. + unsafe { + cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I422), false, dst) + } } VideoBufferType::I444 => { let i444 = rtcbuffer.as_i444().unwrap(); @@ -219,7 +262,11 @@ pub fn to_video_buffer_info( stride_u, stride_v, ); - unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I444), false) } + // SAFETY: `info` holds raw pointers into `i444`, borrowed from + // `rtcbuffer`, which outlives this call. + unsafe { + cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I444), false, dst) + } } VideoBufferType::I010 => { let i010 = rtcbuffer.as_i010().unwrap(); @@ -237,7 +284,11 @@ pub fn to_video_buffer_info( stride_u, stride_v, ); - unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I010), false) } + // SAFETY: `info` holds raw pointers into `i010`, borrowed from + // `rtcbuffer`, which outlives this call. + unsafe { + cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::I010), false, dst) + } } VideoBufferType::NV12 => { let nv12 = rtcbuffer.as_nv12().unwrap(); @@ -253,7 +304,11 @@ pub fn to_video_buffer_info( stride_y, stride_uv, ); - unsafe { cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::Nv12), false) } + // SAFETY: `info` holds raw pointers into `nv12`, borrowed from + // `rtcbuffer`, which outlives this call. + unsafe { + cvtimpl::cvt(info, dst_type.unwrap_or(proto::VideoBufferType::Nv12), false, dst) + } } _ => todo!(), } @@ -544,3 +599,82 @@ pub fn rgb_info( stride: Some(width * 3), } } + +#[cfg(test)] +mod tests { + use livekit::webrtc::video_frame::I420Buffer; + + use super::*; + + /// A realistic shape: 640x480 I420 is 460,800 bytes. + const WIDTH: u32 = 640; + const HEIGHT: u32 = 480; + + fn i420_frame(width: u32, height: u32) -> BoxVideoBuffer { + Box::new(I420Buffer::new(width, height)) as BoxVideoBuffer + } + + fn convert_into(dst: &mut Vec, width: u32, height: u32) -> proto::VideoBufferInfo { + to_video_buffer_info( + i420_frame(width, height), + Some(proto::VideoBufferType::I420), + true, + dst, + ) + .expect("i420 to i420 is supported") + } + + #[test] + fn conversion_writes_into_the_buffer_it_is_given() { + let mut dst = Vec::new(); + + let info = convert_into(&mut dst, WIDTH, HEIGHT); + + let chroma_w = (WIDTH + 1) / 2; + let chroma_h = (HEIGHT + 1) / 2; + assert_eq!(dst.len(), (WIDTH * HEIGHT + chroma_w * chroma_h * 2) as usize); + assert_eq!(info.data_ptr, dst.as_ptr() as u64, "the info points into the caller's buffer"); + } + + #[test] + fn converting_again_into_the_same_buffer_does_not_reallocate() { + // This is what lets a stream hand the same allocation back frame after + // frame instead of churning one per frame. + let mut dst = Vec::new(); + convert_into(&mut dst, WIDTH, HEIGHT); + let address = dst.as_ptr(); + + let info = convert_into(&mut dst, WIDTH, HEIGHT); + + assert_eq!(dst.as_ptr(), address, "the second frame reused the allocation"); + assert_eq!(info.data_ptr, address as u64); + } + + #[test] + fn a_different_frame_size_resizes_the_buffer() { + // A simulcast layer switch changes the frame size mid-stream. + let mut dst = Vec::new(); + convert_into(&mut dst, WIDTH, HEIGHT); + + let info = convert_into(&mut dst, WIDTH / 2, HEIGHT / 2); + + let chroma_w = (WIDTH / 2 + 1) / 2; + let chroma_h = (HEIGHT / 2 + 1) / 2; + assert_eq!(dst.len(), ((WIDTH / 2) * (HEIGHT / 2) + chroma_w * chroma_h * 2) as usize); + assert_eq!(info.width, WIDTH / 2); + assert_eq!(info.data_ptr, dst.as_ptr() as u64); + } + + #[test] + fn conversion_output_is_tightly_packed() { + let mut dst = Vec::new(); + + let info = convert_into(&mut dst, WIDTH, HEIGHT); + + let chroma_w = (WIDTH + 1) / 2; + assert_eq!(info.width, WIDTH); + assert_eq!(info.height, HEIGHT); + let [y, u, v] = &info.components[..] else { panic!("expected three planes") }; + assert_eq!((y.stride, u.stride, v.stride), (WIDTH, chroma_w, chroma_w)); + } +} diff --git a/livekit-ffi/src/room_apis/server/mod.rs b/livekit-ffi/src/room_apis/server/mod.rs index 43a2dcd302..9d7ccdee76 100644 --- a/livekit-ffi/src/room_apis/server/mod.rs +++ b/livekit-ffi/src/room_apis/server/mod.rs @@ -82,7 +82,7 @@ impl FfiHandle for Arc> {} impl FfiHandle for Arc> {} impl FfiHandle for AudioFrame<'static> {} impl FfiHandle for BoxVideoBuffer {} -impl FfiHandle for Box<[u8]> {} +impl FfiHandle for Arc> {} impl FfiHandle for () {} pub struct FfiServer { diff --git a/livekit-ffi/src/room_apis/server/requests.rs b/livekit-ffi/src/room_apis/server/requests.rs index d1a733f58f..55f2dba2b8 100644 --- a/livekit-ffi/src/room_apis/server/requests.rs +++ b/livekit-ffi/src/room_apis/server/requests.rs @@ -579,10 +579,12 @@ unsafe fn on_video_convert( let ref buffer = video_convert.buffer; let flip_y = video_convert.flip_y; let dst_type = video_convert.dst_type(); - match cvtimpl::cvt(buffer.clone(), dst_type, flip_y.unwrap_or(false)) { - Ok((buffer, info)) => { + // One-off conversion, so there is no stream to reuse a buffer from. + let mut dst = Vec::new(); + match cvtimpl::cvt(buffer.clone(), dst_type, flip_y.unwrap_or(false), &mut dst) { + Ok(info) => { let id = server.next_id(); - server.store_handle(id, buffer); + server.store_handle(id, std::sync::Arc::new(dst)); let owned_info = proto::OwnedVideoBuffer { handle: proto::FfiOwnedHandle { id }, info }; Ok(proto::VideoConvertResponse { message: Some(proto::video_convert_response::Message::Buffer(owned_info)), diff --git a/livekit-ffi/src/room_apis/server/video_stream.rs b/livekit-ffi/src/room_apis/server/video_stream.rs index a4fe1d7b51..befde39675 100644 --- a/livekit-ffi/src/room_apis/server/video_stream.rs +++ b/livekit-ffi/src/room_apis/server/video_stream.rs @@ -70,6 +70,60 @@ impl DropTotal { } } +/// Upper bound on the frame buffers one stream keeps alive. +/// +/// Sized for how many frames a consumer holds at once in steady state: one for +/// a binding that copies and releases immediately, two for a renderer holding +/// the frame it is displaying while the next arrives, and one of slack for +/// jitter. Going beyond that means a sustained backlog, where the consumer has +/// already pinned the memory itself and the allocation is not what is hurting. +/// +/// Kept small because the set never shrinks, so a burst would otherwise pin its +/// high-water mark for the life of the stream -- 12 MB per 4K frame. Past the +/// cap the stream allocates per frame, exactly as it did before. +const MAX_FRAME_BUFFERS: usize = 3; + +/// The destination buffers one video stream converts into. +/// +/// Converting a frame used to allocate a fresh buffer the size of the frame and +/// free it when the binding released the handle -- 4.5 MB per frame at +/// 1628x1920 I420, 134 MB/s at 30 fps. These are kept and handed out again +/// instead. +/// +/// A buffer may only be written while nothing else holds it. `Arc` tracks +/// that: the FFI handle map holds a clone for as long as the binding has the +/// frame, so a strong count of 1 means the buffer came back and is safe to +/// reuse. Nothing relies on the binding releasing frames promptly -- if none +/// of the buffers is free, [`acquire`](Self::acquire) adds one. +#[derive(Default)] +struct FrameBuffers { + buffers: Vec>>, +} + +impl FrameBuffers { + /// Borrows a buffer no one else holds, reusing one when possible. + /// + /// The slot always has a strong count of 1, so the caller can take + /// `Arc::get_mut` on it. Returning the slot rather than a clone is what + /// makes that true -- a clone would be the second owner itself. + fn acquire(&mut self) -> &mut Arc> { + if let Some(index) = self.buffers.iter().position(|buffer| Arc::strong_count(buffer) == 1) { + return &mut self.buffers[index]; + } + + if self.buffers.len() < MAX_FRAME_BUFFERS { + self.buffers.push(Arc::new(Vec::new())); + return self.buffers.last_mut().unwrap(); + } + + // Every buffer is still out and we are at the cap. Give up this slot: + // the binding owns that buffer now and frees it on release, and we + // start a fresh one here rather than block or overwrite it. + self.buffers[0] = Arc::new(Vec::new()); + &mut self.buffers[0] + } +} + fn frame_metadata_to_proto(metadata: Option) -> Option { metadata.map(|metadata| proto::FrameMetadata { user_timestamp: metadata.user_timestamp, @@ -175,6 +229,8 @@ impl FfiVideoStream { send_eos: bool, dropped_before: DropTotal, ) { + let mut buffers = FrameBuffers::default(); + loop { tokio::select! { _ = &mut self_dropped_rx => { @@ -191,13 +247,22 @@ impl FfiVideoStream { let metadata = frame_metadata_to_proto(frame.frame_metadata); let timestamp_us = frame.timestamp_us; let rotation = proto::VideoRotation::from(frame.rotation).into(); - let Ok((buffer, info)) = colorcvt::to_video_buffer_info(frame.buffer, dst_type, normalize_stride) else { + let slot = buffers.acquire(); + let Ok(info) = colorcvt::to_video_buffer_info( + frame.buffer, + dst_type, + normalize_stride, + Arc::get_mut(slot).expect("acquire yields an unshared buffer"), + ) else { log::error!("video stream failed to convert video frame to {:?}", dst_type); continue; }; let handle_id = server.next_id(); - server.store_handle(handle_id, buffer); + // The handle co-owns the buffer. That second reference is + // what stops the buffer being written again before the + // binding is done reading it. + server.store_handle(handle_id, slot.clone()); // Read after the frame is popped, so the count covers every // drop that happened before this frame was delivered. The @@ -206,7 +271,6 @@ impl FfiVideoStream { // across a track replacement. let dropped_frames = dropped_before.with_current(native_stream.dropped_frames()); - if let Err(err) = server.send_event( proto::VideoStreamEvent { stream_handle, @@ -380,6 +444,82 @@ mod tests { assert_eq!(metadata.user_data, Some(vec![1, 2, 3])); } + /// Stands in for the FFI handle map: holding this clone is what marks a + /// buffer as still out with the binding. + fn handed_to_binding(slot: &Arc>) -> Arc> { + slot.clone() + } + + #[test] + fn a_released_buffer_is_handed_out_again() { + let mut buffers = FrameBuffers::default(); + + let first = buffers.acquire(); + Arc::get_mut(first).expect("unshared").extend_from_slice(&[1, 2, 3]); + let allocation = Arc::as_ptr(first); + let held = handed_to_binding(first); + drop(held); // the binding released the frame + + let second = buffers.acquire(); + + assert_eq!(Arc::as_ptr(second), allocation, "the same allocation came back"); + assert_eq!(second.as_slice(), &[1, 2, 3], "and it is the same storage"); + assert_eq!(buffers.buffers.len(), 1, "no second buffer was needed"); + } + + #[test] + fn a_buffer_still_held_by_the_binding_is_never_handed_out() { + let mut buffers = FrameBuffers::default(); + + let first = buffers.acquire(); + // Identity by allocation, not by data pointer: an empty Vec's data + // pointer is a shared dangling sentinel and compares equal. + let first_allocation = Arc::as_ptr(first); + let _held = handed_to_binding(first); // binding still has frame 1 + + let second = buffers.acquire(); + + assert_ne!(Arc::as_ptr(second), first_allocation, "writing here would corrupt frame 1"); + assert!(Arc::get_mut(second).is_some(), "the caller can write to it"); + assert_eq!(buffers.buffers.len(), 2); + } + + #[test] + fn two_buffers_suffice_when_one_frame_is_held_at_a_time() { + // What a zero-copy binding does: hold the current frame until the next + // one arrives, then release it. + let mut buffers = FrameBuffers::default(); + let mut previous = None; + + for _ in 0..16 { + let slot = buffers.acquire(); + assert!(Arc::get_mut(slot).is_some()); + let held = handed_to_binding(slot); + previous = Some(held); // releases the frame before it, as Unity does + } + drop(previous); + + assert_eq!(buffers.buffers.len(), 2, "alternating between two is enough"); + } + + #[test] + fn growth_stops_at_the_cap() { + let mut buffers = FrameBuffers::default(); + + // Never release, so every buffer stays out. + let held: Vec<_> = + (0..MAX_FRAME_BUFFERS + 4).map(|_| handed_to_binding(buffers.acquire())).collect(); + + assert_eq!(buffers.buffers.len(), MAX_FRAME_BUFFERS); + assert_eq!(held.len(), MAX_FRAME_BUFFERS + 4, "every frame still got a buffer"); + // Past the cap each frame gets its own, so no two frames share memory. + let mut allocations: Vec<_> = held.iter().map(Arc::as_ptr).collect(); + allocations.sort_unstable(); + let total = allocations.len(); + allocations.dedup(); + assert_eq!(allocations.len(), total, "buffers handed out at once must be distinct"); + } + #[test] fn the_drop_total_carries_across_a_track_replacement() { let total = DropTotal::default(); @@ -403,4 +543,59 @@ mod tests { assert_eq!(total.total(), 4, "EOS reports what no frame event could"); } + + /// The other tests stand in for the FFI handle map with a local clone. + /// This one uses the real map, so a change to how handles are stored or + /// released shows up as a buffer being reused while a binding still has + /// the frame -- which would tear it. + /// + /// `parallel` keeps it out of `dispose_cleans_up_resources`'s window; see + /// the note on that test. + #[serial_test::parallel] + #[test] + fn the_handle_map_is_what_holds_a_buffer_out_of_rotation() { + use crate::FFI_SERVER; + + let mut buffers = FrameBuffers::default(); + let handle_id = FFI_SERVER.next_id(); + + let in_flight = { + let slot = buffers.acquire(); + FFI_SERVER.store_handle(handle_id, slot.clone()); + Arc::as_ptr(slot) + }; + + // While the handle map holds it, the buffer must not come back. + assert_ne!( + Arc::as_ptr(buffers.acquire()), + in_flight, + "reusing it here would overwrite a frame the binding still has" + ); + + assert!(FFI_SERVER.drop_handle(handle_id), "the handle was in the map"); + + // Released by the binding, so it is available again. + let reused = buffers + .buffers + .iter() + .any(|buffer| Arc::as_ptr(buffer) == in_flight && Arc::strong_count(buffer) == 1); + assert!(reused, "dropping the handle returns the buffer to the stream"); + } + + #[test] + fn a_resized_frame_reuses_the_same_buffer() { + // A simulcast layer switch changes the frame size; the allocation + // should still be reused rather than replaced. + let mut buffers = FrameBuffers::default(); + + let first = buffers.acquire(); + Arc::get_mut(first).expect("unshared").resize(1024, 0); + drop(handed_to_binding(first)); + + let second = buffers.acquire(); + Arc::get_mut(second).expect("unshared").resize(2048, 0); + + assert_eq!(buffers.buffers.len(), 1); + assert_eq!(buffers.buffers[0].len(), 2048); + } }