finish proto of livekit-ffi (#36)

* ffi req

* handle req_id

* remaining room events

* fix

* Update ffi.proto

* use sid instead of info

* state -> stream_state

* "s"

* participant is optional on data received

* add ReleaseHandleRequest

* Update ffi.proto

* add DataPacketKind

* VideoSinkInfo

* append info

It looks nicer on the FFI languages

* wip

* keep it simple

* Update ffi.proto

* update server to latest proto

* to_i420

* i420_to_abgr

* to_argb

* add publications to ParticipantInfo

It'll be used when we join a Room

* cleanup + DisposeRequest

* fix compilation

* send the whole buffer info with to_i420

livekit-webrtc/src/media_stream.rs

@@ -122,13 +122,13 @@ impl_media_stream_track_trait!(AudioTrack, audio_to_media);
 
 pub type OnFrameHandler = Box<dyn FnMut(VideoFrame, VideoFrameBuffer) + Send + Sync>;
 pub type OnDiscardedFrameHandler = Box<dyn FnMut() + Send + Sync>;
-pub type OnConstraintsChanged = Box<dyn FnMut(VideoTrackSourceConstraints) + Send + Sync>;
+pub type OnConstraintsChangedHandler = Box<dyn FnMut(VideoTrackSourceConstraints) + Send + Sync>;
 
 #[derive(Default)]
 struct InternalVideoTrackSink {
     on_frame_handler: Mutex<Option<OnFrameHandler>>,
     on_discarded_frame_handler: Mutex<Option<OnDiscardedFrameHandler>>,
-    on_constraints_changed_handler: Mutex<Option<OnConstraintsChanged>>,
+    on_constraints_changed_handler: Mutex<Option<OnConstraintsChangedHandler>>,
 }
 
 pub struct VideoTrackSourceConstraints {
@@ -225,7 +225,7 @@ impl VideoTrack {
         *self.observer.on_discarded_frame_handler.lock().unwrap() = Some(handler);
     }
 
-    pub fn on_constraints_changed(&self, handler: OnConstraintsChanged) {
+    pub fn on_constraints_changed(&self, handler: OnConstraintsChangedHandler) {
         *self.observer.on_constraints_changed_handler.lock().unwrap() = Some(handler);
     }
 }

livekit-webrtc/src/prelude.rs

@@ -1,7 +1,8 @@
 pub use crate::data_channel::{DataChannel, DataChannelInit, DataState};
 pub use crate::jsep::{IceCandidate, SessionDescription};
 pub use crate::media_stream::{
-    AudioTrack, MediaStream, MediaStreamTrackHandle, MediaStreamTrackTrait, VideoTrack,
+    AudioTrack, MediaStream, MediaStreamTrackHandle, MediaStreamTrackTrait,
+    OnConstraintsChangedHandler, OnDiscardedFrameHandler, OnFrameHandler, VideoTrack,
 };
 pub use crate::peer_connection::{
     IceConnectionState, IceGatheringState, PeerConnection, PeerConnectionState,
@@ -16,3 +17,4 @@ pub use crate::rtp_transceiver::RtpTransceiver;
 pub use crate::video_frame::{VideoFrame, VideoRotation};
 pub use crate::video_frame_buffer::*;
 pub use crate::webrtc::RTCRuntime;
+pub use crate::yuv_helper::ConvertError;

livekit-webrtc/src/video_frame_buffer.rs

@@ -4,6 +4,8 @@ use std::pin::Pin;
 use std::slice;
 use webrtc_sys::video_frame_buffer as vfb_sys;
 
+use crate::yuv_helper::{self, ConvertError};
+
 #[derive(Debug)]
 pub enum VideoFrameBufferType {
     Native,
@@ -15,6 +17,15 @@ pub enum VideoFrameBufferType {
     NV12,
 }
 
+// types to convert to
+#[derive(Debug)]
+pub enum VideoFormatType {
+    ARGB,
+    BGRA,
+    ABGR,
+    RGBA,
+}
+
 impl From<vfb_sys::ffi::VideoFrameBufferType> for VideoFrameBufferType {
     fn from(buffer_type: vfb_sys::ffi::VideoFrameBufferType) -> Self {
         match buffer_type {
@@ -102,10 +113,76 @@ impl VideoFrameBuffer {
                 VideoFrameBufferType::NV12 => {
                     Self::NV12(NV12Buffer::new(cxx_handle.pin_mut().get_nv12()))
                 }
-                _ => unreachable!(), // VideoFrameBufferType is represented as i32
             }
         }
     }
+
+    pub fn to_argb(
+        &self,
+        format: VideoFormatType,
+        dst: &mut [u8],
+        dst_stride: i32,
+        dst_width: i32,
+        dst_height: i32,
+    ) -> Result<(), ConvertError> {
+        match self {
+            Self::I420(i420) => match format {
+                VideoFormatType::ARGB => yuv_helper::i420_to_argb(
+                    i420.data_y(),
+                    i420.stride_y(),
+                    i420.data_u(),
+                    i420.stride_u(),
+                    i420.data_v(),
+                    i420.stride_v(),
+                    dst,
+                    dst_stride,
+                    dst_width,
+                    dst_height,
+                )?,
+                VideoFormatType::BGRA => yuv_helper::i420_to_bgra(
+                    i420.data_y(),
+                    i420.stride_y(),
+                    i420.data_u(),
+                    i420.stride_u(),
+                    i420.data_v(),
+                    i420.stride_v(),
+                    dst,
+                    dst_stride,
+                    dst_width,
+                    dst_height,
+                )?,
+                VideoFormatType::ABGR => yuv_helper::i420_to_abgr(
+                    i420.data_y(),
+                    i420.stride_y(),
+                    i420.data_u(),
+                    i420.stride_u(),
+                    i420.data_v(),
+                    i420.stride_v(),
+                    dst,
+                    dst_stride,
+                    dst_width,
+                    dst_height,
+                )?,
+                VideoFormatType::RGBA => yuv_helper::i420_to_rgba(
+                    i420.data_y(),
+                    i420.stride_y(),
+                    i420.data_u(),
+                    i420.stride_u(),
+                    i420.data_v(),
+                    i420.stride_v(),
+                    dst,
+                    dst_stride,
+                    dst_width,
+                    dst_height,
+                )?,
+            },
+            _ => {
+                // TODO(theomonnom): Support other buffer types
+            }
+        };
+
+        Ok(())
+    }
 }
 
 impl VideoFrameBufferTrait for VideoFrameBuffer {

livekit-webrtc/src/yuv_helper.rs

@@ -1,43 +1,98 @@
-use std::convert::TryInto;
+use thiserror::Error;
 
 use webrtc_sys::yuv_helper as yuv_sys;
 
-pub fn i420_to_abgr(
+#[derive(Error, Debug)]
+pub enum ConvertError {
+    #[error("conversion failed: {0}")]
+    Convert(&'static str),
+}
+
+fn i420_safety(
     src_y: &[u8],
     src_stride_y: i32,
     src_u: &[u8],
     src_stride_u: i32,
     src_v: &[u8],
     src_stride_v: i32,
-    dst_abgr: &mut [u8],
-    dst_stride_abgr: i32,
-    width: i32,
+    dst: &mut [u8],
+    dst_stride: i32,
+    _width: i32,
     height: i32,
-) {
-    // Assert minimum capacity for safety
-    let chroma_height = (height + 1) / 2; // the buffer should be padded?
-    let min_y: usize = (src_stride_y * height).try_into().unwrap();
-    let min_u: usize = (src_stride_u * chroma_height).try_into().unwrap();
-    let min_v: usize = (src_stride_v * chroma_height).try_into().unwrap();
-    let min_abgr: usize = (dst_stride_abgr * height).try_into().unwrap();
+) -> Result<(), ConvertError> {
+    let chroma_height = (height + 1) / 2;
+    let min_y = (src_stride_y * height) as usize;
+    let min_u = (src_stride_u * chroma_height) as usize;
+    let min_v = (src_stride_v * chroma_height) as usize;
+    let min_dst = (dst_stride * height) as usize;
 
-    assert!(src_y.len() >= min_y);
-    assert!(src_u.len() >= min_u);
-    assert!(src_v.len() >= min_v);
-    assert!(dst_abgr.len() >= min_abgr);
-
-    unsafe {
-        yuv_sys::ffi::i420_to_abgr(
-            src_y.as_ptr(),
-            src_stride_y,
-            src_u.as_ptr(),
-            src_stride_u,
-            src_v.as_ptr(),
-            src_stride_v,
-            dst_abgr.as_mut_ptr(),
-            dst_stride_abgr,
-            width,
-            height,
-        );
+    if src_y.len() < min_y {
+        return Err(ConvertError::Convert("src_y isn't large enough"));
     }
+
+    if src_u.len() < min_u {
+        return Err(ConvertError::Convert("src_u isn't large enough"));
+    }
+
+    if src_v.len() < min_v {
+        return Err(ConvertError::Convert("src_v isn't large enough"));
+    }
+
+    if dst.len() < min_dst {
+        return Err(ConvertError::Convert("dst isn't large enough"));
+    }
+
+    Ok(())
 }
+
+macro_rules! i420_to_x {
+    ($x:ident) => {
+        pub fn $x(
+            src_y: &[u8],
+            src_stride_y: i32,
+            src_u: &[u8],
+            src_stride_u: i32,
+            src_v: &[u8],
+            src_stride_v: i32,
+            dst: &mut [u8],
+            dst_stride: i32,
+            width: i32,
+            height: i32,
+        ) -> Result<(), ConvertError> {
+            i420_safety(
+                src_y,
+                src_stride_y,
+                src_u,
+                src_stride_u,
+                src_v,
+                src_stride_v,
+                dst,
+                dst_stride,
+                width,
+                height,
+            )?;
+
+            unsafe {
+                yuv_sys::ffi::$x(
+                    src_y.as_ptr(),
+                    src_stride_y,
+                    src_u.as_ptr(),
+                    src_stride_u,
+                    src_v.as_ptr(),
+                    src_stride_v,
+                    dst.as_mut_ptr(),
+                    dst_stride,
+                    width,
+                    height,
+                );
+            }
+
+            Ok(())
+        }
+    };
+}
+
+i420_to_x!(i420_to_argb);
+i420_to_x!(i420_to_bgra);
+i420_to_x!(i420_to_abgr);
+i420_to_x!(i420_to_rgba);