trying to receive VideoFrame, progress...

2022-11-28 15:52:37 +01:00
parent a415748f06
commit 39fa798c7f
20 changed files with 572 additions and 125 deletions
@@ -9,6 +9,7 @@ pub use sys_ms::ffi::ContentHint;
 pub use sys_ms::ffi::TrackState;

 use crate::video_frame::VideoFrame;
+use crate::video_frame_buffer::VideoFrameBuffer;

 pub trait MediaStreamTrackTrait {
    fn kind(&self) -> String;
@@ -115,8 +116,8 @@ macro_rules! impl_media_stream_track_trait {

            fn set_enabled(&self, enabled: bool) -> bool {
                unsafe {
-                    let media =
-                        sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap()) as *mut sys_ms::ffi::MediaStreamTrack;
+                    let media = sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())
+                        as *mut sys_ms::ffi::MediaStreamTrack;

                    Pin::new_unchecked(&mut *media).set_enabled(enabled)
                }
@@ -132,7 +133,7 @@ macro_rules! impl_media_stream_track_trait {
 impl_media_stream_track_trait!(VideoTrack, video_to_media);
 impl_media_stream_track_trait!(AudioTrack, audio_to_media);

-pub type OnFrameHandler = Box<dyn FnMut(VideoFrame) + Send + Sync>;
+pub type OnFrameHandler = Box<dyn FnMut(VideoFrame, VideoFrameBuffer) + Send + Sync>;
 pub type OnDiscardedFrameHandler = Box<dyn FnMut() + Send + Sync>;
 pub type OnConstraintsChanged = Box<dyn FnMut(VideoTrackSourceConstraints) + Send + Sync>;

@@ -160,7 +161,9 @@ impl From<sys_ms::ffi::VideoTrackSourceConstraints> for VideoTrackSourceConstrai
 impl sys_ms::VideoFrameSink for InternalVideoTrackSink {
    fn on_frame(&self, frame: UniquePtr<libwebrtc_sys::video_frame::ffi::VideoFrame>) {
        if let Some(cb) = self.on_frame_handler.lock().unwrap().as_mut() {
-            cb(VideoFrame::new(frame));
+            let frame = VideoFrame::new(frame);
+            let video_frame_buffer = unsafe { frame.video_frame_buffer() };
+            cb(frame, video_frame_buffer);
        }
    }

@@ -203,7 +206,7 @@ impl VideoTrack {
        Arc::new(track)
    }

-    pub fn set_should_receive(&mut self, should_receive: bool) {
+    pub fn set_should_receive(&self, should_receive: bool) {
        self.cxx_handle
            .lock()
            .unwrap()
@@ -211,7 +214,7 @@ impl VideoTrack {
            .set_should_receive(should_receive)
    }

-    pub fn set_content_hint(&mut self, hint: ContentHint) {
+    pub fn set_content_hint(&self, hint: ContentHint) {
        self.cxx_handle
            .lock()
            .unwrap()
@@ -227,15 +230,15 @@ impl VideoTrack {
        self.cxx_handle.lock().unwrap().content_hint()
    }

-    pub fn on_frame(&mut self, handler: OnFrameHandler) {
+    pub fn on_frame(&self, handler: OnFrameHandler) {
        *self.observer.on_frame_handler.lock().unwrap() = Some(handler);
    }

-    pub fn on_discarded_frame(&mut self, handler: OnDiscardedFrameHandler) {
+    pub fn on_discarded_frame(&self, handler: OnDiscardedFrameHandler) {
        *self.observer.on_discarded_frame_handler.lock().unwrap() = Some(handler);
    }

-    pub fn on_constraints_changed(&mut self, handler: OnConstraintsChanged) {
+    pub fn on_constraints_changed(&self, handler: OnConstraintsChanged) {
        *self.observer.on_constraints_changed_handler.lock().unwrap() = Some(handler);
    }
 }
@@ -459,7 +459,7 @@ impl sys_pc::PeerConnectionObserver for InternalObserver {
    }

    fn on_ice_connection_change(&self, new_state: IceConnectionState) {
-        trace!("on_ice_connection_change");
+        trace!("on_ice_connection_change (new_state: {:?})", new_state);
        let mut handler = self.on_ice_connection_change_handler.lock().unwrap();
        if let Some(f) = handler.as_mut() {
            f(new_state);
@@ -467,7 +467,7 @@ impl sys_pc::PeerConnectionObserver for InternalObserver {
    }

    fn on_standardized_ice_connection_change(&self, new_state: IceConnectionState) {
-        trace!("on_standardized_ice_connection_change");
+        trace!("on_standardized_ice_connection_change (new_state: {:?}", new_state);
        let mut handler = self
            .on_standardized_ice_connection_change_handler
            .lock()
@@ -478,7 +478,7 @@ impl sys_pc::PeerConnectionObserver for InternalObserver {
    }

    fn on_connection_change(&self, new_state: PeerConnectionState) {
-        trace!("on_connection_change");
+        trace!("on_connection_change (new_state: {:?})", new_state);
        let mut handler = self.on_connection_change_handler.lock().unwrap();
        if let Some(f) = handler.as_mut() {
            f(new_state);
@@ -486,7 +486,7 @@ impl sys_pc::PeerConnectionObserver for InternalObserver {
    }

    fn on_ice_gathering_change(&self, new_state: IceGatheringState) {
-        trace!("on_ice_gathering_change");
+        trace!("on_ice_gathering_change (new_state: {:?}", new_state);
        let mut handler = self.on_ice_gathering_change_handler.lock().unwrap();
        if let Some(f) = handler.as_mut() {
            f(new_state);
@@ -3,6 +3,8 @@ use libwebrtc_sys::video_frame as vf_sys;

 pub use vf_sys::ffi::VideoRotation;

+use crate::video_frame_buffer::VideoFrameBuffer;
+
 pub struct VideoFrame {
    cxx_handle: UniquePtr<vf_sys::ffi::VideoFrame>,
 }
@@ -31,7 +33,7 @@ impl VideoFrame {
    pub fn timestamp_us(&self) -> i64 {
        self.cxx_handle.timestamp_us()
    }
-    
+
    pub fn ntp_time_ms(&self) -> i64 {
        self.cxx_handle.ntp_time_ms()
    }
@@ -47,4 +49,12 @@ impl VideoFrame {
    pub fn rotation(&self) -> VideoRotation {
        self.cxx_handle.rotation()
    }
+
+    /// # Safety
+    /// Must be called only once, this function create the safe Rust
+    /// wrapper around a VideoFrameBuffer.
+    /// Only one wrapper musts exist at a time.
+    pub(crate) unsafe fn video_frame_buffer(&self) -> VideoFrameBuffer {
+        VideoFrameBuffer::new(self.cxx_handle.video_frame_buffer())
+    }
 }
@@ -1,18 +1,250 @@
 use cxx::UniquePtr;
 use libwebrtc_sys::video_frame_buffer as vfb_sys;
+use std::pin::Pin;
+use std::slice;
+use vfb_sys::ffi::VideoFrameBufferType;

-pub use vfb_sys::ffi::VideoFrameBufferType;
+pub trait VideoFrameBufferTrait {
+    fn width(&self) -> i32;
+    fn height(&self) -> i32;
+    fn to_i420(self) -> I420Buffer;
+}

-pub struct VideoFrameBuffer {
-    cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>,
+pub trait PlanarYuvBuffer: VideoFrameBufferTrait {
+    fn chroma_width(&self) -> i32;
+    fn chroma_height(&self) -> i32;
+    fn stride_y(&self) -> i32;
+    fn stride_u(&self) -> i32;
+    fn stride_v(&self) -> i32;
+}
+
+pub trait PlanarYuv8Buffer: PlanarYuvBuffer {
+    fn data_y(&self) -> &[u8];
+    fn data_u(&self) -> &[u8];
+    fn data_v(&self) -> &[u8];
+}
+
+pub enum VideoFrameBuffer {
+    Native(NativeBuffer),
+    I420(I420Buffer),
+    I420A(I420ABuffer),
+    I422(I422Buffer),
+    I444(I444Buffer),
+    I010(I010Buffer),
+    NV12(NV12Buffer),
 }

 impl VideoFrameBuffer {
-    pub fn new(cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>) -> Self {
+    pub(crate) fn new(mut cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>) -> Self {
+        unsafe {
+            match cxx_handle.buffer_type() {
+                VideoFrameBufferType::Native => Self::Native(NativeBuffer::new(cxx_handle)),
+                VideoFrameBufferType::I420 => {
+                    Self::I420(I420Buffer::new(cxx_handle.pin_mut().get_i420()))
+                }
+                VideoFrameBufferType::I420A => Self::I420A(I420ABuffer::new(cxx_handle)),
+                VideoFrameBufferType::I422 => Self::I422(I422Buffer::new(cxx_handle)),
+                VideoFrameBufferType::I444 => Self::I444(I444Buffer::new(cxx_handle)),
+                VideoFrameBufferType::I010 => Self::I010(I010Buffer::new(cxx_handle)),
+                VideoFrameBufferType::NV12 => Self::NV12(NV12Buffer::new(cxx_handle)),
+                _ => unreachable!(), // VideoFrameBufferType is represented as i32
+            }
+        }
+    }
+}
+
+macro_rules! recursive_cast {
+    ($ptr:expr $(, $fnc:ident)*) => {
+        {
+            let ptr = $ptr;
+            $(
+                let ptr = unsafe { vfb_sys::ffi::$fnc(ptr) };
+            )*
+            ptr
+        }
+    };
+}
+
+macro_rules! impl_video_frame_buffer {
+    ($x:ty $(, $cast:ident)*) => {
+
+        // Allow unused_unsafe when we don't do any cast ( e.g. NativeBuffer )
+        #[allow(unused_unsafe)]
+        impl VideoFrameBufferTrait for $x {
+            fn width(&self) -> i32 {
+                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
+                unsafe {
+                    (*ptr).width()
+                }
+            }
+
+            fn height(&self) -> i32 {
+                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
+                unsafe {
+                    (*ptr).height()
+                }
+            }
+
+            // Require ownership because libwebrtc uses the same pointers
+            fn to_i420(self) -> I420Buffer {
+                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*)
+                    as *const vfb_sys::ffi::VideoFrameBuffer
+                    as *mut vfb_sys::ffi::VideoFrameBuffer;
+
+                unsafe {
+                    I420Buffer::new(Pin::new_unchecked(&mut *ptr).to_i420())
+                }
+            }
+        }
+    };
+}
+
+macro_rules! impl_yuv_buffer {
+    ($x:ty $(, $cast:ident)*) => {
+        impl PlanarYuvBuffer for $x {
+            fn chroma_width(&self) -> i32 {
+                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
+                unsafe {
+                    (*ptr).chroma_width()
+                }
+            }
+
+            fn chroma_height(&self) -> i32 {
+                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
+                unsafe {
+                    (*ptr).chroma_height()
+                }
+            }
+
+            fn stride_y(&self) -> i32 {
+                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
+                unsafe {
+                    (*ptr).stride_y()
+                }
+            }
+
+            fn stride_u(&self) -> i32 {
+                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
+                unsafe {
+                    (*ptr).stride_u()
+                }
+            }
+
+            fn stride_v(&self) -> i32 {
+                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
+                unsafe {
+                    (*ptr).stride_v()
+                }
+            }
+        }
+    };
+}
+
+macro_rules! impl_yuv8_buffer {
+    ($x:ty $(, $cast:ident)*) => {
+        impl PlanarYuv8Buffer for $x {
+            fn data_y(&self) -> &[u8] {
+                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
+                unsafe {
+                    slice::from_raw_parts((*ptr).data_y(), self.stride_y().try_into().unwrap())
+                }
+            }
+
+            fn data_u(&self) -> &[u8] {
+                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
+                unsafe {
+                    slice::from_raw_parts((*ptr).data_u(), self.stride_u().try_into().unwrap())
+                }
+            }
+
+            fn data_v(&self) -> &[u8] {
+                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
+                unsafe {
+                    slice::from_raw_parts((*ptr).data_v(), self.stride_v().try_into().unwrap())
+                }
+            }
+        }
+    };
+}
+
+pub struct NativeBuffer {
+    cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>,
+}
+
+pub struct I420Buffer {
+    cxx_handle: UniquePtr<vfb_sys::ffi::I420Buffer>,
+}
+
+pub struct I420ABuffer {
+    cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>,
+}
+
+pub struct I422Buffer {
+    cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>,
+}
+
+pub struct I444Buffer {
+    cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>,
+}
+
+pub struct I010Buffer {
+    cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>,
+}
+
+pub struct NV12Buffer {
+    cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>,
+}
+
+impl_video_frame_buffer!(NativeBuffer);
+impl_video_frame_buffer!(I420Buffer, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
+impl_video_frame_buffer!(I420ABuffer);
+impl_video_frame_buffer!(I422Buffer);
+impl_video_frame_buffer!(I444Buffer);
+impl_video_frame_buffer!(I010Buffer);
+impl_video_frame_buffer!(NV12Buffer);
+
+impl_yuv_buffer!(I420Buffer, i420_to_yuv8, yuv8_to_yuv);
+
+impl_yuv8_buffer!(I420Buffer, i420_to_yuv8);
+
+impl NativeBuffer {
+    fn new(cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>) -> Self {
+        Self { cxx_handle }
+    }
+}
+
+impl I420Buffer {
+    fn new(cxx_handle: UniquePtr<vfb_sys::ffi::I420Buffer>) -> Self {
+        Self { cxx_handle }
+    }
+}
+
+impl I420ABuffer {
+    fn new(cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>) -> Self {
        Self { cxx_handle }
    }
+}

-    pub fn buffer_type(&self) -> VideoFrameBufferType {
-        self.cxx_handle.buffer_type()
+impl I422Buffer {
+    fn new(cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>) -> Self {
+        Self { cxx_handle }
+    }
+}
+
+impl I444Buffer {
+    fn new(cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>) -> Self {
+        Self { cxx_handle }
+    }
+}
+
+impl I010Buffer {
+    fn new(cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>) -> Self {
+        Self { cxx_handle }
+    }
+}
+
+impl NV12Buffer {
+    fn new(cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>) -> Self {
+        Self { cxx_handle }
    }
 }