feat: add publishing & audio to ffi (#46)

- Divide FFI protocol into multiple files
- Audio support
- AV Streams/Sources
- Create tracks
- Use Dashmap to store handles
- Add a capture test
     - Ignored by GHA atm

livekit-webrtc/src/native/audio_source.rs

@@ -20,14 +20,14 @@ impl NativeAudioSource {
         self.sys_handle.clone()
     }
 
-    pub fn capture_frame(&self, frame: AudioFrame) {
+    pub fn capture_frame(&self, frame: &AudioFrame) {
         // TODO(theomonnom): Should we check for 10ms worth of data here?
         unsafe {
             self.sys_handle.on_captured_frame(
                 frame.data.as_ptr(),
-                frame.sample_rate_hz as i32,
-                frame.num_channels,
-                frame.samples_per_channel,
+                frame.sample_rate as i32,
+                frame.num_channels as usize,
+                frame.samples_per_channel as usize,
             )
         }
     }

livekit-webrtc/src/native/audio_stream.rs

@@ -72,9 +72,9 @@ impl sys_ms::AudioSink for AudioTrackObserver {
         // TODO(theomonnom): Should we avoid copy here?
         let _ = self.frame_tx.send(AudioFrame {
             data: data.to_owned(),
-            sample_rate_hz: sample_rate as u32,
-            num_channels: nb_channels,
-            samples_per_channel: nb_frames,
+            sample_rate: sample_rate as u32,
+            num_channels: nb_channels as u32,
+            samples_per_channel: nb_frames as u32,
         });
     }
 }

livekit-webrtc/src/native/video_frame.rs

@@ -149,11 +149,11 @@ impl NativeBuffer {
         &*self.sys_handle
     }
 
-    pub fn width(&self) -> i32 {
+    pub fn width(&self) -> u32 {
         self.sys_handle.width()
     }
 
-    pub fn height(&self) -> i32 {
+    pub fn height(&self) -> u32 {
         self.sys_handle.height()
     }
 
@@ -167,7 +167,7 @@ impl NativeBuffer {
         &self,
         format: VideoFormatType,
         dst: &mut [u8],
-        dst_stride: i32,
+        dst_stride: u32,
         dst_width: i32,
         dst_height: i32,
     ) -> Result<(), ConvertError> {
@@ -192,49 +192,49 @@ impl I420Buffer {
         unsafe { &*recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb) }
     }
 
-    pub fn width(&self) -> i32 {
+    pub fn width(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
             (*ptr).width()
         }
     }
 
-    pub fn height(&self) -> i32 {
+    pub fn height(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
             (*ptr).height()
         }
     }
 
-    pub fn chroma_width(&self) -> i32 {
+    pub fn chroma_width(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
             (*ptr).chroma_width()
         }
     }
 
-    pub fn chroma_height(&self) -> i32 {
+    pub fn chroma_height(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
             (*ptr).chroma_height()
         }
     }
 
-    pub fn stride_y(&self) -> i32 {
+    pub fn stride_y(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
             (*ptr).stride_y()
         }
     }
 
-    pub fn stride_u(&self) -> i32 {
+    pub fn stride_u(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
             (*ptr).stride_u()
         }
     }
 
-    pub fn stride_v(&self) -> i32 {
+    pub fn stride_v(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
             (*ptr).stride_v()
@@ -258,7 +258,7 @@ impl I420Buffer {
         &self,
         format: VideoFormatType,
         dst: &mut [u8],
-        dst_stride: i32,
+        dst_stride: u32,
         dst_width: i32,
         dst_height: i32,
     ) -> Result<(), ConvertError> {
@@ -292,56 +292,56 @@ impl I420ABuffer {
         unsafe { &*recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb) }
     }
 
-    pub fn width(&self) -> i32 {
+    pub fn width(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
             (*ptr).width()
         }
     }
 
-    pub fn height(&self) -> i32 {
+    pub fn height(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
             (*ptr).height()
         }
     }
 
-    pub fn chroma_width(&self) -> i32 {
+    pub fn chroma_width(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
             (*ptr).chroma_width()
         }
     }
 
-    pub fn chroma_height(&self) -> i32 {
+    pub fn chroma_height(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
             (*ptr).chroma_height()
         }
     }
 
-    pub fn stride_y(&self) -> i32 {
+    pub fn stride_y(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
             (*ptr).stride_y()
         }
     }
 
-    pub fn stride_u(&self) -> i32 {
+    pub fn stride_u(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
             (*ptr).stride_u()
         }
     }
 
-    pub fn stride_v(&self) -> i32 {
+    pub fn stride_v(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
             (*ptr).stride_v()
         }
     }
 
-    pub fn stride_a(&self) -> i32 {
+    pub fn stride_a(&self) -> u32 {
         self.sys_handle.stride_a()
     }
 
@@ -359,7 +359,7 @@ impl I420ABuffer {
         &self,
         format: VideoFormatType,
         dst: &mut [u8],
-        dst_stride: i32,
+        dst_stride: u32,
         dst_width: i32,
         dst_height: i32,
     ) -> Result<(), ConvertError> {
@@ -391,49 +391,49 @@ impl I422Buffer {
         unsafe { &*recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb) }
     }
 
-    pub fn width(&self) -> i32 {
+    pub fn width(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
             (*ptr).width()
         }
     }
 
-    pub fn height(&self) -> i32 {
+    pub fn height(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
             (*ptr).height()
         }
     }
 
-    pub fn chroma_width(&self) -> i32 {
+    pub fn chroma_width(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
             (*ptr).chroma_width()
         }
     }
 
-    pub fn chroma_height(&self) -> i32 {
+    pub fn chroma_height(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
             (*ptr).chroma_height()
         }
     }
 
-    pub fn stride_y(&self) -> i32 {
+    pub fn stride_y(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
             (*ptr).stride_y()
         }
     }
 
-    pub fn stride_u(&self) -> i32 {
+    pub fn stride_u(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
             (*ptr).stride_u()
         }
     }
 
-    pub fn stride_v(&self) -> i32 {
+    pub fn stride_v(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
             (*ptr).stride_v()
@@ -453,7 +453,7 @@ impl I422Buffer {
         &self,
         format: VideoFormatType,
         dst: &mut [u8],
-        dst_stride: i32,
+        dst_stride: u32,
         dst_width: i32,
         dst_height: i32,
     ) -> Result<(), ConvertError> {
@@ -477,49 +477,49 @@ impl I444Buffer {
         unsafe { &*recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb) }
     }
 
-    pub fn width(&self) -> i32 {
+    pub fn width(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
             (*ptr).width()
         }
     }
 
-    pub fn height(&self) -> i32 {
+    pub fn height(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
             (*ptr).height()
         }
     }
 
-    pub fn chroma_width(&self) -> i32 {
+    pub fn chroma_width(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
             (*ptr).chroma_width()
         }
     }
 
-    pub fn chroma_height(&self) -> i32 {
+    pub fn chroma_height(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
             (*ptr).chroma_height()
         }
     }
 
-    pub fn stride_y(&self) -> i32 {
+    pub fn stride_y(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
             (*ptr).stride_y()
         }
     }
 
-    pub fn stride_u(&self) -> i32 {
+    pub fn stride_u(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
             (*ptr).stride_u()
         }
     }
 
-    pub fn stride_v(&self) -> i32 {
+    pub fn stride_v(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
             (*ptr).stride_v()
@@ -539,7 +539,7 @@ impl I444Buffer {
         &self,
         format: VideoFormatType,
         dst: &mut [u8],
-        dst_stride: i32,
+        dst_stride: u32,
         dst_width: i32,
         dst_height: i32,
     ) -> Result<(), ConvertError> {
@@ -564,49 +564,49 @@ impl I010Buffer {
         unsafe { &*recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb) }
     }
 
-    pub fn width(&self) -> i32 {
+    pub fn width(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb);
             (*ptr).width()
         }
     }
 
-    pub fn height(&self) -> i32 {
+    pub fn height(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb);
             (*ptr).height()
         }
     }
 
-    pub fn chroma_width(&self) -> i32 {
+    pub fn chroma_width(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
             (*ptr).chroma_width()
         }
     }
 
-    pub fn chroma_height(&self) -> i32 {
+    pub fn chroma_height(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
             (*ptr).chroma_height()
         }
     }
 
-    pub fn stride_y(&self) -> i32 {
+    pub fn stride_y(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
             (*ptr).stride_y()
         }
     }
 
-    pub fn stride_u(&self) -> i32 {
+    pub fn stride_u(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
             (*ptr).stride_u()
         }
     }
 
-    pub fn stride_v(&self) -> i32 {
+    pub fn stride_v(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
             (*ptr).stride_v()
@@ -627,7 +627,7 @@ impl I010Buffer {
         &self,
         format: VideoFormatType,
         dst: &mut [u8],
-        dst_stride: i32,
+        dst_stride: u32,
         dst_width: i32,
         dst_height: i32,
     ) -> Result<(), ConvertError> {
@@ -669,7 +669,7 @@ impl NV12Buffer {
         }
     }
 
-    pub fn width(&self) -> i32 {
+    pub fn width(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(
                 &*self.sys_handle,
@@ -681,7 +681,7 @@ impl NV12Buffer {
         }
     }
 
-    pub fn height(&self) -> i32 {
+    pub fn height(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(
                 &*self.sys_handle,
@@ -693,28 +693,28 @@ impl NV12Buffer {
         }
     }
 
-    pub fn chroma_width(&self) -> i32 {
+    pub fn chroma_width(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv);
             (*ptr).chroma_width()
         }
     }
 
-    pub fn chroma_height(&self) -> i32 {
+    pub fn chroma_height(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv);
             (*ptr).chroma_height()
         }
     }
 
-    pub fn stride_y(&self) -> i32 {
+    pub fn stride_y(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv);
             (*ptr).stride_y()
         }
     }
 
-    pub fn stride_uv(&self) -> i32 {
+    pub fn stride_uv(&self) -> u32 {
         unsafe {
             let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv);
             (*ptr).stride_uv()
@@ -739,7 +739,7 @@ impl NV12Buffer {
         &self,
         format: VideoFormatType,
         dst: &mut [u8],
-        dst_stride: i32,
+        dst_stride: u32,
         dst_width: i32,
         dst_height: i32,
     ) -> Result<(), ConvertError> {

livekit-webrtc/src/native/video_source.rs

@@ -21,12 +21,12 @@ impl NativeVideoSource {
         self.sys_handle.clone()
     }
 
-    pub fn capture_frame<T: VideoFrameBuffer>(&self, frame: &VideoFrame<T>) {
+    pub fn capture_frame<T: AsRef<dyn VideoFrameBuffer>>(&self, frame: &VideoFrame<T>) {
         let mut builder = vf_sys::ffi::new_video_frame_builder();
         builder.pin_mut().set_rotation(frame.rotation.into());
         builder
             .pin_mut()
-            .set_video_frame_buffer(frame.buffer.sys_handle());
+            .set_video_frame_buffer(frame.buffer.as_ref().sys_handle());
 
         let frame = builder.pin_mut().build();
         self.sys_handle.on_captured_frame(&frame);

livekit-webrtc/src/native/yuv_helper.rs

@@ -10,11 +10,12 @@ pub enum ConvertError {
 #[inline]
 fn argb_assert_safety(
     src: &[u8],
-    src_stride: i32,
+    src_stride: u32,
     _width: i32,
     height: i32,
 ) -> Result<(), ConvertError> {
-    let min = (src_stride * height) as usize;
+    let height_abs = height.abs() as u32;
+    let min = (src_stride * height_abs) as usize;
 
     if src.len() < min {
         return Err(ConvertError::Convert("dst isn't large enough"));
@@ -26,16 +27,17 @@ fn argb_assert_safety(
 #[inline]
 fn i420_assert_safety(
     src_y: &[u8],
-    src_stride_y: i32,
+    src_stride_y: u32,
     src_u: &[u8],
-    src_stride_u: i32,
+    src_stride_u: u32,
     src_v: &[u8],
-    src_stride_v: i32,
+    src_stride_v: u32,
     _width: i32,
     height: i32,
 ) -> Result<(), ConvertError> {
-    let chroma_height = (height + 1) / 2;
-    let min_y = (src_stride_y * height) as usize;
+    let height_abs = height.abs() as u32;
+    let chroma_height = (height_abs + 1) / 2;
+    let min_y = (src_stride_y * height_abs) as usize;
     let min_u = (src_stride_u * chroma_height) as usize;
     let min_v = (src_stride_v * chroma_height) as usize;
 
@@ -58,13 +60,13 @@ macro_rules! i420_to_x {
     ($x:ident) => {
         pub fn $x(
             src_y: &[u8],
-            src_stride_y: i32,
+            src_stride_y: u32,
             src_u: &[u8],
-            src_stride_u: i32,
+            src_stride_u: u32,
             src_v: &[u8],
-            src_stride_v: i32,
+            src_stride_v: u32,
             dst: &mut [u8],
-            dst_stride: i32,
+            dst_stride: u32,
             width: i32,
             height: i32,
         ) -> Result<(), ConvertError> {
@@ -83,13 +85,13 @@ macro_rules! i420_to_x {
             unsafe {
                 yuv_sys::ffi::$x(
                     src_y.as_ptr(),
-                    src_stride_y,
+                    src_stride_y as i32,
                     src_u.as_ptr(),
-                    src_stride_u,
+                    src_stride_u as i32,
                     src_v.as_ptr(),
-                    src_stride_v,
+                    src_stride_v as i32,
                     dst.as_mut_ptr(),
-                    dst_stride,
+                    dst_stride as i32,
                     width,
                     height,
                 )
@@ -105,13 +107,13 @@ macro_rules! x_to_i420 {
     ($x:ident) => {
         pub fn $x(
             src_argb: &[u8],
-            src_stride_argb: i32,
+            src_stride_argb: u32,
             dst_y: &mut [u8],
-            dst_stride_y: i32,
+            dst_stride_y: u32,
             dst_u: &mut [u8],
-            dst_stride_u: i32,
+            dst_stride_u: u32,
             dst_v: &mut [u8],
-            dst_stride_v: i32,
+            dst_stride_v: u32,
             width: i32,
             height: i32,
         ) -> Result<(), ConvertError> {
@@ -130,13 +132,13 @@ macro_rules! x_to_i420 {
             unsafe {
                 yuv_sys::ffi::$x(
                     src_argb.as_ptr(),
-                    src_stride_argb,
+                    src_stride_argb as i32,
                     dst_y.as_mut_ptr(),
-                    dst_stride_y,
+                    dst_stride_y as i32,
                     dst_u.as_mut_ptr(),
-                    dst_stride_u,
+                    dst_stride_u as i32,
                     dst_v.as_mut_ptr(),
-                    dst_stride_v,
+                    dst_stride_v as i32,
                     width,
                     height,
                 )
@@ -150,9 +152,9 @@ macro_rules! x_to_i420 {
 
 pub fn argb_to_rgb24(
     src_argb: &[u8],
-    src_stride_argb: i32,
+    src_stride_argb: u32,
     dst_rgb24: &mut [u8],
-    dst_stride_rgb24: i32,
+    dst_stride_rgb24: u32,
     width: i32,
     height: i32,
 ) -> Result<(), ConvertError> {
@@ -162,9 +164,9 @@ pub fn argb_to_rgb24(
     unsafe {
         yuv_sys::ffi::argb_to_rgb24(
             src_argb.as_ptr(),
-            src_stride_argb,
+            src_stride_argb as i32,
             dst_rgb24.as_mut_ptr(),
-            dst_stride_rgb24,
+            dst_stride_rgb24 as i32,
             width,
             height,
         )