feat: basic ffi server to support other languages (#34)

* wip

* wip

* wip

* wip

not a fan of the way I do handles ..

* errors and async semantic

* fix example

webrtc-sys/include/livekit/video_frame_buffer.h

@@ -12,9 +12,12 @@
 
 namespace livekit {
 
-class PlanarYuvBuffer;
-class PlanarYuv8Buffer;
 class I420Buffer;
+class I420ABuffer;
+class I422Buffer;
+class I444Buffer;
+class I010Buffer;
+class NV12Buffer;
 
 class VideoFrameBuffer {
  public:
@@ -32,13 +35,43 @@ class VideoFrameBuffer {
     return std::make_unique<I420Buffer>(buffer_->ToI420());
   }
 
+  // const_cast is valid here because we take the ownership on the rust side
   std::unique_ptr<I420Buffer> get_i420() {
-    // const_cast is valid here because we take the ownership on the rust side
     return std::make_unique<I420Buffer>(
         rtc::scoped_refptr<webrtc::I420BufferInterface>(
             const_cast<webrtc::I420BufferInterface*>(buffer_->GetI420())));
   }
 
+  std::unique_ptr<I420ABuffer> get_i420a() {
+    return std::make_unique<I420ABuffer>(
+        rtc::scoped_refptr<webrtc::I420ABufferInterface>(
+            const_cast<webrtc::I420ABufferInterface*>(buffer_->GetI420A())));
+  }
+
+  std::unique_ptr<I422Buffer> get_i422() {
+    return std::make_unique<I422Buffer>(
+        rtc::scoped_refptr<webrtc::I422BufferInterface>(
+            const_cast<webrtc::I422BufferInterface*>(buffer_->GetI422())));
+  }
+
+  std::unique_ptr<I444Buffer> get_i444() {
+    return std::make_unique<I444Buffer>(
+        rtc::scoped_refptr<webrtc::I444BufferInterface>(
+            const_cast<webrtc::I444BufferInterface*>(buffer_->GetI444())));
+  }
+
+  std::unique_ptr<I010Buffer> get_i010() {
+    return std::make_unique<I010Buffer>(
+        rtc::scoped_refptr<webrtc::I010BufferInterface>(
+            const_cast<webrtc::I010BufferInterface*>(buffer_->GetI010())));
+  }
+
+  std::unique_ptr<NV12Buffer> get_nv12() {
+    return std::make_unique<NV12Buffer>(
+        rtc::scoped_refptr<webrtc::NV12BufferInterface>(
+            const_cast<webrtc::NV12BufferInterface*>(buffer_->GetNV12())));
+  }
+
  protected:
   rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer_;
 };
@@ -76,24 +109,136 @@ class PlanarYuv8Buffer : public PlanarYuvBuffer {
   }
 };
 
+class PlanarYuv16BBuffer : public PlanarYuvBuffer {
+ public:
+  explicit PlanarYuv16BBuffer(
+      rtc::scoped_refptr<webrtc::PlanarYuv16BBuffer> buffer)
+      : PlanarYuvBuffer(buffer) {}
+
+  const uint16_t* data_y() const { return buffer()->DataY(); }
+  const uint16_t* data_u() const { return buffer()->DataU(); }
+  const uint16_t* data_v() const { return buffer()->DataV(); }
+
+ private:
+  webrtc::PlanarYuv16BBuffer* buffer() const {
+    return static_cast<webrtc::PlanarYuv16BBuffer*>(buffer_.get());
+  }
+};
+
+class BiplanarYuvBuffer : public VideoFrameBuffer {
+ public:
+  explicit BiplanarYuvBuffer(
+      rtc::scoped_refptr<webrtc::BiplanarYuvBuffer> buffer)
+      : VideoFrameBuffer(buffer) {}
+
+  int chroma_width() const { return buffer()->ChromaWidth(); }
+  int chroma_height() const { return buffer()->ChromaHeight(); }
+
+  int stride_y() const { return buffer()->StrideY(); }
+  int stride_uv() const { return buffer()->StrideUV(); }
+
+ private:
+  webrtc::BiplanarYuvBuffer* buffer() const {
+    return static_cast<webrtc::BiplanarYuvBuffer*>(buffer_.get());
+  }
+};
+
+class BiplanarYuv8Buffer : public BiplanarYuvBuffer {
+ public:
+  explicit BiplanarYuv8Buffer(
+      rtc::scoped_refptr<webrtc::BiplanarYuv8Buffer> buffer)
+      : BiplanarYuvBuffer(buffer) {}
+
+  const uint8_t* data_y() const { return buffer()->DataY(); }
+  const uint8_t* data_uv() const { return buffer()->DataUV(); }
+
+ private:
+  webrtc::BiplanarYuv8Buffer* buffer() const {
+    return static_cast<webrtc::BiplanarYuv8Buffer*>(buffer_.get());
+  }
+};
+
 class I420Buffer : public PlanarYuv8Buffer {
  public:
   explicit I420Buffer(rtc::scoped_refptr<webrtc::I420BufferInterface> buffer)
       : PlanarYuv8Buffer(buffer) {}
 };
 
+class I420ABuffer : public I420Buffer {
+ public:
+  explicit I420ABuffer(rtc::scoped_refptr<webrtc::I420ABufferInterface> buffer)
+      : I420Buffer(buffer) {}
+};
+
+class I422Buffer : public PlanarYuv8Buffer {
+ public:
+  explicit I422Buffer(rtc::scoped_refptr<webrtc::I422BufferInterface> buffer)
+      : PlanarYuv8Buffer(buffer) {}
+};
+
+class I444Buffer : public PlanarYuv8Buffer {
+ public:
+  explicit I444Buffer(rtc::scoped_refptr<webrtc::I444BufferInterface> buffer)
+      : PlanarYuv8Buffer(buffer) {}
+};
+
+class I010Buffer : public PlanarYuv16BBuffer {
+ public:
+  explicit I010Buffer(rtc::scoped_refptr<webrtc::I010BufferInterface> buffer)
+      : PlanarYuv16BBuffer(buffer) {}
+};
+
+class NV12Buffer : public BiplanarYuv8Buffer {
+ public:
+  explicit NV12Buffer(rtc::scoped_refptr<webrtc::NV12BufferInterface> buffer)
+      : BiplanarYuv8Buffer(buffer) {}
+};
+
 static const VideoFrameBuffer* yuv_to_vfb(const PlanarYuvBuffer* yuv) {
   return yuv;
 }
 
+static const VideoFrameBuffer* biyuv_to_vfb(const BiplanarYuvBuffer* biyuv) {
+  return biyuv;
+}
+
 static const PlanarYuvBuffer* yuv8_to_yuv(const PlanarYuv8Buffer* yuv8) {
   return yuv8;
 }
 
+static const PlanarYuvBuffer* yuv16b_to_yuv(const PlanarYuv16BBuffer* yuv16) {
+  return yuv16;
+}
+
+static const BiplanarYuvBuffer* biyuv8_to_biyuv(
+    const BiplanarYuv8Buffer* biyuv8) {
+  return biyuv8;
+}
+
 static const PlanarYuv8Buffer* i420_to_yuv8(const I420Buffer* i420) {
   return i420;
 }
 
+static const PlanarYuv8Buffer* i420a_to_yuv8(const I420ABuffer* i420a) {
+  return i420a;
+}
+
+static const PlanarYuv8Buffer* i422_to_yuv8(const I422Buffer* i422) {
+  return i422;
+}
+
+static const PlanarYuv8Buffer* i444_to_yuv8(const I444Buffer* i444) {
+  return i444;
+}
+
+static const PlanarYuv16BBuffer* i010_to_yuv16b(const I010Buffer* i010) {
+  return i010;
+}
+
+static const BiplanarYuv8Buffer* nv12_to_biyuv8(const NV12Buffer* nv12) {
+  return nv12;
+}
+
 static std::unique_ptr<VideoFrameBuffer> _unique_video_frame_buffer() {
   return nullptr;  // Ignore
 }

webrtc-sys/src/lib.rs

@@ -15,3 +15,16 @@ pub mod yuv_helper;
 pub const MEDIA_TYPE_VIDEO: &str = "video";
 pub const MEDIA_TYPE_AUDIO: &str = "audio";
 pub const MEDIA_TYPE_DATA: &str = "data";
+
+macro_rules! impl_thread_safety {
+    ($obj:ty, Send) => {
+        unsafe impl Send for $obj {}
+    };
+
+    ($obj:ty, Send + Sync) => {
+        unsafe impl Send for $obj {}
+        unsafe impl Sync for $obj {}
+    };
+}
+
+pub(crate) use impl_thread_safety;

webrtc-sys/src/video_frame.rs

@@ -1,3 +1,5 @@
+use crate::impl_thread_safety;
+
 #[cxx::bridge(namespace = "livekit")]
 pub mod ffi {
     #[derive(Debug)]
@@ -30,3 +32,5 @@ pub mod ffi {
         fn _unique_video_frame() -> UniquePtr<VideoFrame>; // Ignore
     }
 }
+
+impl_thread_safety!(ffi::VideoFrame, Send + Sync);

webrtc-sys/src/video_frame_buffer.rs

@@ -1,3 +1,5 @@
+use crate::impl_thread_safety;
+
 #[cxx::bridge(namespace = "livekit")]
 pub mod ffi {
     #[derive(Debug)]
@@ -18,16 +20,29 @@ pub mod ffi {
         type VideoFrameBuffer;
         type PlanarYuvBuffer;
         type PlanarYuv8Buffer;
+        type PlanarYuv16BBuffer;
+        type BiplanarYuvBuffer;
+        type BiplanarYuv8Buffer;
         type I420Buffer;
+        type I420ABuffer;
+        type I422Buffer;
+        type I444Buffer;
+        type I010Buffer;
+        type NV12Buffer;
 
         fn buffer_type(self: &VideoFrameBuffer) -> VideoFrameBufferType;
         fn width(self: &VideoFrameBuffer) -> i32;
         fn height(self: &VideoFrameBuffer) -> i32;
 
-        // Require ownership
+        /// # SAFETY
+        /// The functions require ownership
         unsafe fn to_i420(self: Pin<&mut VideoFrameBuffer>) -> UniquePtr<I420Buffer>;
         unsafe fn get_i420(self: Pin<&mut VideoFrameBuffer>) -> UniquePtr<I420Buffer>;
-        // TODO(theomonnom): Bridge other get_*
+        unsafe fn get_i420a(self: Pin<&mut VideoFrameBuffer>) -> UniquePtr<I420ABuffer>;
+        unsafe fn get_i422(self: Pin<&mut VideoFrameBuffer>) -> UniquePtr<I422Buffer>;
+        unsafe fn get_i444(self: Pin<&mut VideoFrameBuffer>) -> UniquePtr<I444Buffer>;
+        unsafe fn get_i010(self: Pin<&mut VideoFrameBuffer>) -> UniquePtr<I010Buffer>;
+        unsafe fn get_nv12(self: Pin<&mut VideoFrameBuffer>) -> UniquePtr<NV12Buffer>;
 
         fn chroma_width(self: &PlanarYuvBuffer) -> i32;
         fn chroma_height(self: &PlanarYuvBuffer) -> i32;
@@ -39,10 +54,43 @@ pub mod ffi {
         fn data_u(self: &PlanarYuv8Buffer) -> *const u8;
         fn data_v(self: &PlanarYuv8Buffer) -> *const u8;
 
+        fn data_y(self: &PlanarYuv16BBuffer) -> *const u16;
+        fn data_u(self: &PlanarYuv16BBuffer) -> *const u16;
+        fn data_v(self: &PlanarYuv16BBuffer) -> *const u16;
+
+        fn chroma_width(self: &BiplanarYuvBuffer) -> i32;
+        fn chroma_height(self: &BiplanarYuvBuffer) -> i32;
+        fn stride_y(self: &BiplanarYuvBuffer) -> i32;
+        fn stride_uv(self: &BiplanarYuvBuffer) -> i32;
+
+        fn data_y(self: &BiplanarYuv8Buffer) -> *const u8;
+        fn data_uv(self: &BiplanarYuv8Buffer) -> *const u8;
+
         unsafe fn yuv_to_vfb(yuv: *const PlanarYuvBuffer) -> *const VideoFrameBuffer;
+        unsafe fn biyuv_to_vfb(yuv: *const BiplanarYuvBuffer) -> *const VideoFrameBuffer;
         unsafe fn yuv8_to_yuv(yuv8: *const PlanarYuv8Buffer) -> *const PlanarYuvBuffer;
+        unsafe fn yuv16b_to_yuv(yuv16b: *const PlanarYuv16BBuffer) -> *const PlanarYuvBuffer;
+        unsafe fn biyuv8_to_biyuv(biyuv8: *const BiplanarYuv8Buffer) -> *const BiplanarYuvBuffer;
         unsafe fn i420_to_yuv8(i420: *const I420Buffer) -> *const PlanarYuv8Buffer;
+        unsafe fn i420a_to_yuv8(i420a: *const I420ABuffer) -> *const PlanarYuv8Buffer;
+        unsafe fn i422_to_yuv8(i422: *const I422Buffer) -> *const PlanarYuv8Buffer;
+        unsafe fn i444_to_yuv8(i444: *const I444Buffer) -> *const PlanarYuv8Buffer;
+        unsafe fn i010_to_yuv16b(i010: *const I010Buffer) -> *const PlanarYuv16BBuffer;
+        unsafe fn nv12_to_biyuv8(nv12: *const NV12Buffer) -> *const BiplanarYuv8Buffer;
 
         fn _unique_video_frame_buffer() -> UniquePtr<VideoFrameBuffer>;
     }
 }
+
+impl_thread_safety!(ffi::VideoFrameBuffer, Send + Sync);
+impl_thread_safety!(ffi::PlanarYuvBuffer, Send + Sync);
+impl_thread_safety!(ffi::PlanarYuv8Buffer, Send + Sync);
+impl_thread_safety!(ffi::PlanarYuv16BBuffer, Send + Sync);
+impl_thread_safety!(ffi::BiplanarYuvBuffer, Send + Sync);
+impl_thread_safety!(ffi::BiplanarYuv8Buffer, Send + Sync);
+impl_thread_safety!(ffi::I420Buffer, Send + Sync);
+impl_thread_safety!(ffi::I420ABuffer, Send + Sync);
+impl_thread_safety!(ffi::I422Buffer, Send + Sync);
+impl_thread_safety!(ffi::I444Buffer, Send + Sync);
+impl_thread_safety!(ffi::I010Buffer, Send + Sync);
+impl_thread_safety!(ffi::NV12Buffer, Send + Sync);