trying to receive VideoFrame, progress...

crates/livekit-webrtc/libwebrtc-sys/include/livekit/media_stream.h

@@ -102,19 +102,19 @@ class NativeVideoFrameSink
 std::unique_ptr<NativeVideoFrameSink> create_native_video_frame_sink(
     rust::Box<VideoFrameSinkWrapper> observer);
 
-const MediaStreamTrack* video_to_media(const VideoTrack* track) {
+static const MediaStreamTrack* video_to_media(const VideoTrack* track) {
   return track;
 }
 
-const MediaStreamTrack* audio_to_media(const AudioTrack* track) {
+static const MediaStreamTrack* audio_to_media(const AudioTrack* track) {
   return track;
 }
 
-const VideoTrack* media_to_video(const MediaStreamTrack* track) {
+static const VideoTrack* media_to_video(const MediaStreamTrack* track) {
   return static_cast<const VideoTrack*>(track);
 }
 
-const AudioTrack* media_to_audio(const MediaStreamTrack* track) {
+static const AudioTrack* media_to_audio(const MediaStreamTrack* track) {
   return static_cast<const AudioTrack*>(track); 
 }
 

crates/livekit-webrtc/libwebrtc-sys/include/livekit/video_frame.h

@@ -30,8 +30,8 @@ class VideoFrame {
   }
 
   // TODO(theomonnom) This shouldn't create a new shared_ptr at each call
-  std::shared_ptr<VideoFrameBuffer> video_frame_buffer() const {
-    return std::make_shared<VideoFrameBuffer>(frame_.video_frame_buffer());
+  std::unique_ptr<VideoFrameBuffer> video_frame_buffer() const {
+    return std::make_unique<VideoFrameBuffer>(frame_.video_frame_buffer());
   }
 
  private:

crates/livekit-webrtc/libwebrtc-sys/include/livekit/video_frame_buffer.h

@@ -5,6 +5,8 @@
 #ifndef LIVEKIT_WEBRTC_VIDEO_FRAME_BUFFER_H
 #define LIVEKIT_WEBRTC_VIDEO_FRAME_BUFFER_H
 
+#include <memory>
+
 #include "api/video/video_frame_buffer.h"
 #include "rust_types.h"
 
@@ -26,8 +28,15 @@ class VideoFrameBuffer {
   int width() const { return buffer_->width(); }
   int height() const { return buffer_->height(); }
 
-  std::shared_ptr<I420Buffer> to_i420() {
-    return std::make_shared<I420Buffer>(buffer_->ToI420());
+  std::unique_ptr<I420Buffer> to_i420() {
+    return std::make_unique<I420Buffer>(buffer_->ToI420());
+  }
+
+  std::unique_ptr<I420Buffer> get_i420() {
+    // const_cast is valid here because we take the ownership on the rust side
+    return std::make_unique<I420Buffer>(
+        rtc::scoped_refptr<webrtc::I420BufferInterface>(
+            const_cast<webrtc::I420BufferInterface*>(buffer_->GetI420())));
   }
 
  protected:
@@ -73,19 +82,20 @@ class I420Buffer : public PlanarYuv8Buffer {
       : PlanarYuv8Buffer(buffer) {}
 };
 
-std::shared_ptr<VideoFrameBuffer> to_video_frame_buffer(
-    std::shared_ptr<PlanarYuvBuffer> buffer) {
-  return buffer;
+static const VideoFrameBuffer* yuv_to_vfb(const PlanarYuvBuffer* yuv) {
+  return yuv;
 }
 
-std::shared_ptr<PlanarYuvBuffer> to_yuv_buffer(
-    std::shared_ptr<PlanarYuv8Buffer> buffer) {
-  return buffer;
+static const PlanarYuvBuffer* yuv8_to_yuv(const PlanarYuv8Buffer* yuv8) {
+  return yuv8;
 }
 
-std::shared_ptr<PlanarYuv8Buffer> to_yuv8_buffer(
-    std::shared_ptr<I420Buffer> buffer) {
-  return buffer;
+static const PlanarYuv8Buffer* i420_to_yuv8(const I420Buffer* i420) {
+  return i420;
+}
+
+static std::unique_ptr<VideoFrameBuffer> _unique_video_frame_buffer() {
+  return nullptr;  // Ignore
 }
 
 }  // namespace livekit

crates/livekit-webrtc/libwebrtc-sys/src/media_stream.cpp

@@ -4,6 +4,7 @@
 
 #include "livekit/media_stream.h"
 
+#include "api/media_stream_interface.h"
 #include "libwebrtc-sys/src/media_stream.rs.h"
 
 namespace livekit {
@@ -53,6 +54,9 @@ rust::String MediaStream::id() const {
   return media_stream_->id();
 }
 
+AudioTrack::AudioTrack(rtc::scoped_refptr<webrtc::AudioTrackInterface> track)
+    : MediaStreamTrack(std::move(track)) {}
+
 VideoTrack::VideoTrack(rtc::scoped_refptr<webrtc::VideoTrackInterface> track)
     : MediaStreamTrack(std::move(track)) {}
 

crates/livekit-webrtc/libwebrtc-sys/src/video_frame.rs

@@ -25,7 +25,7 @@ pub mod ffi {
         fn transport_frame_id(self: &VideoFrame) -> u32;
         fn timestamp(self: &VideoFrame) -> u32;
         fn rotation(self: &VideoFrame) -> VideoRotation;
-        fn video_frame_buffer(self: &VideoFrame) -> SharedPtr<VideoFrameBuffer>;
+        fn video_frame_buffer(self: &VideoFrame) -> UniquePtr<VideoFrameBuffer>;
 
         fn _unique_video_frame() -> UniquePtr<VideoFrame>; // Ignore
     }

crates/livekit-webrtc/libwebrtc-sys/src/video_frame_buffer.rs

@@ -23,7 +23,11 @@ pub mod ffi {
         fn buffer_type(self: &VideoFrameBuffer) -> VideoFrameBufferType;
         fn width(self: &VideoFrameBuffer) -> i32;
         fn height(self: &VideoFrameBuffer) -> i32;
-        fn to_i420(self: Pin<&mut VideoFrameBuffer>) -> SharedPtr<I420Buffer>;
+
+        // Require ownership
+        unsafe fn to_i420(self: Pin<&mut VideoFrameBuffer>) -> UniquePtr<I420Buffer>;
+        unsafe fn get_i420(self: Pin<&mut VideoFrameBuffer>) -> UniquePtr<I420Buffer>;
+        // TODO(theomonnom): Bridge other get_*
 
         fn chroma_width(self: &PlanarYuvBuffer) -> i32;
         fn chroma_height(self: &PlanarYuvBuffer) -> i32;
@@ -35,9 +39,10 @@ pub mod ffi {
         fn data_u(self: &PlanarYuv8Buffer) -> *const u8;
         fn data_v(self: &PlanarYuv8Buffer) -> *const u8;
 
-        fn to_video_frame_buffer(buffer: SharedPtr<PlanarYuvBuffer>)
-            -> SharedPtr<VideoFrameBuffer>;
-        fn to_yuv_buffer(buffer: SharedPtr<PlanarYuv8Buffer>) -> SharedPtr<PlanarYuvBuffer>;
-        fn to_yuv8_buffer(buffer: SharedPtr<I420Buffer>) -> SharedPtr<PlanarYuv8Buffer>;
+        unsafe fn yuv_to_vfb(yuv: *const PlanarYuvBuffer) -> *const VideoFrameBuffer;
+        unsafe fn yuv8_to_yuv(yuv8: *const PlanarYuv8Buffer) -> *const PlanarYuvBuffer;
+        unsafe fn i420_to_yuv8(i420: *const I420Buffer) -> *const PlanarYuv8Buffer;
+
+        fn _unique_video_frame_buffer() -> UniquePtr<VideoFrameBuffer>;
     }
 }