Initial downstream tracks

crates/livekit-webrtc/libwebrtc-sys/CMakeLists.txt

@@ -1,23 +0,0 @@
-# IMPORTANT NOTE
-# This file is just used because some IDEs need to understand how to do autocompletion.
-# This file is completely ignored by the library ( See build.rs for the build system )
-
-cmake_minimum_required(VERSION 3.22)
-project(livekit-webrtc)
-
-set(CMAKE_CXX_STANDARD 17)
-
-add_definitions(-DWEBRTC_WIN)
-
-include_directories(libwebrtc/include)
-include_directories(libwebrtc/include/third_party/abseil-cpp/)
-include_directories(libwebrtc/include/third_party/libc++/)
-include_directories(include/)
-include_directories(../../../target/cxxbridge) # Can be different
-
-file(GLOB_RECURSE SRC src/*.cpp)
-add_library(livekit-webrtc ${SRC})
-
-#include_directories(/Users/theomonnom/Library/Android/sdk/ndk/25.0.8775105/toolchains/llvm/prebuilt/darwin-x86_64/sysroot/usr/include)
-#find_library(ANDROID_LIB_ANDROID android)
-#target_link_libraries(client_sdk_native PRIVATE android)

crates/livekit-webrtc/libwebrtc-sys/build.rs

@@ -52,7 +52,7 @@ fn macos_link_search_path() -> Option<String> {
 
 fn main() {
     // TODO Download precompiled binaries of WebRTC for the target_os
-    let target_os = "macos";
+    let target_os = "windows";
     //let target_arch = "arm64";
 
     let libwebrtc_dir = path::PathBuf::from("libwebrtc");
@@ -79,6 +79,8 @@ fn main() {
         "src/rtp_transceiver.rs",
         "src/rtc_error.rs",
         "src/webrtc.rs",
+        "src/video_frame.rs",
+        "src/video_frame_buffer.rs",
     ]);
 
     builder.file("src/peer_connection.cpp");

crates/livekit-webrtc/libwebrtc-sys/compile_flags.txt

@@ -0,0 +1,8 @@
+-xc++
+-std=c++17
+-Iinclude
+-Ilibwebrtc/include
+-Ilibwebrtc/include/third_party/abseil-cpp
+-Ilibwebrtc/include/third_party/libc++
+-I../../../target/cxxbridge
+-DWEBRTC_WIN

crates/livekit-webrtc/libwebrtc-sys/include/livekit/media_stream.h

@@ -13,26 +13,7 @@
 
 namespace livekit {
 
-class MediaStreamTrack {
- public:
-  explicit MediaStreamTrack(
-      rtc::scoped_refptr<webrtc::MediaStreamTrackInterface> track);
-
-  rust::String kind() const;
-  rust::String id() const;
-
-  bool enabled() const;
-  bool set_enabled(bool enable);
-
-  TrackState state() const;
-
- private:
-  rtc::scoped_refptr<webrtc::MediaStreamTrackInterface> track_;
-};
-
-static std::unique_ptr<MediaStreamTrack> _unique_media_stream_track() {
-  return nullptr;  // Ignore
-}
+class NativeVideoFrameSink;
 
 class MediaStream {
  public:
@@ -47,6 +28,96 @@ class MediaStream {
 static std::unique_ptr<MediaStream> _unique_media_stream() {
   return nullptr;  // Ignore
 }
+
+class MediaStreamTrack {
+ protected:
+  explicit MediaStreamTrack(
+      rtc::scoped_refptr<webrtc::MediaStreamTrackInterface> track);
+
+ public:
+  static std::unique_ptr<MediaStreamTrack> from(
+      rtc::scoped_refptr<webrtc::MediaStreamTrackInterface> track);
+
+  rust::String kind() const;
+  rust::String id() const;
+
+  bool enabled() const;
+  bool set_enabled(bool enable);
+
+  TrackState state() const;
+
+ protected:
+  rtc::scoped_refptr<webrtc::MediaStreamTrackInterface> track_;
+};
+
+static std::unique_ptr<MediaStreamTrack> _unique_media_stream_track() {
+  return nullptr;  // Ignore
+}
+
+class AudioTrack : public MediaStreamTrack {
+ public:
+  explicit AudioTrack(rtc::scoped_refptr<webrtc::AudioTrackInterface> track);
+};
+
+static std::unique_ptr<AudioTrack> _unique_audio_track() {
+  return nullptr;  // Ignore
+}
+
+class VideoTrack : public MediaStreamTrack {
+ public:
+  explicit VideoTrack(rtc::scoped_refptr<webrtc::VideoTrackInterface> track);
+
+  void add_sink(NativeVideoFrameSink& sink);
+  void remove_sink(NativeVideoFrameSink& sink);
+
+  void set_should_receive(bool should_receive);
+  bool should_receive() const;
+  ContentHint content_hint() const;
+  void set_content_hint(ContentHint hint);
+
+ private:
+  webrtc::VideoTrackInterface* track() const {
+    return static_cast<webrtc::VideoTrackInterface*>(track_.get());
+  }
+};
+
+static std::unique_ptr<VideoTrack> _unique_video_track() {
+  return nullptr;  // Ignore
+}
+
+class NativeVideoFrameSink
+    : public rtc::VideoSinkInterface<webrtc::VideoFrame> {
+ public:
+  explicit NativeVideoFrameSink(rust::Box<VideoFrameSinkWrapper> observer);
+
+  void OnFrame(const webrtc::VideoFrame& frame) override;
+  void OnDiscardedFrame() override;
+  void OnConstraintsChanged(
+      const webrtc::VideoTrackSourceConstraints& constraints) override;
+
+ private:
+  rust::Box<VideoFrameSinkWrapper> observer_;
+};
+
+std::unique_ptr<NativeVideoFrameSink> create_native_video_frame_sink(
+    rust::Box<VideoFrameSinkWrapper> observer);
+
+const MediaStreamTrack* video_to_media(const VideoTrack* track) {
+  return track;
+}
+
+const MediaStreamTrack* audio_to_media(const AudioTrack* track) {
+  return track;
+}
+
+const VideoTrack* media_to_video(const MediaStreamTrack* track) {
+  return static_cast<const VideoTrack*>(track);
+}
+
+const AudioTrack* media_to_audio(const MediaStreamTrack* track) {
+  return static_cast<const AudioTrack*>(track); 
+}
+
 }  // namespace livekit
 
 #endif  // CLIENT_SDK_NATIVE_MEDIA_STREAM_INTERFACE_H

crates/livekit-webrtc/libwebrtc-sys/include/livekit/rust_types.h

@@ -15,6 +15,7 @@ struct SetLocalSdpObserverWrapper;
 struct SetRemoteSdpObserverWrapper;
 struct DataChannelObserverWrapper;
 struct AddIceCandidateObserverWrapper;
+struct VideoFrameSinkWrapper;
 
 // Shared types
 enum class PeerConnectionState;
@@ -24,6 +25,9 @@ enum class IceGatheringState;
 enum class SdpType;
 enum class DataState;
 enum class TrackState;
+enum class ContentHint;
+enum class VideoRotation;
+enum class VideoFrameBufferType;
 struct SdpParseError;
 struct RTCOfferAnswerOptions;
 struct RTCError;

crates/livekit-webrtc/libwebrtc-sys/include/livekit/video_frame.h

@@ -0,0 +1,48 @@
+//
+// Created by theom on 14/11/2022.
+//
+
+#ifndef LIVEKIT_WEBRTC_VIDEO_FRAME_H
+#define LIVEKIT_WEBRTC_VIDEO_FRAME_H
+
+#include "api/video/video_frame.h"
+#include "livekit/rust_types.h"
+#include "livekit/video_frame_buffer.h"
+
+namespace livekit {
+
+class VideoFrame {
+ public:
+  explicit VideoFrame(const webrtc::VideoFrame& frame)
+      : frame_(std::move(frame)) {}
+
+  int width() const { return frame_.width(); }
+  int height() const { return frame_.height(); }
+  uint32_t size() const { return frame_.size(); }
+  uint16_t id() const { return frame_.id(); }
+  int64_t timestamp_us() const { return frame_.timestamp_us(); }
+  int64_t ntp_time_ms() const { return frame_.ntp_time_ms(); }
+  uint32_t transport_frame_id() const { return frame_.transport_frame_id(); }
+  uint32_t timestamp() const { return frame_.timestamp(); }
+
+  VideoRotation rotation() const {
+    return static_cast<VideoRotation>(frame_.rotation());
+  }
+
+  // TODO(theomonnom) This shouldn't create a new shared_ptr at each call
+  std::shared_ptr<VideoFrameBuffer> video_frame_buffer() const {
+    return std::make_shared<VideoFrameBuffer>(frame_.video_frame_buffer());
+  }
+
+ private:
+  webrtc::VideoFrame frame_;
+};
+
+static std::unique_ptr<VideoFrame> _unique_video_frame() {
+  return nullptr;  // Ignore
+}
+
+
+}  // namespace livekit
+
+#endif  // LIVEKIT_WEBRTC_VIDEO_FRAME_H

crates/livekit-webrtc/libwebrtc-sys/include/livekit/video_frame_buffer.h

@@ -0,0 +1,93 @@
+//
+// Created by theom on 14/11/2022.
+//
+
+#ifndef LIVEKIT_WEBRTC_VIDEO_FRAME_BUFFER_H
+#define LIVEKIT_WEBRTC_VIDEO_FRAME_BUFFER_H
+
+#include "api/video/video_frame_buffer.h"
+#include "rust_types.h"
+
+namespace livekit {
+
+class PlanarYuvBuffer;
+class PlanarYuv8Buffer;
+class I420Buffer;
+
+class VideoFrameBuffer {
+ public:
+  explicit VideoFrameBuffer(rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer)
+      : buffer_(std::move(buffer)) {}
+
+  VideoFrameBufferType buffer_type() const {
+    return static_cast<VideoFrameBufferType>(buffer_->type());
+  }
+
+  int width() const { return buffer_->width(); }
+  int height() const { return buffer_->height(); }
+
+  std::shared_ptr<I420Buffer> to_i420() {
+    return std::make_shared<I420Buffer>(buffer_->ToI420());
+  }
+
+ protected:
+  rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer_;
+};
+
+class PlanarYuvBuffer : public VideoFrameBuffer {
+ public:
+  explicit PlanarYuvBuffer(rtc::scoped_refptr<webrtc::PlanarYuvBuffer> buffer)
+      : VideoFrameBuffer(buffer) {}
+
+  int chroma_width() const { return buffer()->ChromaWidth(); }
+  int chroma_height() const { return buffer()->ChromaHeight(); }
+
+  int stride_y() const { return buffer()->StrideY(); }
+  int stride_u() const { return buffer()->StrideU(); }
+  int stride_v() const { return buffer()->StrideV(); }
+
+ private:
+  webrtc::PlanarYuvBuffer* buffer() const {
+    return static_cast<webrtc::PlanarYuvBuffer*>(buffer_.get());
+  }
+};
+
+class PlanarYuv8Buffer : public PlanarYuvBuffer {
+ public:
+  explicit PlanarYuv8Buffer(rtc::scoped_refptr<webrtc::PlanarYuv8Buffer> buffer)
+      : PlanarYuvBuffer(buffer) {}
+
+  const uint8_t* data_y() const { return buffer()->DataY(); }
+  const uint8_t* data_u() const { return buffer()->DataU(); }
+  const uint8_t* data_v() const { return buffer()->DataV(); }
+
+ private:
+  webrtc::PlanarYuv8Buffer* buffer() const {
+    return static_cast<webrtc::PlanarYuv8Buffer*>(buffer_.get());
+  }
+};
+
+class I420Buffer : public PlanarYuv8Buffer {
+ public:
+  explicit I420Buffer(rtc::scoped_refptr<webrtc::I420BufferInterface> buffer)
+      : PlanarYuv8Buffer(buffer) {}
+};
+
+std::shared_ptr<VideoFrameBuffer> to_video_frame_buffer(
+    std::shared_ptr<PlanarYuvBuffer> buffer) {
+  return buffer;
+}
+
+std::shared_ptr<PlanarYuvBuffer> to_yuv_buffer(
+    std::shared_ptr<PlanarYuv8Buffer> buffer) {
+  return buffer;
+}
+
+std::shared_ptr<PlanarYuv8Buffer> to_yuv8_buffer(
+    std::shared_ptr<I420Buffer> buffer) {
+  return buffer;
+}
+
+}  // namespace livekit
+
+#endif  // LIVEKIT_WEBRTC_VIDEO_FRAME_BUFFER_H

crates/livekit-webrtc/libwebrtc-sys/src/candidate.cpp

@@ -7,4 +7,4 @@
 namespace livekit {
 Candidate::Candidate(const cricket::Candidate& candidate)
     : candidate_(candidate) {}
-}  // namespace livekit
+}  // namespace livekit

crates/livekit-webrtc/libwebrtc-sys/src/data_channel.cpp

@@ -87,4 +87,4 @@ std::unique_ptr<NativeDataChannelObserver> create_native_data_channel_observer(
     rust::Box<DataChannelObserverWrapper> observer) {
   return std::make_unique<NativeDataChannelObserver>(std::move(observer));
 }
-}  // namespace livekit
+}  // namespace livekit

crates/livekit-webrtc/libwebrtc-sys/src/jsep.cpp

@@ -158,4 +158,4 @@ create_native_set_remote_sdp_observer(
               std::move(observer))});
 }
 
-}  // namespace livekit
+}  // namespace livekit

crates/livekit-webrtc/libwebrtc-sys/src/lib.rs

@@ -7,4 +7,10 @@ pub mod peer_connection_factory;
 pub mod rtc_error;
 pub mod rtp_receiver;
 pub mod rtp_transceiver;
+pub mod video_frame;
+pub mod video_frame_buffer;
 pub mod webrtc;
+
+pub const MEDIA_TYPE_VIDEO: &str = "video";
+pub const MEDIA_TYPE_AUDIO: &str = "audio";
+pub const MEDIA_TYPE_DATA: &str = "data";

crates/livekit-webrtc/libwebrtc-sys/src/media_stream.cpp

@@ -4,12 +4,27 @@
 
 #include "livekit/media_stream.h"
 
+#include "libwebrtc-sys/src/media_stream.rs.h"
+
 namespace livekit {
 
 MediaStreamTrack::MediaStreamTrack(
     rtc::scoped_refptr<webrtc::MediaStreamTrackInterface> track)
     : track_(std::move(track)) {}
 
+std::unique_ptr<MediaStreamTrack> MediaStreamTrack::from(
+    rtc::scoped_refptr<webrtc::MediaStreamTrackInterface> track) {
+  if (track->kind() == webrtc::MediaStreamTrackInterface::kVideoKind) {
+    return std::make_unique<VideoTrack>(
+        rtc::scoped_refptr<webrtc::VideoTrackInterface>(
+            static_cast<webrtc::VideoTrackInterface*>(track.get())));
+  } else {
+    return std::make_unique<AudioTrack>(
+        rtc::scoped_refptr<webrtc::AudioTrackInterface>(
+            static_cast<webrtc::AudioTrackInterface*>(track.get())));
+  }
+}
+
 rust::String MediaStreamTrack::kind() const {
   return track_->kind();
 }
@@ -35,7 +50,60 @@ MediaStream::MediaStream(
     : media_stream_(std::move(stream)) {}
 
 rust::String MediaStream::id() const {
-    return media_stream_->id();
+  return media_stream_->id();
 }
 
-}  // namespace livekit
+VideoTrack::VideoTrack(rtc::scoped_refptr<webrtc::VideoTrackInterface> track)
+    : MediaStreamTrack(std::move(track)) {}
+
+void VideoTrack::add_sink(NativeVideoFrameSink& sink) {
+  track()->AddOrUpdateSink(&sink, rtc::VideoSinkWants());
+}
+
+void VideoTrack::remove_sink(NativeVideoFrameSink& sink) {
+  track()->RemoveSink(&sink);
+}
+
+void VideoTrack::set_should_receive(bool should_receive) {
+  track()->set_should_receive(should_receive);
+}
+
+bool VideoTrack::should_receive() const {
+  return track()->should_receive();
+}
+
+ContentHint VideoTrack::content_hint() const {
+  return static_cast<ContentHint>(track()->content_hint());
+}
+
+void VideoTrack::set_content_hint(ContentHint hint) {
+  track()->set_content_hint(
+      static_cast<webrtc::VideoTrackInterface::ContentHint>(hint));
+}
+
+NativeVideoFrameSink::NativeVideoFrameSink(
+    rust::Box<VideoFrameSinkWrapper> observer)
+    : observer_(std::move(observer)) {}
+
+void NativeVideoFrameSink::OnFrame(const webrtc::VideoFrame& frame) {
+  observer_->on_frame(std::make_unique<VideoFrame>(frame));
+}
+
+void NativeVideoFrameSink::OnDiscardedFrame() {
+  observer_->on_discarded_frame();
+}
+
+void NativeVideoFrameSink::OnConstraintsChanged(
+    const webrtc::VideoTrackSourceConstraints& constraints) {
+  VideoTrackSourceConstraints cst;
+  cst.min_fps = constraints.min_fps.value_or(-1);
+  cst.max_fps = constraints.max_fps.value_or(-1);
+  observer_->on_constraints_changed(cst);
+}
+
+std::unique_ptr<NativeVideoFrameSink> create_native_video_frame_sink(
+    rust::Box<VideoFrameSinkWrapper> observer) {
+  return std::make_unique<NativeVideoFrameSink>(std::move(observer));
+}
+
+}  // namespace livekit

crates/livekit-webrtc/libwebrtc-sys/src/media_stream.rs

@@ -1,3 +1,7 @@
+use cxx::UniquePtr;
+
+use crate::video_frame::ffi::VideoFrame;
+
 #[cxx::bridge(namespace = "livekit")]
 pub mod ffi {
 
@@ -8,11 +12,33 @@ pub mod ffi {
         Ended,
     }
 
+    #[derive(Debug)]
+    #[repr(i32)]
+    pub enum ContentHint {
+        None,
+        Fluid,
+        Detailed,
+        Text,
+    }
+
+    // -1 = optional
+    pub struct VideoTrackSourceConstraints {
+        pub min_fps: f64,
+        pub max_fps: f64,
+    }
+
     unsafe extern "C++" {
         include!("livekit/media_stream.h");
+        include!("livekit/video_frame.h");
 
+        type NativeVideoFrameSink;
         type MediaStreamTrack;
         type MediaStream;
+        type AudioTrack;
+        type VideoTrack;
+        type VideoFrame = crate::video_frame::ffi::VideoFrame;
+
+        fn id(self: &MediaStream) -> String;
 
         fn kind(self: &MediaStreamTrack) -> String;
         fn id(self: &MediaStreamTrack) -> String;
@@ -20,17 +46,84 @@ pub mod ffi {
         fn set_enabled(self: Pin<&mut MediaStreamTrack>, enable: bool) -> bool;
         fn state(self: &MediaStreamTrack) -> TrackState;
 
-        fn id(self: &MediaStream) -> String;
+        unsafe fn add_sink(self: Pin<&mut VideoTrack>, sink: Pin<&mut NativeVideoFrameSink>);
+        unsafe fn remove_sink(self: Pin<&mut VideoTrack>, sink: Pin<&mut NativeVideoFrameSink>);
+
+        fn set_should_receive(self: Pin<&mut VideoTrack>, should_receive: bool);
+        fn should_receive(self: &VideoTrack) -> bool;
+        fn content_hint(self: &VideoTrack) -> ContentHint;
+        fn set_content_hint(self: Pin<&mut VideoTrack>, hint: ContentHint);
+
+        fn create_native_video_frame_sink(
+            observer: Box<VideoFrameSinkWrapper>,
+        ) -> UniquePtr<NativeVideoFrameSink>;
+
+        unsafe fn video_to_media(track: *const VideoTrack) -> *const MediaStreamTrack;
+        unsafe fn audio_to_media(track: *const AudioTrack) -> *const MediaStreamTrack;
+        unsafe fn media_to_video(track: *const MediaStreamTrack) -> *const VideoTrack;
+        unsafe fn media_to_audio(track: *const MediaStreamTrack) -> *const AudioTrack;
 
         fn _unique_media_stream_track() -> UniquePtr<MediaStreamTrack>; // Ignore
         fn _unique_media_stream() -> UniquePtr<MediaStream>; // Ignore
+        fn _unique_audio_track() -> UniquePtr<AudioTrack>; // Ignore
+        fn _unique_video_track() -> UniquePtr<VideoTrack>; // Ignore
+    }
+
+    extern "Rust" {
+        type VideoFrameSinkWrapper;
+
+        fn on_frame(self: &VideoFrameSinkWrapper, frame: UniquePtr<VideoFrame>);
+        fn on_discarded_frame(self: &VideoFrameSinkWrapper);
+        fn on_constraints_changed(
+            self: &VideoFrameSinkWrapper,
+            constraints: VideoTrackSourceConstraints,
+        );
     }
 }
 
 unsafe impl Sync for ffi::MediaStreamTrack {}
-
 unsafe impl Send for ffi::MediaStreamTrack {}
-
 unsafe impl Sync for ffi::MediaStream {}
+unsafe impl Send for ffi::MediaStream {}
+unsafe impl Send for ffi::AudioTrack {}
+unsafe impl Sync for ffi::AudioTrack {}
+unsafe impl Send for ffi::VideoTrack {}
+unsafe impl Sync for ffi::VideoTrack {}
+unsafe impl Send for ffi::NativeVideoFrameSink {}
+unsafe impl Sync for ffi::NativeVideoFrameSink {}
 
-unsafe impl Send for ffi::MediaStream {}
+pub trait VideoFrameSink: Send + Sync {
+    fn on_frame(&self, frame: UniquePtr<VideoFrame>);
+    fn on_discarded_frame(&self);
+    fn on_constraints_changed(&self, constraints: ffi::VideoTrackSourceConstraints);
+}
+
+pub struct VideoFrameSinkWrapper {
+    observer: *mut dyn VideoFrameSink,
+}
+
+impl VideoFrameSinkWrapper {
+    /// # Safety
+    /// VideoFrameSink must lives as long as VideoSinkInterfaceWrapper does
+    pub unsafe fn new(observer: *mut dyn VideoFrameSink) -> Self {
+        Self { observer }
+    }
+
+    fn on_frame(&self, frame: UniquePtr<VideoFrame>) {
+        unsafe {
+            (*self.observer).on_frame(frame);
+        }
+    }
+
+    fn on_discarded_frame(&self) {
+        unsafe {
+            (*self.observer).on_discarded_frame();
+        }
+    }
+
+    fn on_constraints_changed(&self, constraints: ffi::VideoTrackSourceConstraints) {
+        unsafe {
+            (*self.observer).on_constraints_changed(constraints);
+        }
+    }
+}

crates/livekit-webrtc/libwebrtc-sys/src/peer_connection.rs

@@ -344,7 +344,7 @@ pub struct PeerConnectionObserverWrapper {
 }
 
 impl PeerConnectionObserverWrapper {
-    /// SAFETY
+    /// # Safety
     /// PeerConnectionObserver must lives as long as PeerConnectionObserverWrapper does
     pub unsafe fn new(observer: *mut dyn PeerConnectionObserver) -> Self {
         Self { observer }

crates/livekit-webrtc/libwebrtc-sys/src/peer_connection_factory.rs

@@ -45,7 +45,7 @@ pub mod ffi {
         ) -> UniquePtr<PeerConnectionFactory>;
         fn create_rtc_configuration(conf: RTCConfiguration) -> UniquePtr<NativeRTCConfiguration>;
 
-        /// SAFETY
+        /// # Safety
         /// The observer must live as long as the PeerConnection
         unsafe fn create_peer_connection(
             self: &PeerConnectionFactory,

crates/livekit-webrtc/libwebrtc-sys/src/rtp_receiver.cpp

@@ -10,7 +10,7 @@ RtpReceiver::RtpReceiver(
     : receiver_(std::move(receiver)) {}
 
 std::unique_ptr<MediaStreamTrack> RtpReceiver::track() const {
-  return std::make_unique<MediaStreamTrack>(receiver_->track());
+  return MediaStreamTrack::from(receiver_->track());
 }
 
 }  // namespace livekit

crates/livekit-webrtc/libwebrtc-sys/src/rtp_receiver.rs

@@ -15,4 +15,4 @@ pub mod ffi {
 
 unsafe impl Sync for ffi::RtpReceiver {}
 
-unsafe impl Send for ffi::RtpReceiver {}
+unsafe impl Send for ffi::RtpReceiver {}

crates/livekit-webrtc/libwebrtc-sys/src/video_frame.rs

@@ -0,0 +1,32 @@
+#[cxx::bridge(namespace = "livekit")]
+pub mod ffi {
+    #[derive(Debug)]
+    #[repr(i32)]
+    pub enum VideoRotation {
+        VideoRotation0 = 0,
+        VideoRotation90 = 90,
+        VideoRotation180 = 180,
+        VideoRotation270 = 270,
+    }
+
+    unsafe extern "C++" {
+        include!("livekit/video_frame.h");
+        include!("livekit/video_frame_buffer.h");
+
+        type VideoFrame;
+        type VideoFrameBuffer = crate::video_frame_buffer::ffi::VideoFrameBuffer;
+
+        fn width(self: &VideoFrame) -> i32;
+        fn height(self: &VideoFrame) -> i32;
+        fn size(self: &VideoFrame) -> u32;
+        fn id(self: &VideoFrame) -> u16;
+        fn timestamp_us(self: &VideoFrame) -> i64;
+        fn ntp_time_ms(self: &VideoFrame) -> i64;
+        fn transport_frame_id(self: &VideoFrame) -> u32;
+        fn timestamp(self: &VideoFrame) -> u32;
+        fn rotation(self: &VideoFrame) -> VideoRotation;
+        fn video_frame_buffer(self: &VideoFrame) -> SharedPtr<VideoFrameBuffer>;
+
+        fn _unique_video_frame() -> UniquePtr<VideoFrame>; // Ignore
+    }
+}

crates/livekit-webrtc/libwebrtc-sys/src/video_frame_buffer.rs

@@ -0,0 +1,43 @@
+#[cxx::bridge(namespace = "livekit")]
+pub mod ffi {
+    #[derive(Debug)]
+    #[repr(i32)]
+    pub enum VideoFrameBufferType {
+        Native,
+        I420,
+        I420A,
+        I422,
+        I444,
+        I010,
+        NV12,
+    }
+
+    unsafe extern "C++" {
+        include!("livekit/video_frame_buffer.h");
+
+        type VideoFrameBuffer;
+        type PlanarYuvBuffer;
+        type PlanarYuv8Buffer;
+        type I420Buffer;
+
+        fn buffer_type(self: &VideoFrameBuffer) -> VideoFrameBufferType;
+        fn width(self: &VideoFrameBuffer) -> i32;
+        fn height(self: &VideoFrameBuffer) -> i32;
+        fn to_i420(self: Pin<&mut VideoFrameBuffer>) -> SharedPtr<I420Buffer>;
+
+        fn chroma_width(self: &PlanarYuvBuffer) -> i32;
+        fn chroma_height(self: &PlanarYuvBuffer) -> i32;
+        fn stride_y(self: &PlanarYuvBuffer) -> i32;
+        fn stride_u(self: &PlanarYuvBuffer) -> i32;
+        fn stride_v(self: &PlanarYuvBuffer) -> i32;
+
+        fn data_y(self: &PlanarYuv8Buffer) -> *const u8;
+        fn data_u(self: &PlanarYuv8Buffer) -> *const u8;
+        fn data_v(self: &PlanarYuv8Buffer) -> *const u8;
+
+        fn to_video_frame_buffer(buffer: SharedPtr<PlanarYuvBuffer>)
+            -> SharedPtr<VideoFrameBuffer>;
+        fn to_yuv_buffer(buffer: SharedPtr<PlanarYuv8Buffer>) -> SharedPtr<PlanarYuvBuffer>;
+        fn to_yuv8_buffer(buffer: SharedPtr<I420Buffer>) -> SharedPtr<PlanarYuv8Buffer>;
+    }
+}

crates/livekit-webrtc/src/.media_stream.rs.rustfmt

@@ -0,0 +1,277 @@
+use cxx::UniquePtr;
+use libwebrtc_sys::media_stream as sys_ms;
+use libwebrtc_sys::MEDIA_TYPE_VIDEO;
+use std::fmt::{Debug, Formatter};
+use std::pin::Pin;
+use std::sync::{Arc, Mutex};
+
+pub use sys_ms::ffi::ContentHint;
+pub use sys_ms::ffi::TrackState;
+
+use crate::video_frame::VideoFrame;
+
+pub trait MediaStreamTrackTrait {
+    fn kind(&self) -> String;
+    fn id(&self) -> String;
+    fn enabled(&self) -> bool;
+    fn set_enabled(&self, enabled: bool) -> bool;
+    fn state(&self) -> TrackState;
+}
+
+pub enum MediaStreamTrack {
+    Audio(Arc<AudioTrack>),
+    Video(Arc<VideoTrack>),
+}
+
+macro_rules! shared_getter {
+    ($x:ident, $ret:ty) => {
+        fn $x(&self) -> $ret {
+            match self {
+                Self::Video(inner) => inner.$x(),
+                Self::Audio(inner) => inner.$x(),
+            }
+        }
+    };
+}
+
+impl MediaStreamTrack {
+    pub(crate) fn new(cxx_handle: UniquePtr<sys_ms::ffi::MediaStreamTrack>) -> Self {
+        unsafe {
+            if cxx_handle.kind() == MEDIA_TYPE_VIDEO {
+                Self::Video(Arc::new(VideoTrack::new(UniquePtr::from_raw(
+                    sys_ms::ffi::media_to_video(cxx_handle.into_raw())
+                        as *mut sys_ms::ffi::VideoTrack,
+                ))))
+            } else {
+                Self::Audio(Arc::new(AudioTrack::new(UniquePtr::from_raw(
+                    sys_ms::ffi::media_to_audio(cxx_handle.into_raw())
+                        as *mut sys_ms::ffi::AudioTrack,
+                ))))
+            }
+        }
+    }
+}
+
+impl Debug for MediaStreamTrack {
+    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
+        f.debug_struct("MediaStreamTrack")
+            .field("id", &self.id())
+            .field("kind", &self.kind())
+            .field("enabled", &self.enabled())
+            .field("state", &self.state())
+            .finish()
+    }
+}
+
+impl MediaStreamTrackTrait for MediaStreamTrack {
+    shared_getter!(kind, String);
+    shared_getter!(id, String);
+    shared_getter!(enabled, bool);
+    shared_getter!(state, TrackState);
+
+    fn set_enabled(&self, enabled: bool) -> bool {
+        match self {
+            Self::Video(inner) => inner.set_enabled(enabled),
+            Self::Audio(inner) => inner.set_enabled(enabled),
+        }
+    }
+}
+
+pub struct AudioTrack {
+    cxx_handle: Mutex<UniquePtr<sys_ms::ffi::AudioTrack>>,
+}
+
+impl AudioTrack {
+    pub(crate) fn new(cxx_handle: UniquePtr<sys_ms::ffi::AudioTrack>) -> Self {
+        Self {
+            cxx_handle: Mutex::new(cxx_handle),
+        }
+    }
+}
+
+pub struct VideoTrack {
+    cxx_handle: Mutex<UniquePtr<sys_ms::ffi::VideoTrack>>,
+    observer: Box<InternalVideoTrackSink>,
+
+    // Keep alive for c++
+    native_observer: UniquePtr<sys_ms::ffi::NativeVideoFrameSink>,
+}
+
+macro_rules! impl_media_stream_track_trait {
+    ($x:ty, $cast:ident) => {
+        impl MediaStreamTrackTrait for $x {
+            fn kind(&self) -> String {
+                unsafe { (*sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())).kind() }
+            }
+
+            fn id(&self) -> String {
+                unsafe { (*sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())).id() }
+            }
+
+            fn enabled(&self) -> bool {
+                unsafe { (*sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())).enabled() }
+            }
+
+            fn set_enabled(&self, enabled: bool) -> bool {
+                unsafe {
+                    let media =
+                        sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap()) as *mut sys_ms::ffi::MediaStreamTrack;
+
+                    Pin::new_unchecked(&mut *media).set_enabled(enabled)
+                }
+            }
+
+            fn state(&self) -> TrackState {
+                unsafe { (*sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())).state() }
+            }
+        }
+    };
+}
+
+impl_media_stream_track_trait!(VideoTrack, video_to_media);
+impl_media_stream_track_trait!(AudioTrack, audio_to_media);
+
+pub type OnFrameHandler = Box<dyn FnMut(VideoFrame) + Send + Sync>;
+pub type OnDiscardedFrameHandler = Box<dyn FnMut() + Send + Sync>;
+pub type OnConstraintsChanged = Box<dyn FnMut(VideoTrackSourceConstraints) + Send + Sync>;
+
+#[derive(Default)]
+struct InternalVideoTrackSink {
+    on_frame_handler: Mutex<Option<OnFrameHandler>>,
+    on_discarded_frame_handler: Mutex<Option<OnDiscardedFrameHandler>>,
+    on_constraints_changed_handler: Mutex<Option<OnConstraintsChanged>>,
+}
+
+pub struct VideoTrackSourceConstraints {
+    pub min_fps: Option<f64>,
+    pub max_fps: Option<f64>,
+}
+
+impl From<sys_ms::ffi::VideoTrackSourceConstraints> for VideoTrackSourceConstraints {
+    fn from(cst: sys_ms::ffi::VideoTrackSourceConstraints) -> Self {
+        Self {
+            min_fps: (cst.min_fps != 1.0).then_some(cst.min_fps),
+            max_fps: (cst.max_fps != 1.0).then_some(cst.max_fps),
+        }
+    }
+}
+
+impl sys_ms::VideoFrameSink for InternalVideoTrackSink {
+    fn on_frame(&self, frame: UniquePtr<libwebrtc_sys::video_frame::ffi::VideoFrame>) {
+        if let Some(cb) = self.on_frame_handler.lock().unwrap().as_mut() {
+            cb(VideoFrame::new(frame));
+        }
+    }
+
+    fn on_discarded_frame(&self) {
+        if let Some(cb) = self.on_discarded_frame_handler.lock().unwrap().as_mut() {
+            cb();
+        }
+    }
+
+    fn on_constraints_changed(&self, constraints: sys_ms::ffi::VideoTrackSourceConstraints) {
+        if let Some(cb) = self.on_constraints_changed_handler.lock().unwrap().as_mut() {
+            cb(constraints.into());
+        }
+    }
+}
+
+impl VideoTrack {
+    fn new(cxx_handle: UniquePtr<sys_ms::ffi::VideoTrack>) -> Self {
+        let mut observer = Box::new(InternalVideoTrackSink::default());
+
+        let mut track = unsafe {
+            Self {
+                cxx_handle: Mutex::new(cxx_handle),
+                native_observer: sys_ms::ffi::create_native_video_frame_sink(Box::new(
+                    sys_ms::VideoFrameSinkWrapper::new(&mut *observer),
+                )),
+                observer,
+            }
+        };
+
+        unsafe {
+            track
+                .cxx_handle
+                .lock()
+                .unwrap()
+                .pin_mut()
+                .add_sink(track.native_observer.pin_mut());
+        }
+
+        track
+    }
+
+    pub fn set_should_receive(&mut self, should_receive: bool) {
+
+
+        
+        self.cxx_handle
+            .lock()
+            .unwrap()
+            .pin_mut()
+            .set_should_receive(should_receive)
+    }
+
+    pub fn set_content_hint(&mut self, hint: ContentHint) {
+        self.cxx_handle
+            .lock()
+            .unwrap()
+            .pin_mut()
+            .set_content_hint(hint)
+    }
+
+    pub fn should_receive(&self) -> bool {
+        self.cxx_handle.lock().unwrap().should_receive()
+    }
+
+    pub fn content_hint(&self) -> ContentHint {
+        self.cxx_handle.lock().unwrap().content_hint()
+    }
+
+    pub fn on_frame(&mut self, handler: OnFrameHandler) {
+        *self.observer.on_frame_handler.lock().unwrap() = Some(handler);
+    }
+
+    pub fn on_discarded_frame(&mut self, handler: OnDiscardedFrameHandler) {
+        *self.observer.on_discarded_frame_handler.lock().unwrap() = Some(handler);
+    }
+
+    pub fn on_constraints_changed(&mut self, handler: OnConstraintsChanged) {
+        *self.observer.on_constraints_changed_handler.lock().unwrap() = Some(handler);
+    }
+}
+
+impl Drop for VideoTrack {
+    fn drop(&mut self) {
+        unsafe {
+            self.cxx_handle
+                .lock()
+                .unwrap()
+                .pin_mut()
+                .remove_sink(self.native_observer.pin_mut());
+        }
+    }
+}
+
+pub struct MediaStream {
+    cxx_handle: UniquePtr<sys_ms::ffi::MediaStream>,
+}
+
+impl Debug for MediaStream {
+    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
+        f.debug_struct("MediaStream")
+            .field("id", &self.id())
+            .finish()
+    }
+}
+
+impl MediaStream {
+    pub(crate) fn new(cxx_handle: UniquePtr<sys_ms::ffi::MediaStream>) -> Self {
+        Self { cxx_handle }
+    }
+
+    pub fn id(&self) -> String {
+        self.cxx_handle.id()
+    }
+}

crates/livekit-webrtc/src/data_channel.rs

@@ -109,13 +109,13 @@ impl Drop for DataChannel {
 
 pub type OnStateChangeHandler = Box<dyn FnMut() + Send + Sync>;
 pub type OnMessageHandler = Box<dyn FnMut(&[u8], bool) + Send + Sync>;
-// data, is_binary
 pub type OnBufferedAmountChangeHandler = Box<dyn FnMut(u64) + Send + Sync>;
 
+#[derive(Default)]
 struct InternalDataChannelObserver {
-    on_state_change_handler: Arc<Mutex<Option<OnStateChangeHandler>>>,
-    on_message_handler: Arc<Mutex<Option<OnMessageHandler>>>,
-    on_buffered_amount_change_handler: Arc<Mutex<Option<OnBufferedAmountChangeHandler>>>,
+    on_state_change_handler: Mutex<Option<OnStateChangeHandler>>,
+    on_message_handler: Mutex<Option<OnMessageHandler>>,
+    on_buffered_amount_change_handler: Mutex<Option<OnBufferedAmountChangeHandler>>,
 }
 
 impl sys_dc::DataChannelObserver for InternalDataChannelObserver {
@@ -144,16 +144,6 @@ impl sys_dc::DataChannelObserver for InternalDataChannelObserver {
     }
 }
 
-impl Default for InternalDataChannelObserver {
-    fn default() -> Self {
-        Self {
-            on_state_change_handler: Arc::new(Default::default()),
-            on_message_handler: Arc::new(Default::default()),
-            on_buffered_amount_change_handler: Arc::new(Default::default()),
-        }
-    }
-}
-
 #[derive(Debug)]
 pub struct DataChannelInit {
     #[deprecated]

crates/livekit-webrtc/src/jsep.rs

@@ -1,4 +1,4 @@
-use std::fmt::{Debug, Display, Formatter};
+use std::fmt::{Debug, Formatter};
 
 use cxx::UniquePtr;
 

crates/livekit-webrtc/src/lib.rs

@@ -6,4 +6,6 @@ pub mod peer_connection_factory;
 pub mod rtc_error;
 pub mod rtp_receiver;
 pub mod rtp_transceiver;
+pub mod video_frame;
+pub mod video_frame_buffer;
 pub mod webrtc;

crates/livekit-webrtc/src/media_stream.rs

@@ -1,14 +1,59 @@
 use cxx::UniquePtr;
-use std::fmt::{Debug, Formatter};
-
 use libwebrtc_sys::media_stream as sys_ms;
+use libwebrtc_sys::MEDIA_TYPE_VIDEO;
+use std::fmt::{Debug, Formatter};
+use std::pin::Pin;
+use std::sync::{Arc, Mutex};
+
+pub use sys_ms::ffi::ContentHint;
 pub use sys_ms::ffi::TrackState;
 
-pub struct MediaStreamTrack {
-    cxx_handle: UniquePtr<sys_ms::ffi::MediaStreamTrack>,
+use crate::video_frame::VideoFrame;
+
+pub trait MediaStreamTrackTrait {
+    fn kind(&self) -> String;
+    fn id(&self) -> String;
+    fn enabled(&self) -> bool;
+    fn set_enabled(&self, enabled: bool) -> bool;
+    fn state(&self) -> TrackState;
 }
 
-impl Debug for MediaStreamTrack {
+#[derive(Clone)]
+pub enum MediaStreamTrackHandle {
+    Audio(Arc<AudioTrack>),
+    Video(Arc<VideoTrack>),
+}
+
+macro_rules! shared_getter {
+    ($x:ident, $ret:ty) => {
+        fn $x(&self) -> $ret {
+            match self {
+                Self::Video(inner) => inner.$x(),
+                Self::Audio(inner) => inner.$x(),
+            }
+        }
+    };
+}
+
+impl MediaStreamTrackHandle {
+    pub(crate) fn new(cxx_handle: UniquePtr<sys_ms::ffi::MediaStreamTrack>) -> Self {
+        unsafe {
+            if cxx_handle.kind() == MEDIA_TYPE_VIDEO {
+                Self::Video(VideoTrack::new(UniquePtr::from_raw(
+                    sys_ms::ffi::media_to_video(cxx_handle.into_raw())
+                        as *mut sys_ms::ffi::VideoTrack,
+                )))
+            } else {
+                Self::Audio(AudioTrack::new(UniquePtr::from_raw(
+                    sys_ms::ffi::media_to_audio(cxx_handle.into_raw())
+                        as *mut sys_ms::ffi::AudioTrack,
+                )))
+            }
+        }
+    }
+}
+
+impl Debug for MediaStreamTrackHandle {
     fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
         f.debug_struct("MediaStreamTrack")
             .field("id", &self.id())
@@ -19,29 +64,191 @@ impl Debug for MediaStreamTrack {
     }
 }
 
-impl MediaStreamTrack {
-    pub(crate) fn new(cxx_handle: UniquePtr<sys_ms::ffi::MediaStreamTrack>) -> Self {
-        Self { cxx_handle }
+impl MediaStreamTrackTrait for MediaStreamTrackHandle {
+    shared_getter!(kind, String);
+    shared_getter!(id, String);
+    shared_getter!(enabled, bool);
+    shared_getter!(state, TrackState);
+
+    fn set_enabled(&self, enabled: bool) -> bool {
+        match self {
+            Self::Video(inner) => inner.set_enabled(enabled),
+            Self::Audio(inner) => inner.set_enabled(enabled),
+        }
+    }
+}
+
+pub struct AudioTrack {
+    cxx_handle: Mutex<UniquePtr<sys_ms::ffi::AudioTrack>>,
+}
+
+impl AudioTrack {
+    pub(crate) fn new(cxx_handle: UniquePtr<sys_ms::ffi::AudioTrack>) -> Arc<Self> {
+        Arc::new(Self {
+            cxx_handle: Mutex::new(cxx_handle),
+        })
+    }
+}
+
+pub struct VideoTrack {
+    cxx_handle: Mutex<UniquePtr<sys_ms::ffi::VideoTrack>>,
+    observer: Box<InternalVideoTrackSink>,
+
+    // Keep alive for c++
+    native_observer: UniquePtr<sys_ms::ffi::NativeVideoFrameSink>,
+}
+
+macro_rules! impl_media_stream_track_trait {
+    ($x:ty, $cast:ident) => {
+        impl MediaStreamTrackTrait for $x {
+            fn kind(&self) -> String {
+                unsafe { (*sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())).kind() }
+            }
+
+            fn id(&self) -> String {
+                unsafe { (*sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())).id() }
+            }
+
+            fn enabled(&self) -> bool {
+                unsafe { (*sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())).enabled() }
+            }
+
+            fn set_enabled(&self, enabled: bool) -> bool {
+                unsafe {
+                    let media =
+                        sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap()) as *mut sys_ms::ffi::MediaStreamTrack;
+
+                    Pin::new_unchecked(&mut *media).set_enabled(enabled)
+                }
+            }
+
+            fn state(&self) -> TrackState {
+                unsafe { (*sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())).state() }
+            }
+        }
+    };
+}
+
+impl_media_stream_track_trait!(VideoTrack, video_to_media);
+impl_media_stream_track_trait!(AudioTrack, audio_to_media);
+
+pub type OnFrameHandler = Box<dyn FnMut(VideoFrame) + Send + Sync>;
+pub type OnDiscardedFrameHandler = Box<dyn FnMut() + Send + Sync>;
+pub type OnConstraintsChanged = Box<dyn FnMut(VideoTrackSourceConstraints) + Send + Sync>;
+
+#[derive(Default)]
+struct InternalVideoTrackSink {
+    on_frame_handler: Mutex<Option<OnFrameHandler>>,
+    on_discarded_frame_handler: Mutex<Option<OnDiscardedFrameHandler>>,
+    on_constraints_changed_handler: Mutex<Option<OnConstraintsChanged>>,
+}
+
+pub struct VideoTrackSourceConstraints {
+    pub min_fps: Option<f64>,
+    pub max_fps: Option<f64>,
+}
+
+impl From<sys_ms::ffi::VideoTrackSourceConstraints> for VideoTrackSourceConstraints {
+    fn from(cst: sys_ms::ffi::VideoTrackSourceConstraints) -> Self {
+        Self {
+            min_fps: (cst.min_fps != 1.0).then_some(cst.min_fps),
+            max_fps: (cst.max_fps != 1.0).then_some(cst.max_fps),
+        }
+    }
+}
+
+impl sys_ms::VideoFrameSink for InternalVideoTrackSink {
+    fn on_frame(&self, frame: UniquePtr<libwebrtc_sys::video_frame::ffi::VideoFrame>) {
+        if let Some(cb) = self.on_frame_handler.lock().unwrap().as_mut() {
+            cb(VideoFrame::new(frame));
+        }
     }
 
-    fn kind(&self) -> String {
-        self.cxx_handle.kind()
+    fn on_discarded_frame(&self) {
+        if let Some(cb) = self.on_discarded_frame_handler.lock().unwrap().as_mut() {
+            cb();
+        }
     }
 
-    fn id(&self) -> String {
-        self.cxx_handle.id()
+    fn on_constraints_changed(&self, constraints: sys_ms::ffi::VideoTrackSourceConstraints) {
+        if let Some(cb) = self.on_constraints_changed_handler.lock().unwrap().as_mut() {
+            cb(constraints.into());
+        }
+    }
+}
+
+impl VideoTrack {
+    fn new(cxx_handle: UniquePtr<sys_ms::ffi::VideoTrack>) -> Arc<Self> {
+        let mut observer = Box::new(InternalVideoTrackSink::default());
+
+        let mut track = unsafe {
+            Self {
+                cxx_handle: Mutex::new(cxx_handle),
+                native_observer: sys_ms::ffi::create_native_video_frame_sink(Box::new(
+                    sys_ms::VideoFrameSinkWrapper::new(&mut *observer),
+                )),
+                observer,
+            }
+        };
+
+        unsafe {
+            track
+                .cxx_handle
+                .lock()
+                .unwrap()
+                .pin_mut()
+                .add_sink(track.native_observer.pin_mut());
+        }
+
+        Arc::new(track)
     }
 
-    fn enabled(&self) -> bool {
-        self.cxx_handle.enabled()
+    pub fn set_should_receive(&mut self, should_receive: bool) {
+        self.cxx_handle
+            .lock()
+            .unwrap()
+            .pin_mut()
+            .set_should_receive(should_receive)
     }
 
-    fn set_enabled(&mut self, enable: bool) -> bool {
-        self.cxx_handle.pin_mut().set_enabled(enable)
+    pub fn set_content_hint(&mut self, hint: ContentHint) {
+        self.cxx_handle
+            .lock()
+            .unwrap()
+            .pin_mut()
+            .set_content_hint(hint)
     }
 
-    fn state(&self) -> TrackState {
-        self.cxx_handle.state()
+    pub fn should_receive(&self) -> bool {
+        self.cxx_handle.lock().unwrap().should_receive()
+    }
+
+    pub fn content_hint(&self) -> ContentHint {
+        self.cxx_handle.lock().unwrap().content_hint()
+    }
+
+    pub fn on_frame(&mut self, handler: OnFrameHandler) {
+        *self.observer.on_frame_handler.lock().unwrap() = Some(handler);
+    }
+
+    pub fn on_discarded_frame(&mut self, handler: OnDiscardedFrameHandler) {
+        *self.observer.on_discarded_frame_handler.lock().unwrap() = Some(handler);
+    }
+
+    pub fn on_constraints_changed(&mut self, handler: OnConstraintsChanged) {
+        *self.observer.on_constraints_changed_handler.lock().unwrap() = Some(handler);
+    }
+}
+
+impl Drop for VideoTrack {
+    fn drop(&mut self) {
+        unsafe {
+            self.cxx_handle
+                .lock()
+                .unwrap()
+                .pin_mut()
+                .remove_sink(self.native_observer.pin_mut());
+        }
     }
 }
 

crates/livekit-webrtc/src/peer_connection.rs

@@ -18,7 +18,7 @@ pub use libwebrtc_sys::peer_connection::ffi::SignalingState;
 
 use crate::data_channel::{DataChannel, DataChannelInit};
 use crate::jsep::{IceCandidate, SessionDescription};
-use crate::media_stream::MediaStream;
+use crate::media_stream::{MediaStream, VideoTrack, AudioTrack};
 use crate::rtc_error::RTCError;
 use crate::rtp_receiver::RtpReceiver;
 use crate::rtp_transceiver::RtpTransceiver;

crates/livekit-webrtc/src/rtp_receiver.rs

@@ -1,4 +1,4 @@
-use crate::media_stream::MediaStreamTrack;
+use crate::media_stream::{MediaStreamTrackHandle, MediaStreamTrackTrait};
 use cxx::UniquePtr;
 use libwebrtc_sys::rtp_receiver as sys_rec;
 use std::fmt::{Debug, Formatter};
@@ -17,10 +17,12 @@ impl Debug for RtpReceiver {
 
 impl RtpReceiver {
     pub(crate) fn new(cxx_handle: UniquePtr<sys_rec::ffi::RtpReceiver>) -> Self {
-        Self { cxx_handle }
+        Self {
+            cxx_handle,
+        }
     }
 
-    pub fn track(&self) -> MediaStreamTrack {
-        MediaStreamTrack::new(self.cxx_handle.track())
+    pub fn track(&self) -> MediaStreamTrackHandle {
+        MediaStreamTrackHandle::new(self.cxx_handle.track())
     }
 }

crates/livekit-webrtc/src/video_frame.rs

@@ -0,0 +1,50 @@
+use cxx::UniquePtr;
+use libwebrtc_sys::video_frame as vf_sys;
+
+pub use vf_sys::ffi::VideoRotation;
+
+pub struct VideoFrame {
+    cxx_handle: UniquePtr<vf_sys::ffi::VideoFrame>,
+}
+
+impl VideoFrame {
+    pub(crate) fn new(cxx_handle: UniquePtr<vf_sys::ffi::VideoFrame>) -> Self {
+        Self { cxx_handle }
+    }
+
+    pub fn width(&self) -> i32 {
+        self.cxx_handle.width()
+    }
+
+    pub fn height(&self) -> i32 {
+        self.cxx_handle.height()
+    }
+
+    pub fn size(&self) -> u32 {
+        self.cxx_handle.size()
+    }
+
+    pub fn id(&self) -> u16 {
+        self.cxx_handle.id()
+    }
+
+    pub fn timestamp_us(&self) -> i64 {
+        self.cxx_handle.timestamp_us()
+    }
+    
+    pub fn ntp_time_ms(&self) -> i64 {
+        self.cxx_handle.ntp_time_ms()
+    }
+
+    pub fn transport_frame_id(&self) -> u32 {
+        self.cxx_handle.transport_frame_id()
+    }
+
+    pub fn timestamp(&self) -> u32 {
+        self.cxx_handle.timestamp()
+    }
+
+    pub fn rotation(&self) -> VideoRotation {
+        self.cxx_handle.rotation()
+    }
+}

crates/livekit-webrtc/src/video_frame_buffer.rs

@@ -0,0 +1,18 @@
+use cxx::UniquePtr;
+use libwebrtc_sys::video_frame_buffer as vfb_sys;
+
+pub use vfb_sys::ffi::VideoFrameBufferType;
+
+pub struct VideoFrameBuffer {
+    cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>,
+}
+
+impl VideoFrameBuffer {
+    pub fn new(cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>) -> Self {
+        Self { cxx_handle }
+    }
+
+    pub fn buffer_type(&self) -> VideoFrameBufferType {
+        self.cxx_handle.buffer_type()
+    }
+}