Initial downstream tracks

crates/livekit-webrtc/libwebrtc-sys/src/candidate.cpp

@@ -7,4 +7,4 @@
 namespace livekit {
 Candidate::Candidate(const cricket::Candidate& candidate)
     : candidate_(candidate) {}
-}  // namespace livekit
+}  // namespace livekit

crates/livekit-webrtc/libwebrtc-sys/src/data_channel.cpp

@@ -87,4 +87,4 @@ std::unique_ptr<NativeDataChannelObserver> create_native_data_channel_observer(
     rust::Box<DataChannelObserverWrapper> observer) {
   return std::make_unique<NativeDataChannelObserver>(std::move(observer));
 }
-}  // namespace livekit
+}  // namespace livekit

crates/livekit-webrtc/libwebrtc-sys/src/jsep.cpp

@@ -158,4 +158,4 @@ create_native_set_remote_sdp_observer(
               std::move(observer))});
 }
 
-}  // namespace livekit
+}  // namespace livekit

crates/livekit-webrtc/libwebrtc-sys/src/lib.rs

@@ -7,4 +7,10 @@ pub mod peer_connection_factory;
 pub mod rtc_error;
 pub mod rtp_receiver;
 pub mod rtp_transceiver;
+pub mod video_frame;
+pub mod video_frame_buffer;
 pub mod webrtc;
+
+pub const MEDIA_TYPE_VIDEO: &str = "video";
+pub const MEDIA_TYPE_AUDIO: &str = "audio";
+pub const MEDIA_TYPE_DATA: &str = "data";

crates/livekit-webrtc/libwebrtc-sys/src/media_stream.cpp

@@ -4,12 +4,27 @@
 
 #include "livekit/media_stream.h"
 
+#include "libwebrtc-sys/src/media_stream.rs.h"
+
 namespace livekit {
 
 MediaStreamTrack::MediaStreamTrack(
     rtc::scoped_refptr<webrtc::MediaStreamTrackInterface> track)
     : track_(std::move(track)) {}
 
+std::unique_ptr<MediaStreamTrack> MediaStreamTrack::from(
+    rtc::scoped_refptr<webrtc::MediaStreamTrackInterface> track) {
+  if (track->kind() == webrtc::MediaStreamTrackInterface::kVideoKind) {
+    return std::make_unique<VideoTrack>(
+        rtc::scoped_refptr<webrtc::VideoTrackInterface>(
+            static_cast<webrtc::VideoTrackInterface*>(track.get())));
+  } else {
+    return std::make_unique<AudioTrack>(
+        rtc::scoped_refptr<webrtc::AudioTrackInterface>(
+            static_cast<webrtc::AudioTrackInterface*>(track.get())));
+  }
+}
+
 rust::String MediaStreamTrack::kind() const {
   return track_->kind();
 }
@@ -35,7 +50,60 @@ MediaStream::MediaStream(
     : media_stream_(std::move(stream)) {}
 
 rust::String MediaStream::id() const {
-    return media_stream_->id();
+  return media_stream_->id();
 }
 
-}  // namespace livekit
+VideoTrack::VideoTrack(rtc::scoped_refptr<webrtc::VideoTrackInterface> track)
+    : MediaStreamTrack(std::move(track)) {}
+
+void VideoTrack::add_sink(NativeVideoFrameSink& sink) {
+  track()->AddOrUpdateSink(&sink, rtc::VideoSinkWants());
+}
+
+void VideoTrack::remove_sink(NativeVideoFrameSink& sink) {
+  track()->RemoveSink(&sink);
+}
+
+void VideoTrack::set_should_receive(bool should_receive) {
+  track()->set_should_receive(should_receive);
+}
+
+bool VideoTrack::should_receive() const {
+  return track()->should_receive();
+}
+
+ContentHint VideoTrack::content_hint() const {
+  return static_cast<ContentHint>(track()->content_hint());
+}
+
+void VideoTrack::set_content_hint(ContentHint hint) {
+  track()->set_content_hint(
+      static_cast<webrtc::VideoTrackInterface::ContentHint>(hint));
+}
+
+NativeVideoFrameSink::NativeVideoFrameSink(
+    rust::Box<VideoFrameSinkWrapper> observer)
+    : observer_(std::move(observer)) {}
+
+void NativeVideoFrameSink::OnFrame(const webrtc::VideoFrame& frame) {
+  observer_->on_frame(std::make_unique<VideoFrame>(frame));
+}
+
+void NativeVideoFrameSink::OnDiscardedFrame() {
+  observer_->on_discarded_frame();
+}
+
+void NativeVideoFrameSink::OnConstraintsChanged(
+    const webrtc::VideoTrackSourceConstraints& constraints) {
+  VideoTrackSourceConstraints cst;
+  cst.min_fps = constraints.min_fps.value_or(-1);
+  cst.max_fps = constraints.max_fps.value_or(-1);
+  observer_->on_constraints_changed(cst);
+}
+
+std::unique_ptr<NativeVideoFrameSink> create_native_video_frame_sink(
+    rust::Box<VideoFrameSinkWrapper> observer) {
+  return std::make_unique<NativeVideoFrameSink>(std::move(observer));
+}
+
+}  // namespace livekit

crates/livekit-webrtc/libwebrtc-sys/src/media_stream.rs

@@ -1,3 +1,7 @@
+use cxx::UniquePtr;
+
+use crate::video_frame::ffi::VideoFrame;
+
 #[cxx::bridge(namespace = "livekit")]
 pub mod ffi {
 
@@ -8,11 +12,33 @@ pub mod ffi {
         Ended,
     }
 
+    #[derive(Debug)]
+    #[repr(i32)]
+    pub enum ContentHint {
+        None,
+        Fluid,
+        Detailed,
+        Text,
+    }
+
+    // -1 = optional
+    pub struct VideoTrackSourceConstraints {
+        pub min_fps: f64,
+        pub max_fps: f64,
+    }
+
     unsafe extern "C++" {
         include!("livekit/media_stream.h");
+        include!("livekit/video_frame.h");
 
+        type NativeVideoFrameSink;
         type MediaStreamTrack;
         type MediaStream;
+        type AudioTrack;
+        type VideoTrack;
+        type VideoFrame = crate::video_frame::ffi::VideoFrame;
+
+        fn id(self: &MediaStream) -> String;
 
         fn kind(self: &MediaStreamTrack) -> String;
         fn id(self: &MediaStreamTrack) -> String;
@@ -20,17 +46,84 @@ pub mod ffi {
         fn set_enabled(self: Pin<&mut MediaStreamTrack>, enable: bool) -> bool;
         fn state(self: &MediaStreamTrack) -> TrackState;
 
-        fn id(self: &MediaStream) -> String;
+        unsafe fn add_sink(self: Pin<&mut VideoTrack>, sink: Pin<&mut NativeVideoFrameSink>);
+        unsafe fn remove_sink(self: Pin<&mut VideoTrack>, sink: Pin<&mut NativeVideoFrameSink>);
+
+        fn set_should_receive(self: Pin<&mut VideoTrack>, should_receive: bool);
+        fn should_receive(self: &VideoTrack) -> bool;
+        fn content_hint(self: &VideoTrack) -> ContentHint;
+        fn set_content_hint(self: Pin<&mut VideoTrack>, hint: ContentHint);
+
+        fn create_native_video_frame_sink(
+            observer: Box<VideoFrameSinkWrapper>,
+        ) -> UniquePtr<NativeVideoFrameSink>;
+
+        unsafe fn video_to_media(track: *const VideoTrack) -> *const MediaStreamTrack;
+        unsafe fn audio_to_media(track: *const AudioTrack) -> *const MediaStreamTrack;
+        unsafe fn media_to_video(track: *const MediaStreamTrack) -> *const VideoTrack;
+        unsafe fn media_to_audio(track: *const MediaStreamTrack) -> *const AudioTrack;
 
         fn _unique_media_stream_track() -> UniquePtr<MediaStreamTrack>; // Ignore
         fn _unique_media_stream() -> UniquePtr<MediaStream>; // Ignore
+        fn _unique_audio_track() -> UniquePtr<AudioTrack>; // Ignore
+        fn _unique_video_track() -> UniquePtr<VideoTrack>; // Ignore
+    }
+
+    extern "Rust" {
+        type VideoFrameSinkWrapper;
+
+        fn on_frame(self: &VideoFrameSinkWrapper, frame: UniquePtr<VideoFrame>);
+        fn on_discarded_frame(self: &VideoFrameSinkWrapper);
+        fn on_constraints_changed(
+            self: &VideoFrameSinkWrapper,
+            constraints: VideoTrackSourceConstraints,
+        );
     }
 }
 
 unsafe impl Sync for ffi::MediaStreamTrack {}
-
 unsafe impl Send for ffi::MediaStreamTrack {}
-
 unsafe impl Sync for ffi::MediaStream {}
+unsafe impl Send for ffi::MediaStream {}
+unsafe impl Send for ffi::AudioTrack {}
+unsafe impl Sync for ffi::AudioTrack {}
+unsafe impl Send for ffi::VideoTrack {}
+unsafe impl Sync for ffi::VideoTrack {}
+unsafe impl Send for ffi::NativeVideoFrameSink {}
+unsafe impl Sync for ffi::NativeVideoFrameSink {}
 
-unsafe impl Send for ffi::MediaStream {}
+pub trait VideoFrameSink: Send + Sync {
+    fn on_frame(&self, frame: UniquePtr<VideoFrame>);
+    fn on_discarded_frame(&self);
+    fn on_constraints_changed(&self, constraints: ffi::VideoTrackSourceConstraints);
+}
+
+pub struct VideoFrameSinkWrapper {
+    observer: *mut dyn VideoFrameSink,
+}
+
+impl VideoFrameSinkWrapper {
+    /// # Safety
+    /// VideoFrameSink must lives as long as VideoSinkInterfaceWrapper does
+    pub unsafe fn new(observer: *mut dyn VideoFrameSink) -> Self {
+        Self { observer }
+    }
+
+    fn on_frame(&self, frame: UniquePtr<VideoFrame>) {
+        unsafe {
+            (*self.observer).on_frame(frame);
+        }
+    }
+
+    fn on_discarded_frame(&self) {
+        unsafe {
+            (*self.observer).on_discarded_frame();
+        }
+    }
+
+    fn on_constraints_changed(&self, constraints: ffi::VideoTrackSourceConstraints) {
+        unsafe {
+            (*self.observer).on_constraints_changed(constraints);
+        }
+    }
+}

crates/livekit-webrtc/libwebrtc-sys/src/peer_connection.rs

@@ -344,7 +344,7 @@ pub struct PeerConnectionObserverWrapper {
 }
 
 impl PeerConnectionObserverWrapper {
-    /// SAFETY
+    /// # Safety
     /// PeerConnectionObserver must lives as long as PeerConnectionObserverWrapper does
     pub unsafe fn new(observer: *mut dyn PeerConnectionObserver) -> Self {
         Self { observer }

crates/livekit-webrtc/libwebrtc-sys/src/peer_connection_factory.rs

@@ -45,7 +45,7 @@ pub mod ffi {
         ) -> UniquePtr<PeerConnectionFactory>;
         fn create_rtc_configuration(conf: RTCConfiguration) -> UniquePtr<NativeRTCConfiguration>;
 
-        /// SAFETY
+        /// # Safety
         /// The observer must live as long as the PeerConnection
         unsafe fn create_peer_connection(
             self: &PeerConnectionFactory,

crates/livekit-webrtc/libwebrtc-sys/src/rtp_receiver.cpp

@@ -10,7 +10,7 @@ RtpReceiver::RtpReceiver(
     : receiver_(std::move(receiver)) {}
 
 std::unique_ptr<MediaStreamTrack> RtpReceiver::track() const {
-  return std::make_unique<MediaStreamTrack>(receiver_->track());
+  return MediaStreamTrack::from(receiver_->track());
 }
 
 }  // namespace livekit

crates/livekit-webrtc/libwebrtc-sys/src/rtp_receiver.rs

@@ -15,4 +15,4 @@ pub mod ffi {
 
 unsafe impl Sync for ffi::RtpReceiver {}
 
-unsafe impl Send for ffi::RtpReceiver {}
+unsafe impl Send for ffi::RtpReceiver {}

crates/livekit-webrtc/libwebrtc-sys/src/video_frame.rs

@@ -0,0 +1,32 @@
+#[cxx::bridge(namespace = "livekit")]
+pub mod ffi {
+    #[derive(Debug)]
+    #[repr(i32)]
+    pub enum VideoRotation {
+        VideoRotation0 = 0,
+        VideoRotation90 = 90,
+        VideoRotation180 = 180,
+        VideoRotation270 = 270,
+    }
+
+    unsafe extern "C++" {
+        include!("livekit/video_frame.h");
+        include!("livekit/video_frame_buffer.h");
+
+        type VideoFrame;
+        type VideoFrameBuffer = crate::video_frame_buffer::ffi::VideoFrameBuffer;
+
+        fn width(self: &VideoFrame) -> i32;
+        fn height(self: &VideoFrame) -> i32;
+        fn size(self: &VideoFrame) -> u32;
+        fn id(self: &VideoFrame) -> u16;
+        fn timestamp_us(self: &VideoFrame) -> i64;
+        fn ntp_time_ms(self: &VideoFrame) -> i64;
+        fn transport_frame_id(self: &VideoFrame) -> u32;
+        fn timestamp(self: &VideoFrame) -> u32;
+        fn rotation(self: &VideoFrame) -> VideoRotation;
+        fn video_frame_buffer(self: &VideoFrame) -> SharedPtr<VideoFrameBuffer>;
+
+        fn _unique_video_frame() -> UniquePtr<VideoFrame>; // Ignore
+    }
+}

crates/livekit-webrtc/libwebrtc-sys/src/video_frame_buffer.rs

@@ -0,0 +1,43 @@
+#[cxx::bridge(namespace = "livekit")]
+pub mod ffi {
+    #[derive(Debug)]
+    #[repr(i32)]
+    pub enum VideoFrameBufferType {
+        Native,
+        I420,
+        I420A,
+        I422,
+        I444,
+        I010,
+        NV12,
+    }
+
+    unsafe extern "C++" {
+        include!("livekit/video_frame_buffer.h");
+
+        type VideoFrameBuffer;
+        type PlanarYuvBuffer;
+        type PlanarYuv8Buffer;
+        type I420Buffer;
+
+        fn buffer_type(self: &VideoFrameBuffer) -> VideoFrameBufferType;
+        fn width(self: &VideoFrameBuffer) -> i32;
+        fn height(self: &VideoFrameBuffer) -> i32;
+        fn to_i420(self: Pin<&mut VideoFrameBuffer>) -> SharedPtr<I420Buffer>;
+
+        fn chroma_width(self: &PlanarYuvBuffer) -> i32;
+        fn chroma_height(self: &PlanarYuvBuffer) -> i32;
+        fn stride_y(self: &PlanarYuvBuffer) -> i32;
+        fn stride_u(self: &PlanarYuvBuffer) -> i32;
+        fn stride_v(self: &PlanarYuvBuffer) -> i32;
+
+        fn data_y(self: &PlanarYuv8Buffer) -> *const u8;
+        fn data_u(self: &PlanarYuv8Buffer) -> *const u8;
+        fn data_v(self: &PlanarYuv8Buffer) -> *const u8;
+
+        fn to_video_frame_buffer(buffer: SharedPtr<PlanarYuvBuffer>)
+            -> SharedPtr<VideoFrameBuffer>;
+        fn to_yuv_buffer(buffer: SharedPtr<PlanarYuv8Buffer>) -> SharedPtr<PlanarYuvBuffer>;
+        fn to_yuv8_buffer(buffer: SharedPtr<I420Buffer>) -> SharedPtr<PlanarYuv8Buffer>;
+    }
+}