feat: video publishing (#42)

- Prepare webrtc abstraction ( for future wasm support )
- Added track publish support for videos
  - Added LogoTrack example to simple_room demo
- Lot of cleanup
- There are compiler warnings I'll solve on our v1 release

livekit-webrtc/src/native/data_channel.rs

@@ -0,0 +1,137 @@
+use crate::data_channel::{
+    DataBuffer, DataChannelError, DataChannelInit, DataState, OnBufferedAmountChange, OnMessage,
+    OnStateChange,
+};
+use cxx::SharedPtr;
+use std::str;
+use std::sync::{Arc, Mutex};
+use webrtc_sys::data_channel as sys_dc;
+
+impl From<sys_dc::ffi::DataState> for DataState {
+    fn from(value: sys_dc::ffi::DataState) -> Self {
+        match value {
+            sys_dc::ffi::DataState::Connecting => Self::Connecting,
+            sys_dc::ffi::DataState::Open => Self::Open,
+            sys_dc::ffi::DataState::Closing => Self::Closing,
+            sys_dc::ffi::DataState::Closed => Self::Closed,
+            _ => panic!("unknown data channel state"),
+        }
+    }
+}
+
+impl From<DataChannelInit> for sys_dc::ffi::DataChannelInit {
+    fn from(value: DataChannelInit) -> Self {
+        Self {
+            ordered: value.ordered,
+            has_max_retransmit_time: value.max_retransmit_time.is_some(),
+            max_retransmit_time: value.max_retransmit_time.unwrap_or_default(),
+            has_max_retransmits: value.max_retransmits.is_some(),
+            max_retransmits: value.max_retransmits.unwrap_or_default(),
+            protocol: value.protocol,
+            id: value.id,
+            has_priority: false,
+            priority: sys_dc::ffi::Priority::Medium,
+            negotiated: value.negotiated,
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct DataChannel {
+    #[allow(dead_code)]
+    native_observer: SharedPtr<sys_dc::ffi::NativeDataChannelObserver>,
+    observer: Arc<DataChannelObserver>,
+
+    pub(crate) sys_handle: SharedPtr<sys_dc::ffi::DataChannel>,
+}
+
+impl DataChannel {
+    pub fn configure(sys_handle: SharedPtr<sys_dc::ffi::DataChannel>) -> Self {
+        unsafe {
+            let observer = Arc::new(DataChannelObserver::default());
+            let dc = Self {
+                sys_handle: sys_handle.clone(),
+                native_observer: sys_dc::ffi::create_native_data_channel_observer(
+                    Box::new(sys_dc::DataChannelObserverWrapper::new(observer.clone())),
+                    &*sys_handle as *const _ as *mut _,
+                ),
+                observer,
+            };
+
+            dc.sys_handle
+                .register_observer(&*dc.native_observer as *const _ as *mut _);
+            dc
+        }
+    }
+
+    pub fn send(&self, data: &[u8], binary: bool) -> Result<(), DataChannelError> {
+        if !binary {
+            str::from_utf8(data)?;
+        }
+
+        let buffer = sys_dc::ffi::DataBuffer {
+            ptr: data.as_ptr(),
+            len: data.len(),
+            binary,
+        };
+
+        self.sys_handle
+            .send(&buffer)
+            .then_some(())
+            .ok_or(DataChannelError::Send)
+    }
+
+    pub fn label(&self) -> String {
+        self.sys_handle.label()
+    }
+
+    pub fn state(&self) -> DataState {
+        self.sys_handle.state().into()
+    }
+
+    pub fn close(&self) {
+        self.sys_handle.close();
+    }
+
+    pub fn on_state_change(&self, handler: Option<OnStateChange>) {
+        *self.observer.state_change_handler.lock().unwrap() = handler;
+    }
+
+    pub fn on_message(&self, handler: Option<OnMessage>) {
+        *self.observer.message_handler.lock().unwrap() = handler;
+    }
+
+    pub fn on_buffered_amount_change(&self, handler: Option<OnBufferedAmountChange>) {
+        *self.observer.buffered_amount_change_handler.lock().unwrap() = handler;
+    }
+}
+
+#[derive(Default)]
+struct DataChannelObserver {
+    state_change_handler: Mutex<Option<OnStateChange>>,
+    message_handler: Mutex<Option<OnMessage>>,
+    buffered_amount_change_handler: Mutex<Option<OnBufferedAmountChange>>,
+}
+
+impl sys_dc::DataChannelObserver for DataChannelObserver {
+    fn on_state_change(&self, state: sys_dc::ffi::DataState) {
+        let mut handler = self.state_change_handler.lock().unwrap();
+        if let Some(f) = handler.as_mut() {
+            f(state.into());
+        }
+    }
+
+    fn on_message(&self, data: &[u8], binary: bool) {
+        let mut handler = self.message_handler.lock().unwrap();
+        if let Some(f) = handler.as_mut() {
+            f(DataBuffer { data, binary });
+        }
+    }
+
+    fn on_buffered_amount_change(&self, sent_data_size: u64) {
+        let mut handler = self.buffered_amount_change_handler.lock().unwrap();
+        if let Some(f) = handler.as_mut() {
+            f(sent_data_size);
+        }
+    }
+}

livekit-webrtc/src/native/ice_candidate.rs

@@ -0,0 +1,48 @@
+use crate::ice_candidate as ic;
+use crate::session_description::SdpParseError;
+use cxx::SharedPtr;
+use webrtc_sys::jsep as sys_jsep;
+
+#[derive(Clone)]
+pub struct IceCandidate {
+    pub(crate) sys_handle: SharedPtr<sys_jsep::ffi::IceCandidate>,
+}
+
+impl IceCandidate {
+    pub fn parse(
+        sdp_mid: &str,
+        sdp_mline_index: i32,
+        sdp: &str,
+    ) -> Result<ic::IceCandidate, SdpParseError> {
+        let res = sys_jsep::ffi::create_ice_candidate(
+            sdp_mid.to_string(),
+            sdp_mline_index,
+            sdp.to_string(),
+        );
+
+        match res {
+            Ok(sys_handle) => Ok(ic::IceCandidate {
+                handle: IceCandidate { sys_handle },
+            }),
+            Err(e) => Err(unsafe { sys_jsep::ffi::SdpParseError::from(e.what()).into() }),
+        }
+    }
+
+    pub fn sdp_mid(&self) -> String {
+        self.sys_handle.sdp_mid()
+    }
+
+    pub fn sdp_mline_index(&self) -> i32 {
+        self.sys_handle.sdp_mline_index()
+    }
+
+    pub fn candidate(&self) -> String {
+        self.sys_handle.candidate()
+    }
+}
+
+impl ToString for IceCandidate {
+    fn to_string(&self) -> String {
+        self.sys_handle.stringify()
+    }
+}

livekit-webrtc/src/native/media_stream.rs

@@ -0,0 +1,118 @@
+use crate::media_stream::{self, MediaStreamTrack, RtcTrackState};
+use cxx::SharedPtr;
+use webrtc_sys::media_stream as sys_ms;
+use webrtc_sys::media_stream::ffi::{
+    audio_to_media, media_to_audio, media_to_video, video_to_media,
+};
+use webrtc_sys::{MEDIA_TYPE_AUDIO, MEDIA_TYPE_VIDEO};
+
+impl From<sys_ms::ffi::TrackState> for RtcTrackState {
+    fn from(state: sys_ms::ffi::TrackState) -> Self {
+        match state {
+            sys_ms::ffi::TrackState::Live => RtcTrackState::Live,
+            sys_ms::ffi::TrackState::Ended => RtcTrackState::Ended,
+            _ => panic!("unknown TrackState"),
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct MediaStream {
+    pub(crate) sys_handle: SharedPtr<sys_ms::ffi::MediaStream>,
+}
+
+impl MediaStream {
+    pub fn id(&self) -> String {
+        self.sys_handle.id()
+    }
+
+    pub fn audio_tracks(&self) -> Vec<media_stream::RtcAudioTrack> {
+        self.sys_handle
+            .get_audio_tracks()
+            .into_iter()
+            .map(|t| media_stream::RtcAudioTrack {
+                handle: RtcAudioTrack { sys_handle: t.ptr },
+            })
+            .collect()
+    }
+
+    pub fn video_tracks(&self) -> Vec<media_stream::RtcVideoTrack> {
+        self.sys_handle
+            .get_video_tracks()
+            .into_iter()
+            .map(|t| media_stream::RtcVideoTrack {
+                handle: RtcVideoTrack { sys_handle: t.ptr },
+            })
+            .collect()
+    }
+}
+
+pub fn new_media_stream_track(
+    sys_handle: SharedPtr<sys_ms::ffi::MediaStreamTrack>,
+) -> MediaStreamTrack {
+    if sys_handle.kind() == MEDIA_TYPE_AUDIO {
+        MediaStreamTrack::Audio(media_stream::RtcAudioTrack {
+            handle: RtcAudioTrack {
+                sys_handle: media_to_audio(sys_handle),
+            },
+        })
+    } else if sys_handle.kind() == MEDIA_TYPE_VIDEO {
+        MediaStreamTrack::Video(media_stream::RtcVideoTrack {
+            handle: RtcVideoTrack {
+                sys_handle: media_to_video(sys_handle),
+            },
+        })
+    } else {
+        panic!("unknown track kind")
+    }
+}
+
+macro_rules! impl_media_stream_track {
+    ($cast:ident) => {
+        pub fn id(&self) -> String {
+            let ptr = sys_ms::ffi::$cast(self.sys_handle.clone());
+            ptr.id()
+        }
+
+        pub fn enabled(&self) -> bool {
+            let ptr = sys_ms::ffi::$cast(self.sys_handle.clone());
+            ptr.enabled()
+        }
+
+        pub fn set_enabled(&self, enabled: bool) -> bool {
+            let ptr = sys_ms::ffi::$cast(self.sys_handle.clone());
+            ptr.set_enabled(enabled)
+        }
+
+        pub fn state(&self) -> RtcTrackState {
+            let ptr = sys_ms::ffi::$cast(self.sys_handle.clone());
+            ptr.state().into()
+        }
+    };
+}
+
+#[derive(Clone)]
+pub struct RtcVideoTrack {
+    pub(crate) sys_handle: SharedPtr<sys_ms::ffi::VideoTrack>,
+}
+
+impl RtcVideoTrack {
+    impl_media_stream_track!(video_to_media);
+
+    pub fn sys_handle(&self) -> SharedPtr<sys_ms::ffi::MediaStreamTrack> {
+        video_to_media(self.sys_handle.clone())
+    }
+}
+
+#[derive(Clone)]
+pub struct RtcAudioTrack {
+    pub(crate) sys_handle: SharedPtr<sys_ms::ffi::AudioTrack>,
+}
+
+impl RtcAudioTrack {
+    impl_media_stream_track!(audio_to_media);
+
+    pub fn sys_handle(&self) -> SharedPtr<sys_ms::ffi::MediaStreamTrack> {
+        audio_to_media(self.sys_handle.clone())
+    }
+}

livekit-webrtc/src/native/mod.rs

@@ -0,0 +1,48 @@
+pub mod data_channel;
+pub mod ice_candidate;
+pub mod media_stream;
+pub mod peer_connection;
+pub mod peer_connection_factory;
+pub mod rtp_parameters;
+pub mod rtp_receiver;
+pub mod rtp_sender;
+pub mod rtp_transceiver;
+pub mod session_description;
+pub mod video_frame;
+pub mod video_source;
+pub mod video_stream;
+pub mod yuv_helper;
+
+use crate::MediaType;
+use crate::{RtcError, RtcErrorType};
+use webrtc_sys::rtc_error as sys_err;
+use webrtc_sys::webrtc as sys_rtc;
+
+impl From<sys_err::ffi::RTCErrorType> for RtcErrorType {
+    fn from(value: sys_err::ffi::RTCErrorType) -> Self {
+        match value {
+            sys_err::ffi::RTCErrorType::InvalidState => Self::InvalidState,
+            _ => Self::Internal,
+        }
+    }
+}
+
+impl From<sys_err::ffi::RTCError> for RtcError {
+    fn from(value: sys_err::ffi::RTCError) -> Self {
+        Self {
+            error_type: value.error_type.into(),
+            message: value.message,
+        }
+    }
+}
+
+impl From<MediaType> for sys_rtc::ffi::MediaType {
+    fn from(value: MediaType) -> Self {
+        match value {
+            MediaType::Audio => Self::Audio,
+            MediaType::Video => Self::Video,
+            MediaType::Data => Self::Data,
+            MediaType::Unsupported => Self::Unsupported,
+        }
+    }
+}

livekit-webrtc/src/native/peer_connection.rs

@@ -0,0 +1,579 @@
+use crate::data_channel::DataChannel;
+use crate::data_channel::DataChannelInit;
+use crate::ice_candidate::IceCandidate;
+use crate::imp::data_channel as imp_dc;
+use crate::imp::ice_candidate as imp_ic;
+use crate::imp::media_stream as imp_ms;
+use crate::imp::rtp_receiver as imp_rr;
+use crate::imp::rtp_sender as imp_rs;
+use crate::imp::rtp_transceiver as imp_rt;
+use crate::imp::session_description as imp_sdp;
+use crate::media_stream::{MediaStream, MediaStreamTrack};
+use crate::peer_connection::{
+    AnswerOptions, IceCandidateError, IceConnectionState, IceGatheringState, OfferOptions,
+    OnConnectionChange, OnDataChannel, OnIceCandidate, OnIceCandidateError, OnIceConnectionChange,
+    OnIceGatheringChange, OnNegotiationNeeded, OnSignalingChange, OnTrack, PeerConnectionState,
+    SignalingState, TrackEvent,
+};
+use crate::rtp_receiver::RtpReceiver;
+use crate::rtp_sender::RtpSender;
+use crate::rtp_transceiver::RtpTransceiver;
+use crate::rtp_transceiver::RtpTransceiverInit;
+use crate::MediaType;
+use crate::{session_description::SessionDescription, RtcError};
+use cxx::{SharedPtr, UniquePtr};
+use futures::channel::oneshot;
+use std::mem::ManuallyDrop;
+use std::sync::{Arc, Mutex};
+use webrtc_sys::data_channel as sys_dc;
+use webrtc_sys::jsep as sys_jsep;
+use webrtc_sys::peer_connection as sys_pc;
+use webrtc_sys::rtc_error as sys_err;
+
+impl From<OfferOptions> for sys_pc::ffi::RTCOfferAnswerOptions {
+    fn from(options: OfferOptions) -> Self {
+        Self {
+            ice_restart: options.ice_restart,
+            offer_to_receive_audio: options.offer_to_receive_audio as i32,
+            offer_to_receive_video: options.offer_to_receive_video as i32,
+            ..Default::default()
+        }
+    }
+}
+
+impl From<AnswerOptions> for sys_pc::ffi::RTCOfferAnswerOptions {
+    fn from(_options: AnswerOptions) -> Self {
+        Self::default()
+    }
+}
+
+impl From<sys_pc::ffi::PeerConnectionState> for PeerConnectionState {
+    fn from(state: sys_pc::ffi::PeerConnectionState) -> Self {
+        match state {
+            sys_pc::ffi::PeerConnectionState::New => PeerConnectionState::New,
+            sys_pc::ffi::PeerConnectionState::Connecting => PeerConnectionState::Connecting,
+            sys_pc::ffi::PeerConnectionState::Connected => PeerConnectionState::Connected,
+            sys_pc::ffi::PeerConnectionState::Disconnected => PeerConnectionState::Disconnected,
+            sys_pc::ffi::PeerConnectionState::Failed => PeerConnectionState::Failed,
+            sys_pc::ffi::PeerConnectionState::Closed => PeerConnectionState::Closed,
+            _ => panic!("unknown PeerConnectionState"),
+        }
+    }
+}
+
+impl From<sys_pc::ffi::IceConnectionState> for IceConnectionState {
+    fn from(state: sys_pc::ffi::IceConnectionState) -> Self {
+        match state {
+            sys_pc::ffi::IceConnectionState::IceConnectionNew => IceConnectionState::New,
+            sys_pc::ffi::IceConnectionState::IceConnectionChecking => IceConnectionState::Checking,
+            sys_pc::ffi::IceConnectionState::IceConnectionConnected => {
+                IceConnectionState::Connected
+            }
+            sys_pc::ffi::IceConnectionState::IceConnectionCompleted => {
+                IceConnectionState::Completed
+            }
+            sys_pc::ffi::IceConnectionState::IceConnectionFailed => IceConnectionState::Failed,
+            sys_pc::ffi::IceConnectionState::IceConnectionDisconnected => {
+                IceConnectionState::Disconnected
+            }
+            sys_pc::ffi::IceConnectionState::IceConnectionClosed => IceConnectionState::Closed,
+            sys_pc::ffi::IceConnectionState::IceConnectionMax => IceConnectionState::Max,
+            _ => panic!("unknown IceConnectionState"),
+        }
+    }
+}
+
+impl From<sys_pc::ffi::IceGatheringState> for IceGatheringState {
+    fn from(state: sys_pc::ffi::IceGatheringState) -> Self {
+        match state {
+            sys_pc::ffi::IceGatheringState::IceGatheringNew => IceGatheringState::New,
+            sys_pc::ffi::IceGatheringState::IceGatheringGathering => IceGatheringState::Gathering,
+            sys_pc::ffi::IceGatheringState::IceGatheringComplete => IceGatheringState::Complete,
+            _ => panic!("unknown IceGatheringState"),
+        }
+    }
+}
+
+impl From<sys_pc::ffi::SignalingState> for SignalingState {
+    fn from(state: sys_pc::ffi::SignalingState) -> Self {
+        match state {
+            sys_pc::ffi::SignalingState::Stable => SignalingState::Stable,
+            sys_pc::ffi::SignalingState::HaveLocalOffer => SignalingState::HaveLocalOffer,
+            sys_pc::ffi::SignalingState::HaveRemoteOffer => SignalingState::HaveRemoteOffer,
+            sys_pc::ffi::SignalingState::HaveLocalPrAnswer => SignalingState::HaveLocalPrAnswer,
+            sys_pc::ffi::SignalingState::HaveRemotePrAnswer => SignalingState::HaveRemotePrAnswer,
+            sys_pc::ffi::SignalingState::Closed => SignalingState::Closed,
+            _ => panic!("unknown SignalingState"),
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct PeerConnection {
+    native_observer: SharedPtr<sys_pc::ffi::NativePeerConnectionObserver>,
+    observer: Arc<PeerObserver>,
+
+    pub(crate) sys_handle: SharedPtr<sys_pc::ffi::PeerConnection>,
+}
+
+impl PeerConnection {
+    pub fn configure(
+        sys_handle: SharedPtr<sys_pc::ffi::PeerConnection>,
+        observer: Arc<PeerObserver>,
+        native_observer: SharedPtr<sys_pc::ffi::NativePeerConnectionObserver>,
+    ) -> Self {
+        Self {
+            sys_handle,
+            observer,
+            native_observer,
+        }
+    }
+
+    pub async fn create_offer(
+        &self,
+        options: OfferOptions,
+    ) -> Result<SessionDescription, RtcError> {
+        let (mut native_wrapper, mut sdp_rx, mut err_rx) = create_sdp_observer();
+
+        unsafe {
+            self.sys_handle
+                .create_offer(native_wrapper.pin_mut(), options.into());
+        }
+
+        futures::select! {
+            sdp = sdp_rx => Ok(sdp.unwrap()),
+            err = err_rx => Err(err.unwrap()),
+        }
+    }
+
+    pub async fn create_answer(
+        &self,
+        options: AnswerOptions,
+    ) -> Result<SessionDescription, RtcError> {
+        let (mut native_wrapper, mut sdp_rx, mut err_rx) = create_sdp_observer();
+
+        unsafe {
+            self.sys_handle
+                .create_answer(native_wrapper.pin_mut(), options.into());
+        }
+
+        futures::select! {
+            sdp = sdp_rx => Ok(sdp.unwrap()),
+            err = err_rx => Err(err.unwrap()),
+        }
+    }
+
+    pub async fn set_local_description(&self, desc: SessionDescription) -> Result<(), RtcError> {
+        let (tx, rx) = oneshot::channel();
+        let wrapper =
+            sys_jsep::SetLocalSdpObserverWrapper(ManuallyDrop::new(Box::new(move |error| {
+                let _ = tx.send(if error.ok() { Ok(()) } else { Err(error) });
+            })));
+
+        let mut native_wrapper =
+            sys_jsep::ffi::create_native_set_local_sdp_observer(Box::new(wrapper));
+
+        unsafe {
+            self.sys_handle
+                .set_local_description(desc.handle.sys_handle, native_wrapper.pin_mut());
+        }
+
+        rx.await.unwrap().map_err(Into::into)
+    }
+
+    pub async fn set_remote_description(&self, desc: SessionDescription) -> Result<(), RtcError> {
+        let (tx, rx) = oneshot::channel();
+        let wrapper =
+            sys_jsep::SetRemoteSdpObserverWrapper(ManuallyDrop::new(Box::new(move |error| {
+                let _ = tx.send(if error.ok() { Ok(()) } else { Err(error) });
+            })));
+
+        let mut native_wrapper =
+            sys_jsep::ffi::create_native_set_remote_sdp_observer(Box::new(wrapper));
+
+        unsafe {
+            self.sys_handle
+                .set_remote_description(desc.handle.sys_handle, native_wrapper.pin_mut());
+        }
+
+        rx.await.unwrap().map_err(Into::into)
+    }
+
+    pub async fn add_ice_candidate(&self, candidate: IceCandidate) -> Result<(), RtcError> {
+        let (tx, rx) = oneshot::channel();
+        let observer =
+            sys_pc::AddIceCandidateObserverWrapper(ManuallyDrop::new(Box::new(|error| {
+                let _ = tx.send(if error.ok() { Ok(()) } else { Err(error) });
+            })));
+
+        let mut native_observer =
+            sys_pc::ffi::create_native_add_ice_candidate_observer(Box::new(observer));
+        self.sys_handle
+            .add_ice_candidate(candidate.handle.sys_handle, native_observer.pin_mut());
+
+        rx.await.unwrap().map_err(Into::into)
+    }
+
+    pub fn create_data_channel(
+        &self,
+        label: &str,
+        init: DataChannelInit,
+    ) -> Result<DataChannel, RtcError> {
+        let native_init = sys_dc::ffi::create_data_channel_init(init.into());
+        let res = self
+            .sys_handle
+            .create_data_channel(label.to_string(), native_init);
+
+        match res {
+            Ok(sys_handle) => Ok(DataChannel {
+                handle: imp_dc::DataChannel::configure(sys_handle),
+            }),
+            Err(e) => Err(unsafe { sys_err::ffi::RTCError::from(e.what()).into() }),
+        }
+    }
+
+    pub fn add_track<T: AsRef<str>>(
+        &self,
+        track: MediaStreamTrack,
+        stream_ids: &[T],
+    ) -> Result<RtpSender, RtcError> {
+        let stream_ids = stream_ids.iter().map(|s| s.as_ref().to_owned()).collect();
+        let res = self.sys_handle.add_track(track.sys_handle(), &stream_ids);
+
+        match res {
+            Ok(sys_handle) => Ok(RtpSender {
+                handle: imp_rs::RtpSender { sys_handle },
+            }),
+            Err(e) => unsafe { Err(sys_err::ffi::RTCError::from(e.what()).into()) },
+        }
+    }
+
+    pub fn add_transceiver(
+        &self,
+        track: MediaStreamTrack,
+        init: RtpTransceiverInit,
+    ) -> Result<RtpTransceiver, RtcError> {
+        let res = self
+            .sys_handle
+            .add_transceiver(track.sys_handle(), init.into());
+
+        match res {
+            Ok(sys_handle) => Ok(RtpTransceiver {
+                handle: imp_rt::RtpTransceiver {
+                    sys_handle: sys_handle,
+                },
+            }),
+            Err(e) => unsafe { Err(sys_err::ffi::RTCError::from(e.what()).into()) },
+        }
+    }
+
+    pub fn add_transceiver_for_media(
+        &self,
+        media_type: MediaType,
+        init: RtpTransceiverInit,
+    ) -> Result<RtpTransceiver, RtcError> {
+        let res = self
+            .sys_handle
+            .add_transceiver_for_media(media_type.into(), init.into());
+
+        match res {
+            Ok(cxx_handle) => Ok(RtpTransceiver {
+                handle: imp_rt::RtpTransceiver {
+                    sys_handle: cxx_handle,
+                },
+            }),
+            Err(e) => unsafe { Err(sys_err::ffi::RTCError::from(e.what()).into()) },
+        }
+    }
+
+    pub fn close(&self) {
+        self.sys_handle.close();
+    }
+
+    pub fn connection_state(&self) -> PeerConnectionState {
+        self.sys_handle.connection_state().into()
+    }
+
+    pub fn ice_connection_state(&self) -> IceConnectionState {
+        self.sys_handle.ice_connection_state().into()
+    }
+
+    pub fn ice_gathering_state(&self) -> IceGatheringState {
+        self.sys_handle.ice_gathering_state().into()
+    }
+
+    pub fn signaling_state(&self) -> SignalingState {
+        self.sys_handle.signaling_state().into()
+    }
+
+    pub fn current_local_description(&self) -> Option<SessionDescription> {
+        let sdp = self.sys_handle.current_local_description();
+        if sdp.is_null() {
+            return None;
+        }
+
+        Some(SessionDescription {
+            handle: imp_sdp::SessionDescription { sys_handle: sdp },
+        })
+    }
+
+    pub fn current_remote_description(&self) -> Option<SessionDescription> {
+        let sdp = self.sys_handle.current_remote_description();
+        if sdp.is_null() {
+            return None;
+        }
+
+        Some(SessionDescription {
+            handle: imp_sdp::SessionDescription { sys_handle: sdp },
+        })
+    }
+
+    pub fn remove_track(&self, sender: RtpSender) -> Result<(), RtcError> {
+        self.sys_handle
+            .remove_track(sender.handle.sys_handle)
+            .map_err(|e| unsafe { sys_err::ffi::RTCError::from(e.what()).into() })
+    }
+
+    pub fn senders(&self) -> Vec<RtpSender> {
+        self.sys_handle
+            .get_senders()
+            .into_iter()
+            .map(|sender| RtpSender {
+                handle: imp_rs::RtpSender {
+                    sys_handle: sender.ptr,
+                },
+            })
+            .collect()
+    }
+
+    pub fn receivers(&self) -> Vec<RtpReceiver> {
+        self.sys_handle
+            .get_receivers()
+            .into_iter()
+            .map(|receiver| RtpReceiver {
+                handle: imp_rr::RtpReceiver {
+                    sys_handle: receiver.ptr,
+                },
+            })
+            .collect()
+    }
+
+    pub fn transceivers(&self) -> Vec<RtpTransceiver> {
+        self.sys_handle
+            .get_transceivers()
+            .into_iter()
+            .map(|transceiver| RtpTransceiver {
+                handle: imp_rt::RtpTransceiver {
+                    sys_handle: transceiver.ptr,
+                },
+            })
+            .collect()
+    }
+
+    pub fn on_connection_state_change(&self, f: Option<OnConnectionChange>) {
+        *self.observer.connection_change_handler.lock().unwrap() = f;
+    }
+
+    pub fn on_data_channel(&self, f: Option<OnDataChannel>) {
+        *self.observer.data_channel_handler.lock().unwrap() = f;
+    }
+
+    pub fn on_ice_candidate(&self, f: Option<OnIceCandidate>) {
+        *self.observer.ice_candidate_handler.lock().unwrap() = f;
+    }
+
+    pub fn on_ice_candidate_error(&self, f: Option<OnIceCandidateError>) {
+        *self.observer.ice_candidate_error_handler.lock().unwrap() = f;
+    }
+
+    pub fn on_ice_connection_state_change(&self, f: Option<OnIceConnectionChange>) {
+        *self.observer.ice_connection_change_handler.lock().unwrap() = f;
+    }
+
+    pub fn on_ice_gathering_state_change(&self, f: Option<OnIceGatheringChange>) {
+        *self.observer.ice_gathering_change_handler.lock().unwrap() = f;
+    }
+
+    pub fn on_negotiation_needed(&self, f: Option<OnNegotiationNeeded>) {
+        *self.observer.negotiation_needed_handler.lock().unwrap() = f;
+    }
+
+    pub fn on_signaling_state_change(&self, f: Option<OnSignalingChange>) {
+        *self.observer.signaling_change_handler.lock().unwrap() = f;
+    }
+
+    pub fn on_track(&self, f: Option<OnTrack>) {
+        *self.observer.track_handler.lock().unwrap() = f;
+    }
+}
+
+fn create_sdp_observer() -> (
+    UniquePtr<sys_pc::ffi::NativeCreateSdpObserverHandle>,
+    oneshot::Receiver<SessionDescription>,
+    oneshot::Receiver<RtcError>,
+) {
+    let (sdp_tx, sdp_rx) = oneshot::channel();
+    let (err_tx, err_rx) = oneshot::channel();
+
+    let wrapper = sys_jsep::CreateSdpObserverWrapper {
+        on_success: ManuallyDrop::new(Box::new(move |session_description| {
+            let _ = sdp_tx.send(SessionDescription {
+                handle: imp_sdp::SessionDescription {
+                    sys_handle: session_description,
+                },
+            });
+        })),
+        on_failure: ManuallyDrop::new(Box::new(move |error| {
+            let _ = err_tx.send(error.into());
+        })),
+    };
+
+    (
+        sys_jsep::ffi::create_native_create_sdp_observer(Box::new(wrapper)),
+        sdp_rx,
+        err_rx,
+    )
+}
+
+#[derive(Default)]
+pub struct PeerObserver {
+    pub connection_change_handler: Mutex<Option<OnConnectionChange>>,
+    pub data_channel_handler: Mutex<Option<OnDataChannel>>,
+    pub ice_candidate_handler: Mutex<Option<OnIceCandidate>>,
+    pub ice_candidate_error_handler: Mutex<Option<OnIceCandidateError>>,
+    pub ice_connection_change_handler: Mutex<Option<OnIceConnectionChange>>,
+    pub ice_gathering_change_handler: Mutex<Option<OnIceGatheringChange>>,
+    pub negotiation_needed_handler: Mutex<Option<OnNegotiationNeeded>>,
+    pub signaling_change_handler: Mutex<Option<OnSignalingChange>>,
+    pub track_handler: Mutex<Option<OnTrack>>,
+}
+
+impl sys_pc::PeerConnectionObserver for PeerObserver {
+    fn on_signaling_change(&self, new_state: sys_pc::ffi::SignalingState) {
+        if let Some(f) = self.signaling_change_handler.lock().unwrap().as_mut() {
+            f(new_state.into());
+        }
+    }
+
+    fn on_add_stream(&self, _stream: SharedPtr<webrtc_sys::media_stream::ffi::MediaStream>) {}
+
+    fn on_remove_stream(&self, _stream: SharedPtr<webrtc_sys::media_stream::ffi::MediaStream>) {}
+
+    fn on_data_channel(&self, data_channel: SharedPtr<sys_dc::ffi::DataChannel>) {
+        if let Some(f) = self.data_channel_handler.lock().unwrap().as_mut() {
+            f(DataChannel {
+                handle: imp_dc::DataChannel::configure(data_channel),
+            });
+        }
+    }
+
+    fn on_renegotiation_needed(&self) {}
+
+    fn on_negotiation_needed_event(&self, event: u32) {
+        if let Some(f) = self.negotiation_needed_handler.lock().unwrap().as_mut() {
+            f(event);
+        }
+    }
+
+    fn on_ice_connection_change(&self, _new_state: sys_pc::ffi::IceConnectionState) {}
+
+    fn on_standardized_ice_connection_change(&self, new_state: sys_pc::ffi::IceConnectionState) {
+        if let Some(f) = self.ice_connection_change_handler.lock().unwrap().as_mut() {
+            f(new_state.into());
+        }
+    }
+
+    fn on_connection_change(&self, new_state: sys_pc::ffi::PeerConnectionState) {
+        if let Some(f) = self.connection_change_handler.lock().unwrap().as_mut() {
+            f(new_state.into());
+        }
+    }
+
+    fn on_ice_gathering_change(&self, new_state: sys_pc::ffi::IceGatheringState) {
+        if let Some(f) = self.ice_gathering_change_handler.lock().unwrap().as_mut() {
+            f(new_state.into());
+        }
+    }
+
+    fn on_ice_candidate(&self, candidate: SharedPtr<sys_jsep::ffi::IceCandidate>) {
+        if let Some(f) = self.ice_candidate_handler.lock().unwrap().as_mut() {
+            f(IceCandidate {
+                handle: imp_ic::IceCandidate {
+                    sys_handle: candidate,
+                },
+            });
+        }
+    }
+
+    fn on_ice_candidate_error(
+        &self,
+        address: String,
+        port: i32,
+        url: String,
+        error_code: i32,
+        error_text: String,
+    ) {
+        if let Some(f) = self.ice_candidate_error_handler.lock().unwrap().as_mut() {
+            f(IceCandidateError {
+                address,
+                port,
+                url,
+                error_code,
+                error_text,
+            });
+        }
+    }
+
+    fn on_ice_candidates_removed(
+        &self,
+        _removed: Vec<SharedPtr<webrtc_sys::candidate::ffi::Candidate>>,
+    ) {
+    }
+
+    fn on_ice_connection_receiving_change(&self, _receiving: bool) {}
+
+    fn on_ice_selected_candidate_pair_changed(
+        &self,
+        _event: sys_pc::ffi::CandidatePairChangeEvent,
+    ) {
+    }
+
+    fn on_add_track(
+        &self,
+        _receiver: SharedPtr<webrtc_sys::rtp_receiver::ffi::RtpReceiver>,
+        _streams: Vec<SharedPtr<webrtc_sys::media_stream::ffi::MediaStream>>,
+    ) {
+    }
+
+    fn on_track(&self, transceiver: SharedPtr<webrtc_sys::rtp_transceiver::ffi::RtpTransceiver>) {
+        if let Some(f) = self.track_handler.lock().unwrap().as_mut() {
+            let receiver = transceiver.receiver();
+            let streams = receiver.streams();
+            let track = receiver.track();
+
+            f(TrackEvent {
+                receiver: RtpReceiver {
+                    handle: imp_rr::RtpReceiver {
+                        sys_handle: receiver,
+                    },
+                },
+                streams: streams
+                    .into_iter()
+                    .map(|s| MediaStream {
+                        handle: imp_ms::MediaStream { sys_handle: s.ptr },
+                    })
+                    .collect(),
+                track: imp_ms::new_media_stream_track(track),
+                transceiver: RtpTransceiver {
+                    handle: imp_rt::RtpTransceiver {
+                        sys_handle: transceiver,
+                    },
+                },
+            });
+        }
+    }
+
+    fn on_remove_track(&self, _receiver: SharedPtr<webrtc_sys::rtp_receiver::ffi::RtpReceiver>) {}
+
+    fn on_interesting_usage(&self, _usage_pattern: i32) {}
+}

livekit-webrtc/src/native/peer_connection_factory.rs

@@ -0,0 +1,146 @@
+use crate::imp::media_stream as imp_ms;
+use crate::imp::peer_connection as imp_pc;
+use crate::media_stream::RtcVideoTrack;
+use crate::peer_connection::PeerConnection;
+use crate::peer_connection_factory::{
+    ContinualGatheringPolicy, IceServer, IceTransportsType, RtcConfiguration,
+};
+use crate::rtp_parameters::RtpCapabilities;
+use crate::video_source::native::NativeVideoSource;
+use crate::MediaType;
+use crate::RtcError;
+use cxx::SharedPtr;
+use std::sync::Arc;
+use webrtc_sys::peer_connection as sys_pc;
+use webrtc_sys::peer_connection_factory as sys_pcf;
+use webrtc_sys::rtc_error as sys_err;
+use webrtc_sys::webrtc as sys_webrtc;
+
+impl From<IceServer> for sys_pcf::ffi::ICEServer {
+    fn from(value: IceServer) -> Self {
+        sys_pcf::ffi::ICEServer {
+            urls: value.urls,
+            username: value.username,
+            password: value.password,
+        }
+    }
+}
+
+impl From<ContinualGatheringPolicy> for sys_pcf::ffi::ContinualGatheringPolicy {
+    fn from(value: ContinualGatheringPolicy) -> Self {
+        match value {
+            ContinualGatheringPolicy::GatherOnce => {
+                sys_pcf::ffi::ContinualGatheringPolicy::GatherOnce
+            }
+            ContinualGatheringPolicy::GatherContinually => {
+                sys_pcf::ffi::ContinualGatheringPolicy::GatherContinually
+            }
+        }
+    }
+}
+
+impl From<IceTransportsType> for sys_pcf::ffi::IceTransportsType {
+    fn from(value: IceTransportsType) -> Self {
+        match value {
+            IceTransportsType::None => sys_pcf::ffi::IceTransportsType::None,
+            IceTransportsType::Relay => sys_pcf::ffi::IceTransportsType::Relay,
+            IceTransportsType::NoHost => sys_pcf::ffi::IceTransportsType::NoHost,
+            IceTransportsType::All => sys_pcf::ffi::IceTransportsType::All,
+        }
+    }
+}
+
+impl From<RtcConfiguration> for sys_pcf::ffi::RTCConfiguration {
+    fn from(value: RtcConfiguration) -> Self {
+        Self {
+            ice_servers: value.ice_servers.into_iter().map(Into::into).collect(),
+            continual_gathering_policy: value.continual_gathering_policy.into(),
+            ice_transport_type: value.ice_transport_type.into(),
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct RTCRuntime {
+    pub(crate) sys_handle: SharedPtr<sys_webrtc::ffi::RTCRuntime>,
+}
+
+impl Default for RTCRuntime {
+    fn default() -> Self {
+        Self {
+            sys_handle: sys_webrtc::ffi::create_rtc_runtime(),
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct PeerConnectionFactory {
+    sys_handle: SharedPtr<sys_pcf::ffi::PeerConnectionFactory>,
+
+    #[allow(unused)]
+    runtime: RTCRuntime,
+}
+
+impl Default for PeerConnectionFactory {
+    fn default() -> Self {
+        let runtime = RTCRuntime::default();
+        Self {
+            sys_handle: sys_pcf::ffi::create_peer_connection_factory(runtime.sys_handle.clone()),
+            runtime,
+        }
+    }
+}
+
+impl PeerConnectionFactory {
+    pub fn create_peer_connection(
+        &self,
+        config: RtcConfiguration,
+    ) -> Result<PeerConnection, RtcError> {
+        let native_config = sys_pcf::ffi::create_rtc_configuration(config.into());
+
+        unsafe {
+            let observer = Arc::new(imp_pc::PeerObserver::default());
+            let native_observer = sys_pc::ffi::create_native_peer_connection_observer(
+                self.runtime.clone().sys_handle,
+                Box::new(sys_pc::PeerConnectionObserverWrapper::new(observer.clone())),
+            );
+
+            let res = self
+                .sys_handle
+                .create_peer_connection(native_config, &*native_observer as *const _ as *mut _);
+
+            match res {
+                Ok(sys_handle) => Ok(PeerConnection {
+                    handle: imp_pc::PeerConnection::configure(
+                        sys_handle,
+                        observer,
+                        native_observer,
+                    ),
+                }),
+                Err(e) => Err(sys_err::ffi::RTCError::from(e.what()).into()),
+            }
+        }
+    }
+
+    pub fn create_video_track(&self, label: &str, source: NativeVideoSource) -> RtcVideoTrack {
+        RtcVideoTrack {
+            handle: imp_ms::RtcVideoTrack {
+                sys_handle: self
+                    .sys_handle
+                    .create_video_track(label.to_string(), source.handle.sys_handle()),
+            },
+        }
+    }
+
+    pub fn get_rtp_sender_capabilities(&self, media_type: MediaType) -> RtpCapabilities {
+        self.sys_handle
+            .get_rtp_sender_capabilities(media_type.into())
+            .into()
+    }
+
+    pub fn get_rtp_receiver_capabilities(&self, media_type: MediaType) -> RtpCapabilities {
+        self.sys_handle
+            .get_rtp_receiver_capabilities(media_type.into())
+            .into()
+    }
+}

livekit-webrtc/src/native/rtp_parameters.rs

@@ -0,0 +1,285 @@
+use crate::rtp_parameters::*;
+use crate::MediaType;
+use webrtc_sys::rtp_parameters as sys_rp;
+use webrtc_sys::webrtc as sys_webrtc;
+
+impl From<sys_webrtc::ffi::Priority> for Priority {
+    fn from(value: sys_webrtc::ffi::Priority) -> Self {
+        match value {
+            sys_webrtc::ffi::Priority::VeryLow => Self::VeryLow,
+            sys_webrtc::ffi::Priority::Low => Self::Low,
+            sys_webrtc::ffi::Priority::Medium => Self::Medium,
+            sys_webrtc::ffi::Priority::High => Self::High,
+            _ => panic!("unknown Priority"),
+        }
+    }
+}
+
+impl From<sys_rp::ffi::RtpExtension> for RtpHeaderExtensionParameters {
+    fn from(value: sys_rp::ffi::RtpExtension) -> Self {
+        Self {
+            uri: value.uri,
+            id: value.id,
+            encrypted: value.encrypt,
+        }
+    }
+}
+
+impl From<sys_rp::ffi::RtpParameters> for RtpParameters {
+    fn from(value: sys_rp::ffi::RtpParameters) -> Self {
+        Self {
+            codecs: value.codecs.into_iter().map(Into::into).collect(),
+            header_extensions: value
+                .header_extensions
+                .into_iter()
+                .map(Into::into)
+                .collect(),
+            rtcp: value.rtcp.into(),
+        }
+    }
+}
+
+impl From<sys_rp::ffi::RtpCodecParameters> for RtpCodecParameters {
+    fn from(value: sys_rp::ffi::RtpCodecParameters) -> Self {
+        Self {
+            mime_type: value.mime_type,
+            payload_type: value.payload_type as u8,
+            clock_rate: value.has_clock_rate.then_some(value.clock_rate as u64),
+            channels: value.has_num_channels.then_some(value.num_channels as u16),
+        }
+    }
+}
+
+impl From<sys_rp::ffi::RtcpParameters> for RtcpParameters {
+    fn from(value: sys_rp::ffi::RtcpParameters) -> Self {
+        Self {
+            cname: value.cname,
+            reduced_size: value.reduced_size,
+        }
+    }
+}
+
+impl From<sys_rp::ffi::RtpEncodingParameters> for RtpEncodingParameters {
+    fn from(value: sys_rp::ffi::RtpEncodingParameters) -> Self {
+        Self {
+            active: value.active,
+            max_bitrate: value
+                .has_max_bitrate_bps
+                .then_some(value.max_bitrate_bps as u64),
+            max_framerate: value.has_max_framerate.then_some(value.max_framerate),
+            priority: value.network_priority.into(),
+            rid: value.rid,
+            scale_resolution_down_by: value
+                .has_scale_resolution_down_by
+                .then_some(value.scale_resolution_down_by),
+        }
+    }
+}
+
+impl From<sys_rp::ffi::RtpCodecCapability> for RtpCodecCapability {
+    fn from(value: sys_rp::ffi::RtpCodecCapability) -> Self {
+        Self {
+            channels: value.has_num_channels.then_some(value.num_channels as u16),
+            mime_type: value.mime_type,
+            clock_rate: value.has_clock_rate.then_some(value.clock_rate as u64),
+            sdp_fmtp_line: {
+                let parameters: Vec<String> = value
+                    .parameters
+                    .into_iter()
+                    .map(|key_value| {
+                        if !key_value.key.is_empty() {
+                            format!("{}={}", key_value.key, key_value.value)
+                        } else {
+                            key_value.value
+                        }
+                    })
+                    .collect();
+
+                if !parameters.is_empty() {
+                    Some(parameters.join(";"))
+                } else {
+                    None
+                }
+            },
+        }
+    }
+}
+
+impl From<sys_rp::ffi::RtpHeaderExtensionCapability> for RtpHeaderExtensionCapability {
+    fn from(value: sys_rp::ffi::RtpHeaderExtensionCapability) -> Self {
+        Self {
+            direction: value.direction.into(),
+            uri: value.uri,
+        }
+    }
+}
+
+impl From<sys_rp::ffi::RtpCapabilities> for RtpCapabilities {
+    fn from(value: sys_rp::ffi::RtpCapabilities) -> Self {
+        Self {
+            codecs: value.codecs.into_iter().map(Into::into).collect(),
+            header_extensions: value
+                .header_extensions
+                .into_iter()
+                .map(Into::into)
+                .collect(),
+        }
+    }
+}
+
+impl From<Priority> for sys_webrtc::ffi::Priority {
+    fn from(value: Priority) -> Self {
+        match value {
+            Priority::VeryLow => Self::VeryLow,
+            Priority::Low => Self::Low,
+            Priority::Medium => Self::Medium,
+            Priority::High => Self::High,
+        }
+    }
+}
+
+impl From<RtpHeaderExtensionParameters> for sys_rp::ffi::RtpExtension {
+    fn from(value: RtpHeaderExtensionParameters) -> Self {
+        Self {
+            uri: value.uri,
+            id: value.id,
+            encrypt: value.encrypted,
+        }
+    }
+}
+
+impl From<RtpParameters> for sys_rp::ffi::RtpParameters {
+    fn from(value: RtpParameters) -> Self {
+        Self {
+            codecs: value.codecs.into_iter().map(Into::into).collect(),
+            header_extensions: value
+                .header_extensions
+                .into_iter()
+                .map(Into::into)
+                .collect(),
+            encodings: Vec::new(),
+            rtcp: value.rtcp.into(),
+            transaction_id: "".to_string(),
+            mid: "".to_string(),
+            has_degradation_preference: false,
+            degradation_preference: sys_rp::ffi::DegradationPreference::Balanced,
+        }
+    }
+}
+
+impl From<RtpCodecParameters> for sys_rp::ffi::RtpCodecParameters {
+    fn from(value: RtpCodecParameters) -> Self {
+        Self {
+            payload_type: value.payload_type as i32,
+            mime_type: value.mime_type,
+            has_clock_rate: value.clock_rate.is_some(),
+            clock_rate: value.clock_rate.unwrap_or_default() as i32,
+            has_num_channels: value.channels.is_some(),
+            num_channels: value.channels.unwrap_or_default() as i32,
+            name: "".to_string(),
+            kind: sys_rp::ffi::MediaType::Audio,
+            has_max_ptime: false,
+            max_ptime: 0,
+            has_ptime: false,
+            ptime: 0,
+            rtcp_feedback: Vec::new(),
+            parameters: Vec::new(),
+        }
+    }
+}
+
+impl From<RtcpParameters> for sys_rp::ffi::RtcpParameters {
+    fn from(value: RtcpParameters) -> Self {
+        Self {
+            cname: value.cname,
+            reduced_size: value.reduced_size,
+            has_ssrc: false,
+            ssrc: 0,
+            mux: false,
+        }
+    }
+}
+
+impl From<RtpEncodingParameters> for sys_rp::ffi::RtpEncodingParameters {
+    fn from(value: RtpEncodingParameters) -> Self {
+        Self {
+            active: value.active,
+            has_max_bitrate_bps: value.max_bitrate.is_some(),
+            max_bitrate_bps: value.max_bitrate.unwrap_or_default() as i32,
+            has_max_framerate: value.max_framerate.is_some(),
+            max_framerate: value.max_framerate.unwrap_or_default(),
+            network_priority: value.priority.into(),
+            rid: value.rid,
+            has_scale_resolution_down_by: value.scale_resolution_down_by.is_some(),
+            scale_resolution_down_by: value.scale_resolution_down_by.unwrap_or_default(),
+            adaptive_ptime: false,
+            bitrate_priority: sys_rp::DEFAULT_BITRATE_PRIORITY,
+            has_min_bitrate_bps: false,
+            min_bitrate_bps: 0,
+            has_num_temporal_layers: false,
+            num_temporal_layers: 0,
+            has_scalability_mode: false,
+            scalability_mode: "".to_string(),
+            has_ssrc: false,
+            ssrc: 0,
+        }
+    }
+}
+
+impl From<RtpCodecCapability> for sys_rp::ffi::RtpCodecCapability {
+    fn from(value: RtpCodecCapability) -> Self {
+        let mime_type: Vec<&str> = value.mime_type.split('/').collect();
+        let kind = match mime_type[0] {
+            "audio" => sys_webrtc::ffi::MediaType::Audio,
+            "video" => sys_webrtc::ffi::MediaType::Video,
+            _ => panic!("invalid media type"),
+        };
+        let name = mime_type[1].to_string();
+
+        Self {
+            name,
+            kind,
+            has_clock_rate: value.clock_rate.is_some(),
+            clock_rate: value.clock_rate.unwrap_or_default() as i32,
+            has_num_channels: value.channels.is_some(),
+            num_channels: value.channels.unwrap_or_default() as i32,
+            parameters: {
+                value
+                    .sdp_fmtp_line
+                    .map(|sdp_fmtp_line| {
+                        sdp_fmtp_line
+                            .split(';')
+                            .map(|v| {
+                                let key_value: Vec<&str> = v.split('=').collect();
+                                if key_value.len() == 2 {
+                                    sys_rp::ffi::StringKeyValue {
+                                        key: key_value[0].to_string(),
+                                        value: key_value[1].to_string(),
+                                    }
+                                } else {
+                                    sys_rp::ffi::StringKeyValue {
+                                        key: "".to_string(),
+                                        value: key_value[0].to_string(),
+                                    }
+                                }
+                            })
+                            .collect()
+                    })
+                    .unwrap_or_default()
+            },
+            // Ignore
+            mime_type: String::default(), // !!
+            has_preferred_payload_type: false,
+            preferred_payload_type: 0,
+            has_max_ptime: false,
+            max_ptime: 0,
+            has_ptime: false,
+            ptime: 0,
+            rtcp_feedback: Vec::default(),
+            options: Vec::default(),
+            max_temporal_layer_extensions: 0,
+            max_spatial_layer_extensions: 0,
+            svc_multi_stream_support: false,
+        }
+    }
+}

livekit-webrtc/src/native/rtp_receiver.rs

@@ -0,0 +1,24 @@
+use super::media_stream::new_media_stream_track;
+use crate::{media_stream::MediaStreamTrack, rtp_parameters::RtpParameters};
+use cxx::SharedPtr;
+use webrtc_sys::rtp_receiver as sys_rr;
+
+#[derive(Clone)]
+pub struct RtpReceiver {
+    pub(crate) sys_handle: SharedPtr<sys_rr::ffi::RtpReceiver>,
+}
+
+impl RtpReceiver {
+    pub fn track(&self) -> Option<MediaStreamTrack> {
+        let track_handle = self.sys_handle.track();
+        if track_handle.is_null() {
+            return None;
+        }
+
+        Some(new_media_stream_track(track_handle))
+    }
+
+    pub fn parameters(&self) -> RtpParameters {
+        self.sys_handle.get_parameters().into()
+    }
+}

livekit-webrtc/src/native/rtp_sender.rs

@@ -0,0 +1,46 @@
+use super::media_stream::new_media_stream_track;
+use crate::{
+    media_stream::MediaStreamTrack, rtp_parameters::RtpParameters, RtcError, RtcErrorType,
+};
+use cxx::SharedPtr;
+use webrtc_sys::{rtc_error::ffi::RTCError, rtp_sender as sys_rs};
+
+#[derive(Clone)]
+pub struct RtpSender {
+    pub(crate) sys_handle: SharedPtr<sys_rs::ffi::RtpSender>,
+}
+
+impl RtpSender {
+    pub fn track(&self) -> Option<MediaStreamTrack> {
+        let track_handle = self.sys_handle.track();
+        if track_handle.is_null() {
+            return None;
+        }
+
+        Some(new_media_stream_track(track_handle))
+    }
+
+    pub fn set_track(&self, track: Option<MediaStreamTrack>) -> Result<(), RtcError> {
+        if !self
+            .sys_handle
+            .set_track(track.map_or(SharedPtr::null(), |t| t.sys_handle()))
+        {
+            return Err(RtcError {
+                error_type: RtcErrorType::InvalidState,
+                message: "Failed to set track".to_string(),
+            });
+        }
+
+        Ok(())
+    }
+
+    pub fn parameters(&self) -> RtpParameters {
+        self.sys_handle.get_parameters().into()
+    }
+
+    pub fn set_parameters(&self, parameters: RtpParameters) -> Result<(), RtcError> {
+        self.sys_handle
+            .set_parameters(parameters.into())
+            .map_err(|e| unsafe { RTCError::from(e.what()).into() })
+    }
+}

livekit-webrtc/src/native/rtp_transceiver.rs

@@ -0,0 +1,94 @@
+use crate::imp::rtp_receiver::RtpReceiver;
+use crate::imp::rtp_sender::RtpSender;
+use crate::rtp_parameters::RtpCodecCapability;
+use crate::rtp_receiver;
+use crate::rtp_sender;
+use crate::rtp_transceiver::RtpTransceiverDirection;
+use crate::rtp_transceiver::RtpTransceiverInit;
+use crate::MediaType;
+use crate::RtcError;
+use cxx::SharedPtr;
+use webrtc_sys::rtc_error as sys_err;
+use webrtc_sys::rtp_transceiver as sys_rt;
+use webrtc_sys::webrtc as sys_webrtc;
+
+impl From<sys_webrtc::ffi::RtpTransceiverDirection> for RtpTransceiverDirection {
+    fn from(value: sys_webrtc::ffi::RtpTransceiverDirection) -> Self {
+        match value {
+            sys_webrtc::ffi::RtpTransceiverDirection::SendRecv => Self::SendRecv,
+            sys_webrtc::ffi::RtpTransceiverDirection::SendOnly => Self::SendOnly,
+            sys_webrtc::ffi::RtpTransceiverDirection::RecvOnly => Self::RecvOnly,
+            sys_webrtc::ffi::RtpTransceiverDirection::Inactive => Self::Inactive,
+            _ => panic!("unknown RtpTransceiverDirection"),
+        }
+    }
+}
+
+impl From<RtpTransceiverDirection> for sys_webrtc::ffi::RtpTransceiverDirection {
+    fn from(value: RtpTransceiverDirection) -> Self {
+        match value {
+            RtpTransceiverDirection::SendRecv => Self::SendRecv,
+            RtpTransceiverDirection::SendOnly => Self::SendOnly,
+            RtpTransceiverDirection::RecvOnly => Self::RecvOnly,
+            RtpTransceiverDirection::Inactive => Self::Inactive,
+            _ => panic!("unknown RtpTransceiverDirection"),
+        }
+    }
+}
+
+impl From<RtpTransceiverInit> for sys_rt::ffi::RtpTransceiverInit {
+    fn from(value: RtpTransceiverInit) -> Self {
+        Self {
+            direction: value.direction.into(),
+            stream_ids: value.stream_ids,
+            send_encodings: value.send_encodings.into_iter().map(Into::into).collect(),
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct RtpTransceiver {
+    pub(crate) sys_handle: SharedPtr<sys_rt::ffi::RtpTransceiver>,
+}
+
+impl RtpTransceiver {
+    pub fn mid(&self) -> Option<String> {
+        self.sys_handle.mid().ok()
+    }
+
+    pub fn current_direction(&self) -> Option<RtpTransceiverDirection> {
+        self.sys_handle.current_direction().ok().map(Into::into)
+    }
+
+    pub fn direction(&self) -> RtpTransceiverDirection {
+        self.sys_handle.direction().into()
+    }
+
+    pub fn sender(&self) -> rtp_sender::RtpSender {
+        rtp_sender::RtpSender {
+            handle: RtpSender {
+                sys_handle: self.sys_handle.sender(),
+            },
+        }
+    }
+
+    pub fn receiver(&self) -> rtp_receiver::RtpReceiver {
+        rtp_receiver::RtpReceiver {
+            handle: RtpReceiver {
+                sys_handle: self.sys_handle.receiver(),
+            },
+        }
+    }
+
+    pub fn set_codec_preferences(&self, codecs: Vec<RtpCodecCapability>) -> Result<(), RtcError> {
+        self.sys_handle
+            .set_codec_preferences(codecs.into_iter().map(Into::into).collect())
+            .map_err(|e| unsafe { sys_err::ffi::RTCError::from(e.what()).into() })
+    }
+
+    pub fn stop(&self) -> Result<(), RtcError> {
+        self.sys_handle
+            .stop_standard()
+            .map_err(|e| unsafe { sys_err::ffi::RTCError::from(e.what()).into() })
+    }
+}

livekit-webrtc/src/native/session_description.rs

@@ -0,0 +1,72 @@
+use crate::session_description::{self, SdpParseError, SdpType};
+use cxx::UniquePtr;
+use webrtc_sys::jsep as sys_jsep;
+
+impl From<sys_jsep::ffi::SdpType> for SdpType {
+    fn from(sdp_type: sys_jsep::ffi::SdpType) -> Self {
+        match sdp_type {
+            sys_jsep::ffi::SdpType::Offer => SdpType::Offer,
+            sys_jsep::ffi::SdpType::PrAnswer => SdpType::PrAnswer,
+            sys_jsep::ffi::SdpType::Answer => SdpType::Answer,
+            sys_jsep::ffi::SdpType::Rollback => SdpType::Rollback,
+            _ => panic!("unknown SdpType"),
+        }
+    }
+}
+
+impl From<SdpType> for sys_jsep::ffi::SdpType {
+    fn from(sdp_type: SdpType) -> Self {
+        match sdp_type {
+            SdpType::Offer => sys_jsep::ffi::SdpType::Offer,
+            SdpType::PrAnswer => sys_jsep::ffi::SdpType::PrAnswer,
+            SdpType::Answer => sys_jsep::ffi::SdpType::Answer,
+            SdpType::Rollback => sys_jsep::ffi::SdpType::Rollback,
+        }
+    }
+}
+
+impl From<sys_jsep::ffi::SdpParseError> for SdpParseError {
+    fn from(e: sys_jsep::ffi::SdpParseError) -> Self {
+        Self {
+            line: e.line,
+            description: e.description,
+        }
+    }
+}
+
+pub struct SessionDescription {
+    pub(crate) sys_handle: UniquePtr<sys_jsep::ffi::SessionDescription>,
+}
+
+impl SessionDescription {
+    pub fn parse(
+        sdp: &str,
+        sdp_type: SdpType,
+    ) -> Result<session_description::SessionDescription, SdpParseError> {
+        let res = sys_jsep::ffi::create_session_description(sdp_type.into(), sdp.to_owned());
+        match res {
+            Ok(sys_handle) => Ok(session_description::SessionDescription {
+                handle: SessionDescription { sys_handle },
+            }),
+            Err(e) => Err(unsafe { sys_jsep::ffi::SdpParseError::from(e.what()).into() }),
+        }
+    }
+
+    pub fn sdp_type(&self) -> SdpType {
+        self.sys_handle.sdp_type().into()
+    }
+}
+
+impl ToString for SessionDescription {
+    fn to_string(&self) -> String {
+        self.sys_handle.stringify()
+    }
+}
+
+impl Clone for SessionDescription {
+    fn clone(&self) -> Self {
+        SessionDescription {
+            sys_handle: self.sys_handle.clone(),
+        }
+    }
+}

livekit-webrtc/src/native/video_frame.rs

@@ -0,0 +1,764 @@
+use super::yuv_helper::{self, ConvertError};
+use crate::video_frame::VideoRotation;
+use crate::video_frame::{self as vf, VideoFormatType};
+use cxx::UniquePtr;
+use std::slice;
+use webrtc_sys::video_frame as vf_sys;
+use webrtc_sys::video_frame_buffer as vfb_sys;
+
+/// We don't use vf::VideoFrameBuffer trait for the types inside this module to avoid confusion
+/// because irectly using platform specific types is not valid (e.g user callback)
+/// All the types inside this module are only used internally. For public types, see the top level video_frame.rs
+
+pub fn new_video_frame_buffer(
+    mut sys_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>,
+) -> Box<dyn vf::VideoFrameBuffer + Send + Sync> {
+    unsafe {
+        match sys_handle.buffer_type().into() {
+            vfb_sys::ffi::VideoFrameBufferType::Native => Box::new(vf::native::NativeBuffer {
+                handle: NativeBuffer { sys_handle },
+            }),
+            vfb_sys::ffi::VideoFrameBufferType::I420 => Box::new(vf::I420Buffer {
+                handle: I420Buffer {
+                    sys_handle: sys_handle.pin_mut().get_i420(),
+                },
+            }),
+            vfb_sys::ffi::VideoFrameBufferType::I420A => Box::new(vf::I420ABuffer {
+                handle: I420ABuffer {
+                    sys_handle: sys_handle.pin_mut().get_i420a(),
+                },
+            }),
+            vfb_sys::ffi::VideoFrameBufferType::I422 => Box::new(vf::I422Buffer {
+                handle: I422Buffer {
+                    sys_handle: sys_handle.pin_mut().get_i422(),
+                },
+            }),
+            vfb_sys::ffi::VideoFrameBufferType::I444 => Box::new(vf::I444Buffer {
+                handle: I444Buffer {
+                    sys_handle: sys_handle.pin_mut().get_i444(),
+                },
+            }),
+            vfb_sys::ffi::VideoFrameBufferType::I010 => Box::new(vf::I010Buffer {
+                handle: I010Buffer {
+                    sys_handle: sys_handle.pin_mut().get_i010(),
+                },
+            }),
+            vfb_sys::ffi::VideoFrameBufferType::NV12 => Box::new(vf::NV12Buffer {
+                handle: NV12Buffer {
+                    sys_handle: sys_handle.pin_mut().get_nv12(),
+                },
+            }),
+            _ => unreachable!(),
+        }
+    }
+}
+
+impl From<vf_sys::ffi::VideoRotation> for VideoRotation {
+    fn from(rotation: vf_sys::ffi::VideoRotation) -> Self {
+        match rotation {
+            vf_sys::ffi::VideoRotation::VideoRotation0 => Self::VideoRotation0,
+            vf_sys::ffi::VideoRotation::VideoRotation90 => Self::VideoRotation90,
+            vf_sys::ffi::VideoRotation::VideoRotation180 => Self::VideoRotation180,
+            vf_sys::ffi::VideoRotation::VideoRotation270 => Self::VideoRotation270,
+            _ => panic!("invalid VideoRotation"),
+        }
+    }
+}
+
+impl From<VideoRotation> for vf_sys::ffi::VideoRotation {
+    fn from(rotation: VideoRotation) -> Self {
+        match rotation {
+            VideoRotation::VideoRotation0 => Self::VideoRotation0,
+            VideoRotation::VideoRotation90 => Self::VideoRotation90,
+            VideoRotation::VideoRotation180 => Self::VideoRotation180,
+            VideoRotation::VideoRotation270 => Self::VideoRotation270,
+        }
+    }
+}
+
+macro_rules! recursive_cast {
+    ($ptr:expr $(, $fnc:ident)*) => {
+        {
+            let ptr = $ptr;
+            $(
+                let ptr = vfb_sys::ffi::$fnc(ptr);
+            )*
+            ptr
+        }
+    };
+}
+
+pub struct NativeBuffer {
+    sys_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>,
+}
+
+pub struct I420Buffer {
+    sys_handle: UniquePtr<vfb_sys::ffi::I420Buffer>,
+}
+
+pub struct I420ABuffer {
+    sys_handle: UniquePtr<vfb_sys::ffi::I420ABuffer>,
+}
+
+pub struct I422Buffer {
+    sys_handle: UniquePtr<vfb_sys::ffi::I422Buffer>,
+}
+
+pub struct I444Buffer {
+    sys_handle: UniquePtr<vfb_sys::ffi::I444Buffer>,
+}
+
+pub struct I010Buffer {
+    sys_handle: UniquePtr<vfb_sys::ffi::I010Buffer>,
+}
+
+pub struct NV12Buffer {
+    sys_handle: UniquePtr<vfb_sys::ffi::NV12Buffer>,
+}
+
+macro_rules! impl_to_argb {
+    (I420Buffer [$($variant:ident: $fnc:ident),+], $format:ident, $self:ident, $dst:ident, $dst_stride:ident, $dst_width:ident, $dst_height:ident) => {
+        match $format {
+        $(
+            VideoFormatType::$variant => {
+                let (data_y, data_u, data_v) = $self.data();
+                yuv_helper::$fnc(
+                    data_y,
+                    $self.stride_y(),
+                    data_u,
+                    $self.stride_u(),
+                    data_v,
+                    $self.stride_v(),
+                    $dst,
+                    $dst_stride,
+                    $dst_width,
+                    $dst_height,
+                )
+            }
+        )+
+        }
+    };
+    (I420ABuffer) => {
+        todo!();
+    }
+}
+
+#[allow(unused_unsafe)]
+impl NativeBuffer {
+    pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
+        &*self.sys_handle
+    }
+
+    pub fn width(&self) -> i32 {
+        self.sys_handle.width()
+    }
+
+    pub fn height(&self) -> i32 {
+        self.sys_handle.height()
+    }
+
+    pub fn to_i420(&self) -> I420Buffer {
+        I420Buffer {
+            sys_handle: unsafe { self.sys_handle.to_i420() },
+        }
+    }
+
+    pub fn to_argb(
+        &self,
+        format: VideoFormatType,
+        dst: &mut [u8],
+        dst_stride: i32,
+        dst_width: i32,
+        dst_height: i32,
+    ) -> Result<(), ConvertError> {
+        self.to_i420()
+            .to_argb(format, dst, dst_stride, dst_width, dst_height)
+    }
+}
+
+impl I420Buffer {
+    pub fn new(width: u32, height: u32) -> vf::I420Buffer {
+        vf::I420Buffer {
+            handle: I420Buffer {
+                sys_handle: vfb_sys::ffi::new_i420_buffer(
+                    width.try_into().unwrap(),
+                    height.try_into().unwrap(),
+                ),
+            },
+        }
+    }
+
+    pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
+        unsafe { &*recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb) }
+    }
+
+    pub fn width(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
+            (*ptr).width()
+        }
+    }
+
+    pub fn height(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
+            (*ptr).height()
+        }
+    }
+
+    pub fn chroma_width(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
+            (*ptr).chroma_width()
+        }
+    }
+
+    pub fn chroma_height(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
+            (*ptr).chroma_height()
+        }
+    }
+
+    pub fn stride_y(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
+            (*ptr).stride_y()
+        }
+    }
+
+    pub fn stride_u(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
+            (*ptr).stride_u()
+        }
+    }
+
+    pub fn stride_v(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
+            (*ptr).stride_v()
+        }
+    }
+
+    pub fn to_i420(&self) -> I420Buffer {
+        I420Buffer {
+            sys_handle: unsafe {
+                // We make a copy of the buffer because internally, when calling ToI420()
+                // if the buffer is of type I420, libwebrtc will reuse the same underlying pointer
+                // for the new created type
+                let copy = vfb_sys::ffi::copy_i420_buffer(&self.sys_handle);
+                let ptr = recursive_cast!(&*copy, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
+                (*ptr).to_i420()
+            },
+        }
+    }
+
+    pub fn to_argb(
+        &self,
+        format: VideoFormatType,
+        dst: &mut [u8],
+        dst_stride: i32,
+        dst_width: i32,
+        dst_height: i32,
+    ) -> Result<(), ConvertError> {
+        impl_to_argb!(
+            I420Buffer
+            [
+                ARGB: i420_to_argb,
+                BGRA: i420_to_bgra,
+                ABGR: i420_to_abgr,
+                RGBA: i420_to_rgba
+            ],
+            format, self, dst, dst_stride, dst_width, dst_height
+        )
+    }
+
+    pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8);
+            let chroma_height = (self.height() + 1) / 2;
+            (
+                slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
+                slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * chroma_height) as usize),
+                slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * chroma_height) as usize),
+            )
+        }
+    }
+}
+
+impl I420ABuffer {
+    pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
+        unsafe { &*recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb) }
+    }
+
+    pub fn width(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
+            (*ptr).width()
+        }
+    }
+
+    pub fn height(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
+            (*ptr).height()
+        }
+    }
+
+    pub fn chroma_width(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
+            (*ptr).chroma_width()
+        }
+    }
+
+    pub fn chroma_height(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
+            (*ptr).chroma_height()
+        }
+    }
+
+    pub fn stride_y(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
+            (*ptr).stride_y()
+        }
+    }
+
+    pub fn stride_u(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
+            (*ptr).stride_u()
+        }
+    }
+
+    pub fn stride_v(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
+            (*ptr).stride_v()
+        }
+    }
+
+    pub fn stride_a(&self) -> i32 {
+        self.sys_handle.stride_a()
+    }
+
+    pub fn to_i420(&self) -> I420Buffer {
+        I420Buffer {
+            sys_handle: unsafe {
+                let ptr =
+                    recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
+                (*ptr).to_i420()
+            },
+        }
+    }
+
+    pub fn to_argb(
+        &self,
+        format: VideoFormatType,
+        dst: &mut [u8],
+        dst_stride: i32,
+        dst_width: i32,
+        dst_height: i32,
+    ) -> Result<(), ConvertError> {
+        self.to_i420()
+            .to_argb(format, dst, dst_stride, dst_width, dst_height)
+    }
+
+    pub fn data(&self) -> (&[u8], &[u8], &[u8], Option<&[u8]>) {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8);
+            let chroma_height = (self.height() + 1) / 2;
+            let data_a = self.sys_handle.data_a();
+            let has_data_a = !data_a.is_null();
+            (
+                slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
+                slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * chroma_height) as usize),
+                slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * chroma_height) as usize),
+                has_data_a.then_some(slice::from_raw_parts(
+                    data_a,
+                    (self.stride_a() * self.height()) as usize,
+                )),
+            )
+        }
+    }
+}
+
+impl I422Buffer {
+    pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
+        unsafe { &*recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb) }
+    }
+
+    pub fn width(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
+            (*ptr).width()
+        }
+    }
+
+    pub fn height(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
+            (*ptr).height()
+        }
+    }
+
+    pub fn chroma_width(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
+            (*ptr).chroma_width()
+        }
+    }
+
+    pub fn chroma_height(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
+            (*ptr).chroma_height()
+        }
+    }
+
+    pub fn stride_y(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
+            (*ptr).stride_y()
+        }
+    }
+
+    pub fn stride_u(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
+            (*ptr).stride_u()
+        }
+    }
+
+    pub fn stride_v(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
+            (*ptr).stride_v()
+        }
+    }
+
+    pub fn to_i420(&self) -> I420Buffer {
+        I420Buffer {
+            sys_handle: unsafe {
+                let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
+                (*ptr).to_i420()
+            },
+        }
+    }
+
+    pub fn to_argb(
+        &self,
+        format: VideoFormatType,
+        dst: &mut [u8],
+        dst_stride: i32,
+        dst_width: i32,
+        dst_height: i32,
+    ) -> Result<(), ConvertError> {
+        self.to_i420()
+            .to_argb(format, dst, dst_stride, dst_width, dst_height)
+    }
+
+    pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8);
+            (
+                slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
+                slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * self.height()) as usize),
+                slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * self.height()) as usize),
+            )
+        }
+    }
+}
+impl I444Buffer {
+    pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
+        unsafe { &*recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb) }
+    }
+
+    pub fn width(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
+            (*ptr).width()
+        }
+    }
+
+    pub fn height(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
+            (*ptr).height()
+        }
+    }
+
+    pub fn chroma_width(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
+            (*ptr).chroma_width()
+        }
+    }
+
+    pub fn chroma_height(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
+            (*ptr).chroma_height()
+        }
+    }
+
+    pub fn stride_y(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
+            (*ptr).stride_y()
+        }
+    }
+
+    pub fn stride_u(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
+            (*ptr).stride_u()
+        }
+    }
+
+    pub fn stride_v(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
+            (*ptr).stride_v()
+        }
+    }
+
+    pub fn to_i420(&self) -> I420Buffer {
+        I420Buffer {
+            sys_handle: unsafe {
+                let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
+                (*ptr).to_i420()
+            },
+        }
+    }
+
+    pub fn to_argb(
+        &self,
+        format: VideoFormatType,
+        dst: &mut [u8],
+        dst_stride: i32,
+        dst_width: i32,
+        dst_height: i32,
+    ) -> Result<(), ConvertError> {
+        self.to_i420()
+            .to_argb(format, dst, dst_stride, dst_width, dst_height)
+    }
+
+    pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8);
+            (
+                slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
+                slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * self.height()) as usize),
+                slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * self.height()) as usize),
+            )
+        }
+    }
+}
+
+impl I010Buffer {
+    pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
+        unsafe { &*recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb) }
+    }
+
+    pub fn width(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb);
+            (*ptr).width()
+        }
+    }
+
+    pub fn height(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb);
+            (*ptr).height()
+        }
+    }
+
+    pub fn chroma_width(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
+            (*ptr).chroma_width()
+        }
+    }
+
+    pub fn chroma_height(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
+            (*ptr).chroma_height()
+        }
+    }
+
+    pub fn stride_y(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
+            (*ptr).stride_y()
+        }
+    }
+
+    pub fn stride_u(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
+            (*ptr).stride_u()
+        }
+    }
+
+    pub fn stride_v(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
+            (*ptr).stride_v()
+        }
+    }
+
+    pub fn to_i420(&self) -> I420Buffer {
+        I420Buffer {
+            sys_handle: unsafe {
+                let ptr =
+                    recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb);
+                (*ptr).to_i420()
+            },
+        }
+    }
+
+    pub fn to_argb(
+        &self,
+        format: VideoFormatType,
+        dst: &mut [u8],
+        dst_stride: i32,
+        dst_width: i32,
+        dst_height: i32,
+    ) -> Result<(), ConvertError> {
+        self.to_i420()
+            .to_argb(format, dst, dst_stride, dst_width, dst_height)
+    }
+
+    pub fn data(&self) -> (&[u16], &[u16], &[u16]) {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b);
+            let chroma_height = (self.height() + 1) / 2;
+            (
+                slice::from_raw_parts(
+                    (*ptr).data_y(),
+                    (self.stride_y() * self.height()) as usize / 2,
+                ),
+                slice::from_raw_parts(
+                    (*ptr).data_u(),
+                    (self.stride_u() * chroma_height) as usize / 2,
+                ),
+                slice::from_raw_parts(
+                    (*ptr).data_v(),
+                    (self.stride_v() * chroma_height) as usize / 2,
+                ),
+            )
+        }
+    }
+}
+
+impl NV12Buffer {
+    pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
+        unsafe {
+            &*recursive_cast!(
+                &*self.sys_handle,
+                nv12_to_biyuv8,
+                biyuv8_to_biyuv,
+                biyuv_to_vfb
+            )
+        }
+    }
+
+    pub fn width(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(
+                &*self.sys_handle,
+                nv12_to_biyuv8,
+                biyuv8_to_biyuv,
+                biyuv_to_vfb
+            );
+            (*ptr).width()
+        }
+    }
+
+    pub fn height(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(
+                &*self.sys_handle,
+                nv12_to_biyuv8,
+                biyuv8_to_biyuv,
+                biyuv_to_vfb
+            );
+            (*ptr).height()
+        }
+    }
+
+    pub fn chroma_width(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv);
+            (*ptr).chroma_width()
+        }
+    }
+
+    pub fn chroma_height(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv);
+            (*ptr).chroma_height()
+        }
+    }
+
+    pub fn stride_y(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv);
+            (*ptr).stride_y()
+        }
+    }
+
+    pub fn stride_uv(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv);
+            (*ptr).stride_uv()
+        }
+    }
+
+    pub fn to_i420(&self) -> I420Buffer {
+        I420Buffer {
+            sys_handle: unsafe {
+                let ptr = recursive_cast!(
+                    &*self.sys_handle,
+                    nv12_to_biyuv8,
+                    biyuv8_to_biyuv,
+                    biyuv_to_vfb
+                );
+                (*ptr).to_i420()
+            },
+        }
+    }
+
+    pub fn to_argb(
+        &self,
+        format: VideoFormatType,
+        dst: &mut [u8],
+        dst_stride: i32,
+        dst_width: i32,
+        dst_height: i32,
+    ) -> Result<(), ConvertError> {
+        self.to_i420()
+            .to_argb(format, dst, dst_stride, dst_width, dst_height)
+    }
+
+    pub fn data(&self) -> (&[u8], &[u8]) {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8);
+            let chroma_height = (self.height() + 1) / 2;
+
+            (
+                slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
+                slice::from_raw_parts(
+                    (*ptr).data_uv(),
+                    (self.stride_uv() * chroma_height) as usize,
+                ),
+            )
+        }
+    }
+}

livekit-webrtc/src/native/video_source.rs

@@ -0,0 +1,34 @@
+use crate::video_frame::{VideoFrame, VideoFrameBuffer};
+use cxx::SharedPtr;
+use webrtc_sys::media_stream as ms_sys;
+use webrtc_sys::video_frame as vf_sys;
+
+#[derive(Clone)]
+pub struct NativeVideoSource {
+    sys_handle: SharedPtr<ms_sys::ffi::AdaptedVideoTrackSource>,
+}
+
+impl Default for NativeVideoSource {
+    fn default() -> Self {
+        Self {
+            sys_handle: ms_sys::ffi::new_adapted_video_track_source(),
+        }
+    }
+}
+
+impl NativeVideoSource {
+    pub fn sys_handle(&self) -> SharedPtr<ms_sys::ffi::AdaptedVideoTrackSource> {
+        self.sys_handle.clone()
+    }
+
+    pub fn capture_frame<T: VideoFrameBuffer>(&self, frame: &VideoFrame<T>) {
+        let mut builder = vf_sys::ffi::new_video_frame_builder();
+        builder.pin_mut().set_rotation(frame.rotation.into());
+        builder
+            .pin_mut()
+            .set_video_frame_buffer(frame.buffer.sys_handle());
+
+        let frame = builder.pin_mut().build();
+        self.sys_handle.on_captured_frame(&frame);
+    }
+}

livekit-webrtc/src/native/video_stream.rs

@@ -0,0 +1,84 @@
+use super::video_frame::new_video_frame_buffer;
+use crate::media_stream::RtcVideoTrack;
+use crate::video_frame::{BoxVideoFrame, VideoFrame};
+use cxx::UniquePtr;
+use futures::stream::Stream;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+use tokio::sync::mpsc;
+use webrtc_sys::media_stream as sys_ms;
+
+pub struct NativeVideoStream {
+    native_observer: UniquePtr<sys_ms::ffi::NativeVideoFrameSink>,
+    _observer: Box<VideoTrackObserver>,
+    video_track: RtcVideoTrack,
+    frame_rx: mpsc::UnboundedReceiver<BoxVideoFrame>,
+}
+
+impl NativeVideoStream {
+    pub fn new(video_track: RtcVideoTrack) -> Self {
+        let (frame_tx, frame_rx) = mpsc::unbounded_channel();
+        let mut observer = Box::new(VideoTrackObserver { frame_tx });
+        let mut native_observer = unsafe {
+            sys_ms::ffi::new_native_video_frame_sink(Box::new(sys_ms::VideoFrameSinkWrapper::new(
+                &mut *observer,
+            )))
+        };
+
+        unsafe {
+            sys_ms::ffi::media_to_video(video_track.sys_handle())
+                .add_sink(native_observer.pin_mut());
+        }
+
+        Self {
+            native_observer,
+            _observer: observer,
+            video_track,
+            frame_rx,
+        }
+    }
+
+    pub fn track(&self) -> RtcVideoTrack {
+        self.video_track.clone()
+    }
+
+    pub fn close(&mut self) {
+        self.frame_rx.close();
+        unsafe {
+            sys_ms::ffi::media_to_video(self.video_track.sys_handle())
+                .remove_sink(self.native_observer.pin_mut());
+        }
+    }
+}
+
+impl Drop for NativeVideoStream {
+    fn drop(&mut self) {
+        self.close();
+    }
+}
+
+impl Stream for NativeVideoStream {
+    type Item = BoxVideoFrame;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
+        self.frame_rx.poll_recv(cx)
+    }
+}
+
+struct VideoTrackObserver {
+    frame_tx: mpsc::UnboundedSender<BoxVideoFrame>,
+}
+
+impl sys_ms::VideoFrameSink for VideoTrackObserver {
+    fn on_frame(&self, frame: UniquePtr<webrtc_sys::video_frame::ffi::VideoFrame>) {
+        let _ = self.frame_tx.send(VideoFrame {
+            rotation: frame.rotation().into(),
+            timestamp: frame.timestamp_us(),
+            buffer: new_video_frame_buffer(unsafe { frame.video_frame_buffer() }),
+        });
+    }
+
+    fn on_discarded_frame(&self) {}
+
+    fn on_constraints_changed(&self, _constraints: sys_ms::ffi::VideoTrackSourceConstraints) {}
+}

livekit-webrtc/src/native/yuv_helper.rs

@@ -0,0 +1,183 @@
+use thiserror::Error;
+use webrtc_sys::yuv_helper as yuv_sys;
+
+#[derive(Error, Debug)]
+pub enum ConvertError {
+    #[error("conversion failed: {0}")]
+    Convert(&'static str),
+}
+
+#[inline]
+fn argb_assert_safety(
+    src: &[u8],
+    src_stride: i32,
+    _width: i32,
+    height: i32,
+) -> Result<(), ConvertError> {
+    let min = (src_stride * height) as usize;
+
+    if src.len() < min {
+        return Err(ConvertError::Convert("dst isn't large enough"));
+    }
+
+    Ok(())
+}
+
+#[inline]
+fn i420_assert_safety(
+    src_y: &[u8],
+    src_stride_y: i32,
+    src_u: &[u8],
+    src_stride_u: i32,
+    src_v: &[u8],
+    src_stride_v: i32,
+    _width: i32,
+    height: i32,
+) -> Result<(), ConvertError> {
+    let chroma_height = (height + 1) / 2;
+    let min_y = (src_stride_y * height) as usize;
+    let min_u = (src_stride_u * chroma_height) as usize;
+    let min_v = (src_stride_v * chroma_height) as usize;
+
+    if src_y.len() < min_y {
+        return Err(ConvertError::Convert("src_y isn't large enough"));
+    }
+
+    if src_u.len() < min_u {
+        return Err(ConvertError::Convert("src_u isn't large enough"));
+    }
+
+    if src_v.len() < min_v {
+        return Err(ConvertError::Convert("src_v isn't large enough"));
+    }
+
+    Ok(())
+}
+
+macro_rules! i420_to_x {
+    ($x:ident) => {
+        pub fn $x(
+            src_y: &[u8],
+            src_stride_y: i32,
+            src_u: &[u8],
+            src_stride_u: i32,
+            src_v: &[u8],
+            src_stride_v: i32,
+            dst: &mut [u8],
+            dst_stride: i32,
+            width: i32,
+            height: i32,
+        ) -> Result<(), ConvertError> {
+            argb_assert_safety(dst, dst_stride, width, height)?;
+            i420_assert_safety(
+                src_y,
+                src_stride_y,
+                src_u,
+                src_stride_u,
+                src_v,
+                src_stride_v,
+                width,
+                height,
+            )?;
+
+            unsafe {
+                yuv_sys::ffi::$x(
+                    src_y.as_ptr(),
+                    src_stride_y,
+                    src_u.as_ptr(),
+                    src_stride_u,
+                    src_v.as_ptr(),
+                    src_stride_v,
+                    dst.as_mut_ptr(),
+                    dst_stride,
+                    width,
+                    height,
+                )
+                .unwrap();
+            }
+
+            Ok(())
+        }
+    };
+}
+
+macro_rules! x_to_i420 {
+    ($x:ident) => {
+        pub fn $x(
+            src_argb: &[u8],
+            src_stride_argb: i32,
+            dst_y: &mut [u8],
+            dst_stride_y: i32,
+            dst_u: &mut [u8],
+            dst_stride_u: i32,
+            dst_v: &mut [u8],
+            dst_stride_v: i32,
+            width: i32,
+            height: i32,
+        ) -> Result<(), ConvertError> {
+            argb_assert_safety(src_argb, src_stride_argb, width, height)?;
+            i420_assert_safety(
+                dst_y,
+                dst_stride_y,
+                dst_u,
+                dst_stride_u,
+                dst_v,
+                dst_stride_v,
+                width,
+                height,
+            )?;
+
+            unsafe {
+                yuv_sys::ffi::$x(
+                    src_argb.as_ptr(),
+                    src_stride_argb,
+                    dst_y.as_mut_ptr(),
+                    dst_stride_y,
+                    dst_u.as_mut_ptr(),
+                    dst_stride_u,
+                    dst_v.as_mut_ptr(),
+                    dst_stride_v,
+                    width,
+                    height,
+                )
+                .unwrap();
+            }
+
+            Ok(())
+        }
+    };
+}
+
+pub fn argb_to_rgb24(
+    src_argb: &[u8],
+    src_stride_argb: i32,
+    dst_rgb24: &mut [u8],
+    dst_stride_rgb24: i32,
+    width: i32,
+    height: i32,
+) -> Result<(), ConvertError> {
+    argb_assert_safety(src_argb, src_stride_argb, width, height)?;
+    argb_assert_safety(dst_rgb24, dst_stride_rgb24, width, height)?;
+
+    unsafe {
+        yuv_sys::ffi::argb_to_rgb24(
+            src_argb.as_ptr(),
+            src_stride_argb,
+            dst_rgb24.as_mut_ptr(),
+            dst_stride_rgb24,
+            width,
+            height,
+        )
+        .unwrap();
+    }
+
+    Ok(())
+}
+
+x_to_i420!(argb_to_i420);
+x_to_i420!(abgr_to_i420);
+
+i420_to_x!(i420_to_argb);
+i420_to_x!(i420_to_bgra);
+i420_to_x!(i420_to_abgr);
+i420_to_x!(i420_to_rgba);