From 16ea02075fb60e41f0719a351dce66b3b153aebd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sun, 12 Feb 2023 19:44:04 +0100 Subject: [PATCH] RtpTransceiver bindings & requirements for publishing tracks (#39) - RtpSender - RtpReceiver - RtpTransceiver - RtpParameters - VideoFrameBuilder - Interior mutability on c++ side - Cleanup bindings ( Use #pragma once instead of include guards ) - webrtc-sys/src/* headers are now used in only one place - Avoid confusion between generated headers and our headers - AdaptedVideoTrackSource - Cleanup the workaround with unsupported Vec> - Put everything inside one file --- livekit-ffi/src/server/mod.rs | 2 +- livekit-ffi/src/server/room.rs | 1 - livekit-webrtc/src/jsep.rs | 10 +- livekit-webrtc/src/lib.rs | 5 +- livekit-webrtc/src/media_stream.rs | 211 ++++---- livekit-webrtc/src/peer_connection.rs | 146 ++++-- livekit-webrtc/src/prelude.rs | 6 +- livekit-webrtc/src/rtp_parameters.rs | 462 ++++++++++++++++++ livekit-webrtc/src/rtp_receiver.rs | 49 +- livekit-webrtc/src/rtp_sender.rs | 81 +++ livekit-webrtc/src/rtp_transceiver.rs | 119 ++++- livekit-webrtc/src/video_frame.rs | 74 +++ livekit-webrtc/src/video_frame_buffer.rs | 134 ++++- livekit-webrtc/src/webrtc.rs | 4 + .../src/room/participant/local_participant.rs | 2 + webrtc-sys/build.rs | 7 + webrtc-sys/compile_flags.txt | 1 + webrtc-sys/include/livekit/candidate.h | 12 +- webrtc-sys/include/livekit/data_channel.h | 17 +- webrtc-sys/include/livekit/helper.h | 41 ++ webrtc-sys/include/livekit/jsep.h | 21 +- webrtc-sys/include/livekit/media_stream.h | 119 +++-- .../include/livekit/objc_video_factory.h | 6 +- webrtc-sys/include/livekit/peer_connection.h | 69 ++- .../include/livekit/peer_connection_factory.h | 12 +- webrtc-sys/include/livekit/rtc_error.h | 8 +- webrtc-sys/include/livekit/rtp_parameters.h | 45 ++ webrtc-sys/include/livekit/rtp_receiver.h | 35 +- webrtc-sys/include/livekit/rtp_sender.h | 52 ++ webrtc-sys/include/livekit/rtp_transceiver.h | 58 ++- webrtc-sys/include/livekit/rust_types.h | 38 -- .../include/livekit/video_decoder_factory.h | 7 +- .../include/livekit/video_encoder_factory.h | 7 +- webrtc-sys/include/livekit/video_frame.h | 62 ++- .../include/livekit/video_frame_buffer.h | 171 +++---- webrtc-sys/include/livekit/webrtc.h | 13 +- webrtc-sys/include/livekit/yuv_helper.h | 6 +- webrtc-sys/src/candidate.rs | 2 +- webrtc-sys/src/data_channel.cpp | 6 +- webrtc-sys/src/data_channel.rs | 15 +- webrtc-sys/src/helper.rs | 54 ++ webrtc-sys/src/jsep.cpp | 6 +- webrtc-sys/src/jsep.rs | 29 +- webrtc-sys/src/lib.rs | 3 + webrtc-sys/src/media_stream.cpp | 173 ++++++- webrtc-sys/src/media_stream.rs | 62 ++- webrtc-sys/src/peer_connection.cpp | 85 +++- webrtc-sys/src/peer_connection.rs | 160 +++--- webrtc-sys/src/peer_connection_factory.cpp | 1 - webrtc-sys/src/peer_connection_factory.rs | 6 +- webrtc-sys/src/rtc_error.cpp | 2 +- webrtc-sys/src/rtp_parameters.cpp | 446 +++++++++++++++++ webrtc-sys/src/rtp_parameters.rs | 194 ++++++++ webrtc-sys/src/rtp_receiver.cpp | 39 +- webrtc-sys/src/rtp_receiver.rs | 32 +- webrtc-sys/src/rtp_sender.cpp | 58 +++ webrtc-sys/src/rtp_sender.rs | 33 ++ webrtc-sys/src/rtp_transceiver.cpp | 123 ++++- webrtc-sys/src/rtp_transceiver.rs | 54 +- webrtc-sys/src/video_frame.cpp | 74 +++ webrtc-sys/src/video_frame.rs | 22 +- webrtc-sys/src/video_frame_buffer.cpp | 198 ++++++++ webrtc-sys/src/video_frame_buffer.rs | 2 + webrtc-sys/src/webrtc.rs | 35 +- 64 files changed, 3357 insertions(+), 670 deletions(-) create mode 100644 livekit-webrtc/src/rtp_parameters.rs create mode 100644 livekit-webrtc/src/rtp_sender.rs create mode 100644 webrtc-sys/include/livekit/helper.h create mode 100644 webrtc-sys/include/livekit/rtp_parameters.h create mode 100644 webrtc-sys/include/livekit/rtp_sender.h delete mode 100644 webrtc-sys/include/livekit/rust_types.h create mode 100644 webrtc-sys/src/helper.rs create mode 100644 webrtc-sys/src/rtp_parameters.cpp create mode 100644 webrtc-sys/src/rtp_parameters.rs create mode 100644 webrtc-sys/src/rtp_sender.cpp create mode 100644 webrtc-sys/src/rtp_sender.rs create mode 100644 webrtc-sys/src/video_frame.cpp create mode 100644 webrtc-sys/src/video_frame_buffer.cpp diff --git a/livekit-ffi/src/server/mod.rs b/livekit-ffi/src/server/mod.rs index 27744a3..af9ebbc 100644 --- a/livekit-ffi/src/server/mod.rs +++ b/livekit-ffi/src/server/mod.rs @@ -91,7 +91,7 @@ impl FFIServer { pub async fn close(&self) { // Close all rooms - for (k, (handle, shutdown_tx)) in self.rooms.write().drain() { + for (_, (handle, shutdown_tx)) in self.rooms.write().drain() { let _ = shutdown_tx.send(()); let _ = handle.await; } diff --git a/livekit-ffi/src/server/room.rs b/livekit-ffi/src/server/room.rs index b1d200d..eb773c3 100644 --- a/livekit-ffi/src/server/room.rs +++ b/livekit-ffi/src/server/room.rs @@ -116,4 +116,3 @@ fn on_video_frame(server: &'static FFIServer, track_sid: TrackSid) -> OnFrameHan ); }) } -// diff --git a/livekit-webrtc/src/jsep.rs b/livekit-webrtc/src/jsep.rs index 39be1f1..d9bba01 100644 --- a/livekit-webrtc/src/jsep.rs +++ b/livekit-webrtc/src/jsep.rs @@ -1,13 +1,13 @@ use std::fmt::{Debug, Formatter}; -use cxx::UniquePtr; +use cxx::{SharedPtr, UniquePtr}; pub use sys_jsep::ffi::{SdpParseError, SdpType}; use webrtc_sys::jsep as sys_jsep; -// TODO Maybe we can replace that by a serialized IceCandidateInit +#[derive(Clone)] pub struct IceCandidate { - cxx_handle: UniquePtr, + cxx_handle: SharedPtr, } impl Debug for IceCandidate { @@ -34,11 +34,11 @@ impl IceCandidate { } } - pub(crate) fn new(cxx_handle: UniquePtr) -> Self { + pub(crate) fn new(cxx_handle: SharedPtr) -> Self { Self { cxx_handle } } - pub(crate) fn release(self) -> UniquePtr { + pub(crate) fn release(self) -> SharedPtr { self.cxx_handle } diff --git a/livekit-webrtc/src/lib.rs b/livekit-webrtc/src/lib.rs index f5372a8..1bdbe38 100644 --- a/livekit-webrtc/src/lib.rs +++ b/livekit-webrtc/src/lib.rs @@ -3,12 +3,13 @@ pub mod jsep; pub mod media_stream; pub mod peer_connection; pub mod peer_connection_factory; +pub mod prelude; pub mod rtc_error; +pub mod rtp_parameters; pub mod rtp_receiver; +pub mod rtp_sender; pub mod rtp_transceiver; pub mod video_frame; pub mod video_frame_buffer; pub mod webrtc; pub mod yuv_helper; - -pub mod prelude; diff --git a/livekit-webrtc/src/media_stream.rs b/livekit-webrtc/src/media_stream.rs index 118d527..ec5bd50 100644 --- a/livekit-webrtc/src/media_stream.rs +++ b/livekit-webrtc/src/media_stream.rs @@ -1,7 +1,6 @@ -use cxx::UniquePtr; +use cxx::{SharedPtr, UniquePtr}; use livekit_utils::enum_dispatch; use std::fmt::{Debug, Formatter}; -use std::pin::Pin; use std::sync::{Arc, Mutex}; use webrtc_sys::media_stream as sys_ms; use webrtc_sys::MEDIA_TYPE_VIDEO; @@ -27,19 +26,19 @@ pub enum MediaStreamTrackHandle { } impl MediaStreamTrackHandle { - pub(crate) fn new(cxx_handle: UniquePtr) -> Self { - unsafe { - if cxx_handle.kind() == MEDIA_TYPE_VIDEO { - Self::Video(VideoTrack::new(UniquePtr::from_raw( - sys_ms::ffi::media_to_video(cxx_handle.into_raw()) - as *mut sys_ms::ffi::VideoTrack, - ))) - } else { - Self::Audio(AudioTrack::new(UniquePtr::from_raw( - sys_ms::ffi::media_to_audio(cxx_handle.into_raw()) - as *mut sys_ms::ffi::AudioTrack, - ))) - } + pub(crate) fn new(cxx_handle: SharedPtr) -> Self { + if cxx_handle.kind() == MEDIA_TYPE_VIDEO { + Self::Video(VideoTrack::new(cxx_handle)) + } else { + Self::Audio(AudioTrack::new(cxx_handle)) + } + } + + // TODO(theomonnom): enum_dispatch with visibility support? + pub(crate) fn cxx_handle(&self) -> SharedPtr { + match self { + Self::Video(video) => video.cxx_handle(), + Self::Audio(audio) => audio.cxx_handle(), } } } @@ -67,58 +66,123 @@ impl MediaStreamTrackTrait for MediaStreamTrackHandle { } pub struct AudioTrack { - cxx_handle: Mutex>, + cxx_handle: SharedPtr, } impl AudioTrack { - pub(crate) fn new(cxx_handle: UniquePtr) -> Arc { - Arc::new(Self { - cxx_handle: Mutex::new(cxx_handle), - }) + fn new(cxx_handle: SharedPtr) -> Arc { + Arc::new(Self { cxx_handle }) + } + + pub(crate) fn cxx_handle(&self) -> SharedPtr { + self.cxx_handle.clone() } } pub struct VideoTrack { - cxx_handle: Mutex>, + cxx_handle: SharedPtr, observer: Box, // Keep alive for c++ native_observer: UniquePtr, } +impl VideoTrack { + fn new(cxx_handle: SharedPtr) -> Arc { + let mut observer = Box::new(InternalVideoTrackSink::default()); + + let mut track = unsafe { + Self { + cxx_handle, + native_observer: { + sys_ms::ffi::create_native_video_frame_sink(Box::new( + sys_ms::VideoFrameSinkWrapper::new(&mut *observer), + )) + }, + observer, + } + }; + + unsafe { + (*track.video_handle()).add_sink(track.native_observer.pin_mut()); + } + + Arc::new(track) + } + + pub(crate) fn cxx_handle(&self) -> SharedPtr { + self.cxx_handle.clone() + } + + fn video_handle(&self) -> *const sys_ms::ffi::VideoTrack { + unsafe { sys_ms::ffi::media_to_video(&*self.cxx_handle) } + } + + pub fn set_should_receive(&self, should_receive: bool) { + unsafe { (*self.video_handle()).set_should_receive(should_receive) } + } + + pub fn set_content_hint(&self, hint: ContentHint) { + unsafe { (*self.video_handle()).set_content_hint(hint) } + } + + pub fn should_receive(&self) -> bool { + unsafe { (*self.video_handle()).should_receive() } + } + + pub fn content_hint(&self) -> ContentHint { + unsafe { (*self.video_handle()).content_hint() } + } + + pub fn on_frame(&self, handler: OnFrameHandler) { + *self.observer.on_frame_handler.lock().unwrap() = Some(handler); + } + + pub fn on_discarded_frame(&self, handler: OnDiscardedFrameHandler) { + *self.observer.on_discarded_frame_handler.lock().unwrap() = Some(handler); + } + + pub fn on_constraints_changed(&self, handler: OnConstraintsChangedHandler) { + *self.observer.on_constraints_changed_handler.lock().unwrap() = Some(handler); + } +} + +impl Drop for VideoTrack { + fn drop(&mut self) { + unsafe { + (*self.video_handle()).remove_sink(self.native_observer.pin_mut()); + } + } +} + macro_rules! impl_media_stream_track_trait { - ($x:ty, $cast:ident) => { + ($x:ty) => { impl MediaStreamTrackTrait for $x { fn kind(&self) -> String { - unsafe { (*sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())).kind() } + self.cxx_handle.kind() } fn id(&self) -> String { - unsafe { (*sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())).id() } + self.cxx_handle.id() } fn enabled(&self) -> bool { - unsafe { (*sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())).enabled() } + self.cxx_handle.enabled() } fn set_enabled(&self, enabled: bool) -> bool { - unsafe { - let media = sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap()) - as *mut sys_ms::ffi::MediaStreamTrack; - - Pin::new_unchecked(&mut *media).set_enabled(enabled) - } + self.cxx_handle.set_enabled(enabled) } fn state(&self) -> TrackState { - unsafe { (*sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())).state() } + self.cxx_handle.state() } } }; } -impl_media_stream_track_trait!(VideoTrack, video_to_media); -impl_media_stream_track_trait!(AudioTrack, audio_to_media); +impl_media_stream_track_trait!(VideoTrack); +impl_media_stream_track_trait!(AudioTrack); pub type OnFrameHandler = Box; pub type OnDiscardedFrameHandler = Box; @@ -167,83 +231,8 @@ impl sys_ms::VideoFrameSink for InternalVideoTrackSink { } } -impl VideoTrack { - fn new(cxx_handle: UniquePtr) -> Arc { - let mut observer = Box::new(InternalVideoTrackSink::default()); - - let mut track = unsafe { - Self { - cxx_handle: Mutex::new(cxx_handle), - native_observer: sys_ms::ffi::create_native_video_frame_sink(Box::new( - sys_ms::VideoFrameSinkWrapper::new(&mut *observer), - )), - observer, - } - }; - - unsafe { - track - .cxx_handle - .lock() - .unwrap() - .pin_mut() - .add_sink(track.native_observer.pin_mut()); - } - - Arc::new(track) - } - - pub fn set_should_receive(&self, should_receive: bool) { - self.cxx_handle - .lock() - .unwrap() - .pin_mut() - .set_should_receive(should_receive) - } - - pub fn set_content_hint(&self, hint: ContentHint) { - self.cxx_handle - .lock() - .unwrap() - .pin_mut() - .set_content_hint(hint) - } - - pub fn should_receive(&self) -> bool { - self.cxx_handle.lock().unwrap().should_receive() - } - - pub fn content_hint(&self) -> ContentHint { - self.cxx_handle.lock().unwrap().content_hint() - } - - pub fn on_frame(&self, handler: OnFrameHandler) { - *self.observer.on_frame_handler.lock().unwrap() = Some(handler); - } - - pub fn on_discarded_frame(&self, handler: OnDiscardedFrameHandler) { - *self.observer.on_discarded_frame_handler.lock().unwrap() = Some(handler); - } - - pub fn on_constraints_changed(&self, handler: OnConstraintsChangedHandler) { - *self.observer.on_constraints_changed_handler.lock().unwrap() = Some(handler); - } -} - -impl Drop for VideoTrack { - fn drop(&mut self) { - unsafe { - self.cxx_handle - .lock() - .unwrap() - .pin_mut() - .remove_sink(self.native_observer.pin_mut()); - } - } -} - pub struct MediaStream { - cxx_handle: UniquePtr, + cxx_handle: SharedPtr, } impl Debug for MediaStream { @@ -255,7 +244,7 @@ impl Debug for MediaStream { } impl MediaStream { - pub(crate) fn new(cxx_handle: UniquePtr) -> Self { + pub(crate) fn new(cxx_handle: SharedPtr) -> Self { Self { cxx_handle } } diff --git a/livekit-webrtc/src/peer_connection.rs b/livekit-webrtc/src/peer_connection.rs index c13b2e9..796bc15 100644 --- a/livekit-webrtc/src/peer_connection.rs +++ b/livekit-webrtc/src/peer_connection.rs @@ -1,28 +1,26 @@ +use crate::prelude::*; +use cxx::{SharedPtr, UniquePtr}; +use log::trace; use std::fmt::{Debug, Formatter}; use std::mem::ManuallyDrop; use std::sync::{Arc, Mutex}; - -use cxx::UniquePtr; -use log::trace; use tokio::sync::{mpsc, oneshot}; +use webrtc_sys::candidate as sys_ca; use webrtc_sys::data_channel as sys_dc; use webrtc_sys::jsep as sys_jsep; +use webrtc_sys::media_stream as sys_ms; use webrtc_sys::peer_connection as sys_pc; +use webrtc_sys::rtp_receiver as sys_rr; +use webrtc_sys::rtp_sender as sys_rs; +use webrtc_sys::rtp_transceiver as sys_rt; + pub use webrtc_sys::peer_connection::ffi::IceConnectionState; pub use webrtc_sys::peer_connection::ffi::IceGatheringState; -use webrtc_sys::peer_connection::ffi::NativeCreateSdpObserverHandle; pub use webrtc_sys::peer_connection::ffi::PeerConnectionState; pub use webrtc_sys::peer_connection::ffi::RTCOfferAnswerOptions; pub use webrtc_sys::peer_connection::ffi::SignalingState; -use crate::data_channel::{DataChannel, DataChannelInit}; -use crate::jsep::{IceCandidate, SessionDescription}; -use crate::media_stream::{AudioTrack, MediaStream, VideoTrack}; -use crate::rtc_error::RTCError; -use crate::rtp_receiver::RtpReceiver; -use crate::rtp_transceiver::RtpTransceiver; - pub struct PeerConnection { cxx_handle: UniquePtr, observer: Box, @@ -38,8 +36,6 @@ impl Debug for PeerConnection { .field("signaling_state", &self.signaling_state()) .field("ice_connection_state", &self.ice_connection_state()) .field("ice_gathering_state", &self.ice_gathering_state()) - .field("local_description", &self.local_description()) - .field("remote_description", &self.remote_description()) .finish() } } @@ -58,7 +54,7 @@ impl PeerConnection { } fn create_sdp_observer() -> ( - UniquePtr, + UniquePtr, mpsc::Receiver>, ) { let (tx, rx) = mpsc::channel(1); @@ -81,14 +77,13 @@ impl PeerConnection { } pub async fn create_offer( - &mut self, + &self, options: RTCOfferAnswerOptions, ) -> Result { let (mut native_wrapper, mut rx) = Self::create_sdp_observer(); unsafe { self.cxx_handle - .pin_mut() .create_offer(native_wrapper.pin_mut(), options); } @@ -96,24 +91,20 @@ impl PeerConnection { } pub async fn create_answer( - &mut self, + &self, options: RTCOfferAnswerOptions, ) -> Result { let (mut native_wrapper, mut rx) = Self::create_sdp_observer(); unsafe { self.cxx_handle - .pin_mut() .create_answer(native_wrapper.pin_mut(), options); } rx.recv().await.unwrap() } - pub async fn set_local_description( - &mut self, - desc: SessionDescription, - ) -> Result<(), RTCError> { + pub async fn set_local_description(&self, desc: SessionDescription) -> Result<(), RTCError> { let (tx, rx) = oneshot::channel(); let wrapper = sys_jsep::SetLocalSdpObserverWrapper(ManuallyDrop::new(Box::new(move |error| { @@ -124,17 +115,13 @@ impl PeerConnection { unsafe { self.cxx_handle - .pin_mut() .set_local_description(desc.release(), native_wrapper.pin_mut()); } rx.await.unwrap() } - pub async fn set_remote_description( - &mut self, - desc: SessionDescription, - ) -> Result<(), RTCError> { + pub async fn set_remote_description(&self, desc: SessionDescription) -> Result<(), RTCError> { let (tx, rx) = oneshot::channel(); let wrapper = sys_jsep::SetRemoteSdpObserverWrapper(ManuallyDrop::new(Box::new(move |error| { @@ -145,22 +132,92 @@ impl PeerConnection { unsafe { self.cxx_handle - .pin_mut() .set_remote_description(desc.release(), native_wrapper.pin_mut()); } rx.await.unwrap() } + pub fn add_track( + &self, + track: MediaStreamTrackHandle, + stream_ids: &Vec, + ) -> Result { + let res = self.cxx_handle.add_track(track.cxx_handle(), stream_ids); + match res { + Ok(cxx_handle) => Ok(RtpSender::new(cxx_handle)), + Err(e) => unsafe { Err(RTCError::from(e.what())) }, + } + } + + pub fn remove_track(&self, sender: RtpSender) -> Result<(), RTCError> { + self.cxx_handle + .remove_track(sender.cxx_handle()) + .map_err(|e| unsafe { RTCError::from(e.what()) }) + } + + pub fn add_transceiver( + &self, + track: MediaStreamTrackHandle, + init: RtpTransceiverInit, + ) -> Result { + let res = self + .cxx_handle + .add_transceiver(track.cxx_handle(), init.into()); + + match res { + Ok(cxx_handle) => Ok(RtpTransceiver::new(cxx_handle)), + Err(e) => unsafe { Err(RTCError::from(e.what())) }, + } + } + + pub fn add_transceiver_for_media( + &self, + media_type: MediaType, + init: RtpTransceiverInit, + ) -> Result { + let res = self + .cxx_handle + .add_transceiver_for_media(media_type, init.into()); + + match res { + Ok(cxx_handle) => Ok(RtpTransceiver::new(cxx_handle)), + Err(e) => unsafe { Err(RTCError::from(e.what())) }, + } + } + + pub fn senders(&self) -> Vec { + self.cxx_handle + .get_senders() + .into_iter() + .map(|sender| RtpSender::new(sender.ptr)) + .collect() + } + + pub fn receivers(&self) -> Vec { + self.cxx_handle + .get_receivers() + .into_iter() + .map(|receiver| RtpReceiver::new(receiver.ptr)) + .collect() + } + + pub fn transceivers(&self) -> Vec { + self.cxx_handle + .get_transceivers() + .into_iter() + .map(|transceiver| RtpTransceiver::new(transceiver.ptr)) + .collect() + } + pub fn create_data_channel( - &mut self, + &self, label: &str, init: DataChannelInit, ) -> Result { let native_init = sys_dc::ffi::create_data_channel_init(init.into()); let res = self .cxx_handle - .pin_mut() .create_data_channel(label.to_string(), native_init); match res { @@ -170,7 +227,7 @@ impl PeerConnection { } // TODO(theomonnom) Use IceCandidateInit instead of IceCandidate - pub async fn add_ice_candidate(&mut self, candidate: IceCandidate) -> Result<(), RTCError> { + pub async fn add_ice_candidate(&self, candidate: IceCandidate) -> Result<(), RTCError> { let (tx, rx) = oneshot::channel(); let observer = sys_pc::AddIceCandidateObserverWrapper(ManuallyDrop::new(Box::new(|error| { @@ -180,7 +237,6 @@ impl PeerConnection { let mut native_observer = sys_pc::ffi::create_native_add_ice_candidate_observer(Box::new(observer)); self.cxx_handle - .pin_mut() .add_ice_candidate(candidate.release(), native_observer.pin_mut()); rx.await.unwrap() @@ -415,7 +471,7 @@ impl sys_pc::PeerConnectionObserver for InternalObserver { } } - fn on_add_stream(&self, stream: UniquePtr) { + fn on_add_stream(&self, stream: SharedPtr) { trace!("on_add_stream"); let mut handler = self.on_add_stream_handler.lock().unwrap(); if let Some(f) = handler.as_mut() { @@ -423,7 +479,7 @@ impl sys_pc::PeerConnectionObserver for InternalObserver { } } - fn on_remove_stream(&self, stream: UniquePtr) { + fn on_remove_stream(&self, stream: SharedPtr) { trace!("on_remove_stream"); let mut handler = self.on_remove_stream_handler.lock().unwrap(); if let Some(f) = handler.as_mut() { @@ -431,7 +487,7 @@ impl sys_pc::PeerConnectionObserver for InternalObserver { } } - fn on_data_channel(&self, data_channel: UniquePtr) { + fn on_data_channel(&self, data_channel: UniquePtr) { trace!("on_data_channel"); let mut handler = self.on_data_channel_handler.lock().unwrap(); if let Some(f) = handler.as_mut() { @@ -493,7 +549,7 @@ impl sys_pc::PeerConnectionObserver for InternalObserver { } } - fn on_ice_candidate(&self, candidate: UniquePtr) { + fn on_ice_candidate(&self, candidate: SharedPtr) { trace!("on_ice_candidate"); let mut handler = self.on_ice_candidate_handler.lock().unwrap(); if let Some(f) = handler.as_mut() { @@ -516,10 +572,7 @@ impl sys_pc::PeerConnectionObserver for InternalObserver { } } - fn on_ice_candidates_removed( - &self, - removed: Vec>, - ) { + fn on_ice_candidates_removed(&self, removed: Vec>) { trace!("on_ice_candidates_removed"); let mut handler = self.on_ice_candidates_removed_handler.lock().unwrap(); if let Some(f) = handler.as_mut() { @@ -538,10 +591,7 @@ impl sys_pc::PeerConnectionObserver for InternalObserver { } } - fn on_ice_selected_candidate_pair_changed( - &self, - event: webrtc_sys::peer_connection::ffi::CandidatePairChangeEvent, - ) { + fn on_ice_selected_candidate_pair_changed(&self, event: sys_pc::ffi::CandidatePairChangeEvent) { trace!("on_ice_selected_candidate_pair_changed"); let mut handler = self .on_ice_selected_candidate_pair_changed_handler @@ -554,8 +604,8 @@ impl sys_pc::PeerConnectionObserver for InternalObserver { fn on_add_track( &self, - receiver: UniquePtr, - streams: Vec>, + receiver: SharedPtr, + streams: Vec>, ) { trace!("on_add_track"); let mut handler = self.on_add_track_handler.lock().unwrap(); @@ -565,7 +615,7 @@ impl sys_pc::PeerConnectionObserver for InternalObserver { } } - fn on_track(&self, transceiver: UniquePtr) { + fn on_track(&self, transceiver: SharedPtr) { trace!("on_track"); let mut handler = self.on_track_handler.lock().unwrap(); if let Some(f) = handler.as_mut() { @@ -573,7 +623,7 @@ impl sys_pc::PeerConnectionObserver for InternalObserver { } } - fn on_remove_track(&self, receiver: UniquePtr) { + fn on_remove_track(&self, receiver: SharedPtr) { trace!("on_remove_track"); let mut handler = self.on_remove_track_handler.lock().unwrap(); if let Some(f) = handler.as_mut() { diff --git a/livekit-webrtc/src/prelude.rs b/livekit-webrtc/src/prelude.rs index c58dfbc..65a5ede 100644 --- a/livekit-webrtc/src/prelude.rs +++ b/livekit-webrtc/src/prelude.rs @@ -12,9 +12,11 @@ pub use crate::peer_connection_factory::{ ContinualGatheringPolicy, ICEServer, IceTransportsType, PeerConnectionFactory, RTCConfiguration, }; pub use crate::rtc_error::RTCError; +pub use crate::rtp_parameters::*; pub use crate::rtp_receiver::RtpReceiver; -pub use crate::rtp_transceiver::RtpTransceiver; +pub use crate::rtp_sender::RtpSender; +pub use crate::rtp_transceiver::{RtpTransceiver, RtpTransceiverInit}; pub use crate::video_frame::{VideoFrame, VideoRotation}; pub use crate::video_frame_buffer::*; -pub use crate::webrtc::RTCRuntime; +pub use crate::webrtc::*; pub use crate::yuv_helper::ConvertError; diff --git a/livekit-webrtc/src/rtp_parameters.rs b/livekit-webrtc/src/rtp_parameters.rs new file mode 100644 index 0000000..fff79d7 --- /dev/null +++ b/livekit-webrtc/src/rtp_parameters.rs @@ -0,0 +1,462 @@ +use crate::prelude::*; +use std::collections::HashMap; +use std::vec::Vec; +use webrtc_sys::rtp_parameters as ps_sys; + +// Don't exporting structs here (only enum), cxx doesn't support Option and HashMap +pub use ps_sys::ffi::{ + DegradationPreference, FecMechanism, RtcpFeedbackMessageType, RtcpFeedbackType, + RtpExtensionFilter, +}; + +#[derive(Debug, Clone)] +pub struct RtcpFeedback { + pub feedback_type: RtcpFeedbackType, + pub message_type: Option, +} + +#[derive(Debug, Clone)] +pub struct RtpCodecCapability { + pub mime_type: String, + pub name: String, + pub kind: MediaType, + pub clock_rate: Option, + pub preferred_payload_type: Option, + pub max_ptime: Option, + pub ptime: Option, + pub num_channels: Option, + pub rtcp_feedback: Vec, + pub parameters: HashMap, + pub options: HashMap, + pub max_temporal_layer_extensions: i32, + pub max_spatial_layer_extensions: i32, + pub svc_multi_stream_support: bool, +} + +#[derive(Debug, Clone)] +pub struct RtpHeaderExtensionCapability { + pub uri: String, + pub preferred_id: Option, + pub preferred_encrypt: bool, + pub direction: RtpTransceiverDirection, +} + +#[derive(Debug, Clone)] +pub struct RtpExtension { + pub uri: String, + pub id: i32, + pub encrypt: bool, +} + +#[derive(Debug, Clone)] +pub struct RtpFecParameters { + pub ssrc: Option, + pub mechanism: FecMechanism, +} + +#[derive(Debug, Clone)] +pub struct RtpRtxParameters { + pub ssrc: Option, +} + +#[derive(Debug, Clone)] +pub struct RtpEncodingParameters { + pub ssrc: Option, + pub bitrate_priority: f64, + pub network_priority: Priority, + pub max_bitrate_bps: Option, + pub min_bitrate_bps: Option, + pub max_framerate: Option, + pub num_temporal_layers: Option, + pub scale_resolution_down_by: Option, + pub scalability_mode: Option, + pub active: bool, + pub rid: String, + pub adaptive_ptime: bool, +} + +#[derive(Debug, Clone)] +pub struct RtpCodecParameters { + pub mime_type: String, + pub name: String, + pub kind: MediaType, + pub payload_type: i32, + pub clock_rate: Option, + pub num_channels: Option, + pub max_ptime: Option, + pub ptime: Option, + pub rtcp_feedback: Vec, + pub parameters: HashMap, +} + +#[derive(Debug, Clone)] +pub struct RtpCapabilities { + pub codecs: Vec, + pub header_extensions: Vec, + pub fec: Vec, +} + +#[derive(Debug, Clone)] +pub struct RtcpParameters { + pub ssrc: Option, + pub cname: String, + pub reduced_size: bool, + pub mux: bool, +} + +#[derive(Debug, Clone)] +pub struct RtpParameters { + pub transaction_id: String, + pub mid: String, + pub codecs: Vec, + pub header_extensions: Vec, + pub encodings: Vec, + pub rtcp: RtcpParameters, + pub degradation_preference: Option, +} + +fn into_map(vec: Vec) -> HashMap { + let mut map = HashMap::with_capacity(vec.len()); + for pair in vec { + map.insert(pair.key, pair.value); + } + map +} + +impl From for RtcpFeedback { + fn from(value: ps_sys::ffi::RtcpFeedback) -> Self { + Self { + feedback_type: value.feedback_type, + message_type: value.has_message_type.then_some(value.message_type), + } + } +} + +impl From for RtpCodecCapability { + fn from(value: ps_sys::ffi::RtpCodecCapability) -> Self { + Self { + mime_type: value.mime_type, + name: value.name, + kind: value.kind, + clock_rate: value.has_clock_rate.then_some(value.clock_rate), + preferred_payload_type: value + .has_preferred_payload_type + .then_some(value.preferred_payload_type), + max_ptime: value.has_max_ptime.then_some(value.max_ptime), + ptime: value.has_ptime.then_some(value.ptime), + num_channels: value.has_num_channels.then_some(value.num_channels), + rtcp_feedback: value.rtcp_feedback.into_iter().map(Into::into).collect(), + parameters: into_map(value.parameters), + options: into_map(value.options), + max_temporal_layer_extensions: value.max_temporal_layer_extensions, + max_spatial_layer_extensions: value.max_spatial_layer_extensions, + svc_multi_stream_support: value.svc_multi_stream_support, + } + } +} + +impl From for RtpHeaderExtensionCapability { + fn from(value: ps_sys::ffi::RtpHeaderExtensionCapability) -> Self { + Self { + uri: value.uri, + preferred_id: value.has_preferred_id.then_some(value.preferred_id), + preferred_encrypt: value.preferred_encrypt, + direction: value.direction, + } + } +} + +impl From for RtpExtension { + fn from(value: ps_sys::ffi::RtpExtension) -> Self { + Self { + uri: value.uri, + id: value.id, + encrypt: value.encrypt, + } + } +} + +impl From for RtpFecParameters { + fn from(value: ps_sys::ffi::RtpFecParameters) -> Self { + Self { + ssrc: value.has_ssrc.then_some(value.ssrc), + mechanism: value.mechanism, + } + } +} + +impl From for RtpRtxParameters { + fn from(value: ps_sys::ffi::RtpRtxParameters) -> Self { + Self { + ssrc: value.has_ssrc.then_some(value.ssrc), + } + } +} + +impl From for RtpEncodingParameters { + fn from(value: ps_sys::ffi::RtpEncodingParameters) -> Self { + Self { + ssrc: value.has_ssrc.then_some(value.ssrc), + bitrate_priority: value.bitrate_priority, + network_priority: value.network_priority, + max_bitrate_bps: value.has_max_bitrate_bps.then_some(value.max_bitrate_bps), + min_bitrate_bps: value.has_min_bitrate_bps.then_some(value.min_bitrate_bps), + max_framerate: value.has_max_framerate.then_some(value.max_framerate), + num_temporal_layers: value + .has_num_temporal_layers + .then_some(value.num_temporal_layers), + scale_resolution_down_by: value + .has_scale_resolution_down_by + .then_some(value.scale_resolution_down_by), + scalability_mode: value.has_scalability_mode.then_some(value.scalability_mode), + active: value.active, + rid: value.rid, + adaptive_ptime: value.adaptive_ptime, + } + } +} + +impl From for RtpCodecParameters { + fn from(value: ps_sys::ffi::RtpCodecParameters) -> Self { + Self { + mime_type: value.mime_type, + name: value.name, + kind: value.kind, + payload_type: value.payload_type, + clock_rate: value.has_clock_rate.then_some(value.clock_rate), + num_channels: value.has_num_channels.then_some(value.num_channels), + max_ptime: value.has_max_ptime.then_some(value.max_ptime), + ptime: value.has_ptime.then_some(value.ptime), + rtcp_feedback: value.rtcp_feedback.into_iter().map(Into::into).collect(), + parameters: into_map(value.parameters), + } + } +} + +impl From for RtpCapabilities { + fn from(value: ps_sys::ffi::RtpCapabilities) -> Self { + Self { + codecs: value.codecs.into_iter().map(Into::into).collect(), + header_extensions: value + .header_extensions + .into_iter() + .map(Into::into) + .collect(), + fec: value.fec.into_iter().map(Into::into).collect(), + } + } +} + +impl From for RtcpParameters { + fn from(value: ps_sys::ffi::RtcpParameters) -> Self { + Self { + ssrc: value.has_ssrc.then_some(value.ssrc), + cname: value.cname, + reduced_size: value.reduced_size, + mux: value.mux, + } + } +} + +impl From for RtpParameters { + fn from(value: ps_sys::ffi::RtpParameters) -> Self { + Self { + transaction_id: value.transaction_id, + mid: value.mid, + codecs: value.codecs.into_iter().map(Into::into).collect(), + header_extensions: value + .header_extensions + .into_iter() + .map(Into::into) + .collect(), + encodings: value.encodings.into_iter().map(Into::into).collect(), + rtcp: value.rtcp.into(), + degradation_preference: value + .has_degradation_preference + .then_some(value.degradation_preference), + } + } +} + +// Ignore the value inside unwrap_or for the following implementations + +fn into_vec(map: HashMap) -> Vec { + let mut vec = Vec::with_capacity(map.len()); + for (key, value) in map { + vec.push(ps_sys::ffi::StringKeyValue { key, value }) + } + vec +} + +impl From for ps_sys::ffi::RtcpFeedback { + fn from(value: RtcpFeedback) -> Self { + Self { + feedback_type: value.feedback_type, + has_message_type: value.message_type.is_some(), + message_type: value + .message_type + .unwrap_or(RtcpFeedbackMessageType::GenericNACK), + } + } +} + +impl From for ps_sys::ffi::RtpCodecCapability { + fn from(value: RtpCodecCapability) -> Self { + Self { + mime_type: value.mime_type, + name: value.name, + kind: value.kind, + has_clock_rate: value.clock_rate.is_some(), + clock_rate: value.clock_rate.unwrap_or(0), + has_preferred_payload_type: value.preferred_payload_type.is_some(), + preferred_payload_type: value.preferred_payload_type.unwrap_or(0), + has_max_ptime: value.max_ptime.is_some(), + max_ptime: value.max_ptime.unwrap_or(0), + has_ptime: value.ptime.is_some(), + ptime: value.ptime.unwrap_or(0), + has_num_channels: value.num_channels.is_some(), + num_channels: value.num_channels.unwrap_or(0), + rtcp_feedback: value.rtcp_feedback.into_iter().map(Into::into).collect(), + parameters: into_vec(value.parameters), + options: into_vec(value.options), + max_temporal_layer_extensions: value.max_temporal_layer_extensions, + max_spatial_layer_extensions: value.max_spatial_layer_extensions, + svc_multi_stream_support: value.svc_multi_stream_support, + } + } +} + +impl From for ps_sys::ffi::RtpHeaderExtensionCapability { + fn from(value: RtpHeaderExtensionCapability) -> Self { + Self { + uri: value.uri, + has_preferred_id: value.preferred_id.is_some(), + preferred_id: value.preferred_id.unwrap_or(0), + preferred_encrypt: value.preferred_encrypt, + direction: value.direction, + } + } +} + +impl From for ps_sys::ffi::RtpExtension { + fn from(value: RtpExtension) -> Self { + Self { + uri: value.uri, + id: value.id, + encrypt: value.encrypt, + } + } +} + +impl From for ps_sys::ffi::RtpFecParameters { + fn from(value: RtpFecParameters) -> Self { + Self { + has_ssrc: value.ssrc.is_some(), + ssrc: value.ssrc.unwrap_or(0), + mechanism: value.mechanism, + } + } +} + +impl From for ps_sys::ffi::RtpRtxParameters { + fn from(value: RtpRtxParameters) -> Self { + Self { + has_ssrc: value.ssrc.is_some(), + ssrc: value.ssrc.unwrap_or(0), + } + } +} + +impl From for ps_sys::ffi::RtpEncodingParameters { + fn from(value: RtpEncodingParameters) -> Self { + Self { + has_ssrc: value.ssrc.is_some(), + ssrc: value.ssrc.unwrap_or(0), + bitrate_priority: value.bitrate_priority, + network_priority: value.network_priority, + has_max_bitrate_bps: value.max_bitrate_bps.is_some(), + max_bitrate_bps: value.max_bitrate_bps.unwrap_or(0), + has_min_bitrate_bps: value.min_bitrate_bps.is_some(), + min_bitrate_bps: value.min_bitrate_bps.unwrap_or(0), + has_max_framerate: value.max_framerate.is_some(), + max_framerate: value.max_framerate.unwrap_or(0.0), + has_num_temporal_layers: value.num_temporal_layers.is_some(), + num_temporal_layers: value.num_temporal_layers.unwrap_or(0), + has_scale_resolution_down_by: value.scale_resolution_down_by.is_some(), + scale_resolution_down_by: value.scale_resolution_down_by.unwrap_or(0.0), + has_scalability_mode: value.scalability_mode.is_some(), + scalability_mode: value.scalability_mode.unwrap_or(String::new()), + active: value.active, + rid: value.rid, + adaptive_ptime: value.adaptive_ptime, + } + } +} + +impl From for ps_sys::ffi::RtpCodecParameters { + fn from(value: RtpCodecParameters) -> Self { + Self { + mime_type: value.mime_type, + name: value.name, + kind: value.kind, + payload_type: value.payload_type, + has_clock_rate: value.clock_rate.is_some(), + clock_rate: value.clock_rate.unwrap_or(0), + has_num_channels: value.num_channels.is_some(), + num_channels: value.num_channels.unwrap_or(0), + has_max_ptime: value.max_ptime.is_some(), + max_ptime: value.max_ptime.unwrap_or(0), + has_ptime: value.ptime.is_some(), + ptime: value.ptime.unwrap_or(0), + rtcp_feedback: value.rtcp_feedback.into_iter().map(Into::into).collect(), + parameters: into_vec(value.parameters), + } + } +} + +impl From for ps_sys::ffi::RtpCapabilities { + fn from(value: RtpCapabilities) -> Self { + Self { + codecs: value.codecs.into_iter().map(Into::into).collect(), + header_extensions: value + .header_extensions + .into_iter() + .map(Into::into) + .collect(), + fec: value.fec.into_iter().map(Into::into).collect(), + } + } +} + +impl From for ps_sys::ffi::RtcpParameters { + fn from(value: RtcpParameters) -> Self { + Self { + has_ssrc: value.ssrc.is_some(), + ssrc: value.ssrc.unwrap_or(0), + cname: value.cname, + reduced_size: value.reduced_size, + mux: value.mux, + } + } +} + +impl From for ps_sys::ffi::RtpParameters { + fn from(value: RtpParameters) -> Self { + Self { + transaction_id: value.transaction_id, + mid: value.mid, + codecs: value.codecs.into_iter().map(Into::into).collect(), + header_extensions: value + .header_extensions + .into_iter() + .map(Into::into) + .collect(), + encodings: value.encodings.into_iter().map(Into::into).collect(), + rtcp: value.rtcp.into(), + has_degradation_preference: value.degradation_preference.is_some(), + degradation_preference: value + .degradation_preference + .unwrap_or(DegradationPreference::Balanced), + } + } +} diff --git a/livekit-webrtc/src/rtp_receiver.rs b/livekit-webrtc/src/rtp_receiver.rs index 853c65e..f7d3ad0 100644 --- a/livekit-webrtc/src/rtp_receiver.rs +++ b/livekit-webrtc/src/rtp_receiver.rs @@ -1,26 +1,67 @@ -use crate::media_stream::{MediaStreamTrackHandle, MediaStreamTrackTrait}; -use cxx::UniquePtr; +use crate::media_stream::{MediaStream, MediaStreamTrackHandle}; +use crate::rtp_parameters::RtpParameters; +use cxx::SharedPtr; use std::fmt::{Debug, Formatter}; use webrtc_sys::rtp_receiver as sys_rec; +use webrtc_sys::webrtc as sys_webrtc; +pub use sys_webrtc::ffi::MediaType; + +#[derive(Clone)] pub struct RtpReceiver { - cxx_handle: UniquePtr, + cxx_handle: SharedPtr, } impl Debug for RtpReceiver { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { f.debug_struct("RtpReceiver") .field("track", &self.track()) + .field("media_type", &self.media_type()) + .field("id", &self.id()) .finish() } } impl RtpReceiver { - pub(crate) fn new(cxx_handle: UniquePtr) -> Self { + pub(crate) fn new(cxx_handle: SharedPtr) -> Self { Self { cxx_handle } } + pub(crate) fn cxx_handle(&self) -> SharedPtr { + self.cxx_handle.clone() + } + pub fn track(&self) -> MediaStreamTrackHandle { MediaStreamTrackHandle::new(self.cxx_handle.track()) } + + pub fn stream_ids(&self) -> Vec { + self.cxx_handle.stream_ids() + } + + pub fn streams(&self) -> Vec { + let ptrs = self.cxx_handle.streams(); + let mut vec = Vec::with_capacity(ptrs.len()); + for stream in ptrs { + vec.push(MediaStream::new(stream.ptr)); + } + vec + } + + pub fn media_type(&self) -> MediaType { + self.cxx_handle.media_type() + } + + pub fn id(&self) -> String { + self.cxx_handle.id() + } + + pub fn parameters(&self) -> RtpParameters { + self.cxx_handle.get_parameters().into() + } + + pub fn set_jitter_buffer_minimum_delay(&self, delay_seconds: Option) { + self.cxx_handle + .set_jitter_buffer_minimum_delay(delay_seconds.is_some(), delay_seconds.unwrap_or(0.0)); + } } diff --git a/livekit-webrtc/src/rtp_sender.rs b/livekit-webrtc/src/rtp_sender.rs new file mode 100644 index 0000000..0c5eb66 --- /dev/null +++ b/livekit-webrtc/src/rtp_sender.rs @@ -0,0 +1,81 @@ +use crate::media_stream::{MediaStream, MediaStreamTrackHandle}; +use crate::prelude::*; +use crate::rtp_parameters::{RtpEncodingParameters, RtpParameters}; +use cxx::SharedPtr; +use std::fmt::{Debug, Formatter}; +use webrtc_sys::rtp_sender as sys_rs; +use webrtc_sys::webrtc as sys_webrtc; + +pub use sys_webrtc::ffi::MediaType; + +#[derive(Clone)] +pub struct RtpSender { + cxx_handle: SharedPtr, +} + +impl Debug for RtpSender { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + f.debug_struct("RtpSender") + .field("track", &self.track()) + .field("media_type", &self.media_type()) + .field("ssrc", &self.ssrc()) + .field("id", &self.id()) + .finish() + } +} + +impl RtpSender { + pub(crate) fn new(cxx_handle: SharedPtr) -> Self { + Self { cxx_handle } + } + + pub(crate) fn cxx_handle(&self) -> SharedPtr { + self.cxx_handle.clone() + } + + pub fn set_track(&self, track: MediaStreamTrackHandle) -> bool { + self.cxx_handle.set_track(track.cxx_handle()) + } + + pub fn track(&self) -> MediaStreamTrackHandle { + MediaStreamTrackHandle::new(self.cxx_handle.track()) + } + + pub fn ssrc(&self) -> u32 { + self.cxx_handle.ssrc() + } + + pub fn media_type(&self) -> MediaType { + self.cxx_handle.media_type() + } + + pub fn id(&self) -> String { + self.cxx_handle.id() + } + + pub fn stream_ids(&self) -> Vec { + self.cxx_handle.stream_ids() + } + + pub fn set_streams(&self, stream_ids: &Vec) { + self.cxx_handle.set_streams(stream_ids); + } + + pub fn init_send_encodings(&self) -> Vec { + self.cxx_handle + .init_send_encodings() + .into_iter() + .map(Into::into) + .collect() + } + + pub fn parameters(&self) -> RtpParameters { + self.cxx_handle.get_parameters().into() + } + + pub fn set_parameters(&self, params: RtpParameters) -> Result<(), RTCError> { + self.cxx_handle + .set_parameters(params.into()) + .map_err(|e| unsafe { RTCError::from(e.what()) }) + } +} diff --git a/livekit-webrtc/src/rtp_transceiver.rs b/livekit-webrtc/src/rtp_transceiver.rs index bb7598c..d6f6853 100644 --- a/livekit-webrtc/src/rtp_transceiver.rs +++ b/livekit-webrtc/src/rtp_transceiver.rs @@ -1,2 +1,119 @@ +use crate::prelude::*; +use cxx::SharedPtr; +use std::fmt::{Debug, Formatter}; +use webrtc_sys::rtp_transceiver as sys_rt; + #[derive(Debug)] -pub struct RtpTransceiver {} +pub struct RtpTransceiverInit { + pub direction: RtpTransceiverDirection, + pub stream_ids: Vec, + pub send_encodings: Vec, +} + +impl From for sys_rt::ffi::RtpTransceiverInit { + fn from(value: RtpTransceiverInit) -> Self { + Self { + direction: value.direction, + stream_ids: value.stream_ids, + send_encodings: value.send_encodings.into_iter().map(Into::into).collect() + } + } +} + +#[derive(Clone)] +pub struct RtpTransceiver { + cxx_handle: SharedPtr, +} + +impl Debug for RtpTransceiver { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + f.debug_struct("RtpTransceiver") + .field("media_type", &self.media_type()) + .field("mid", &self.mid()) + .field("direction", &self.direction()) + .field("stopped", &self.stopped()) + .field("stopping", &self.stopping()) + .finish() + } +} + +impl RtpTransceiver { + pub(crate) fn new(cxx_handle: SharedPtr) -> Self { + Self { cxx_handle } + } + + pub(crate) fn cxx_handle(&self) -> SharedPtr { + self.cxx_handle.clone() + } + + pub fn media_type(&self) -> MediaType { + self.cxx_handle.media_type() + } + + pub fn mid(&self) -> Option { + self.cxx_handle.mid().ok() + } + + pub fn sender(&self) -> RtpSender { + RtpSender::new(self.cxx_handle.sender()) + } + + pub fn receiver(&self) -> RtpReceiver { + RtpReceiver::new(self.cxx_handle.receiver()) + } + + pub fn stopped(&self) -> bool { + self.cxx_handle.stopped() + } + + pub fn stopping(&self) -> bool { + self.cxx_handle.stopping() + } + + pub fn direction(&self) -> RtpTransceiverDirection { + self.cxx_handle.direction() + } + + pub fn set_direction(&self, direction: RtpTransceiverDirection) -> Result<(), RTCError> { + self.cxx_handle.set_direction(direction) + .map_err(|e| unsafe { RTCError::from(e.what()) }) + } + + pub fn current_direction(&self) -> Option { + self.cxx_handle.current_direction().ok() + } + + pub fn fired_direction(&self) -> Option { + self.cxx_handle.fired_direction().ok() + } + + pub fn stop_standard(&self) -> Result<(), RTCError> { + self.cxx_handle.stop_standard() + .map_err(|e| unsafe { RTCError::from(e.what()) }) + } + + pub fn set_codec_preferences(&self, codecs: Vec) -> Result<(), RTCError> { + let ffi_codecs = codecs.into_iter().map(Into::into).collect(); + self.cxx_handle.set_codec_preferences(ffi_codecs) + .map_err(|e| unsafe { RTCError::from(e.what()) }) + } + + pub fn codec_preferences(&self) -> Vec { + self.cxx_handle.codec_preferences().into_iter().map(Into::into).collect() + } + + pub fn header_extensions_to_offer(&self) -> Vec { + self.cxx_handle.header_extensions_to_offer().into_iter().map(Into::into).collect() + } + + pub fn header_extensions_negotiated(&self) -> Vec { + self.cxx_handle.header_extensions_negotiated().into_iter().map(Into::into).collect() + } + + pub fn set_offered_rtp_header_extensions(&self, headers: Vec) -> Result<(), RTCError> { + let ffi_headers = headers.into_iter().map(Into::into).collect(); + self.cxx_handle.set_offered_rtp_header_extensions(ffi_headers) + .map_err(|e| unsafe { RTCError::from(e.what()) }) + } + +} diff --git a/livekit-webrtc/src/video_frame.rs b/livekit-webrtc/src/video_frame.rs index 47261fb..9ef0329 100644 --- a/livekit-webrtc/src/video_frame.rs +++ b/livekit-webrtc/src/video_frame.rs @@ -1,5 +1,6 @@ use crate::video_frame_buffer::VideoFrameBuffer; use cxx::UniquePtr; +use std::fmt::{Debug, Formatter}; use webrtc_sys::video_frame as vf_sys; #[derive(Debug)] @@ -22,10 +23,33 @@ impl From for VideoRotation { } } +impl From for vf_sys::ffi::VideoRotation { + fn from(rotation: VideoRotation) -> Self { + match rotation { + VideoRotation::VideoRotation0 => Self::VideoRotation0, + VideoRotation::VideoRotation90 => Self::VideoRotation90, + VideoRotation::VideoRotation180 => Self::VideoRotation180, + VideoRotation::VideoRotation270 => Self::VideoRotation270, + } + } +} + pub struct VideoFrame { cxx_handle: UniquePtr, } +impl Debug for VideoFrame { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + f.debug_struct("VideoFrame") + .field("width", &self.width()) + .field("height", &self.height()) + .field("id", &self.id()) + .field("rotation", &self.rotation()) + .field("timestamp", &self.timestamp()) + .finish() + } +} + impl VideoFrame { pub(crate) fn new(cxx_handle: UniquePtr) -> Self { Self { cxx_handle } @@ -74,4 +98,54 @@ impl VideoFrame { pub(crate) unsafe fn video_frame_buffer(&self) -> VideoFrameBuffer { VideoFrameBuffer::new(self.cxx_handle.video_frame_buffer()) } + + pub fn builder() -> VideoFrameBuilder { + VideoFrameBuilder::default() + } +} + +pub struct VideoFrameBuilder { + cxx_handle: UniquePtr, +} + +impl Debug for VideoFrameBuilder { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + f.debug_struct("VideoFrameBuilder").finish() + } +} + +impl Default for VideoFrameBuilder { + fn default() -> Self { + Self { + cxx_handle: vf_sys::ffi::create_video_frame_builder(), + } + } +} + +impl VideoFrameBuilder { + pub fn set_video_frame_buffer(mut self, buffer: VideoFrameBuffer) -> Self { + self.cxx_handle + .pin_mut() + .set_video_frame_buffer(buffer.release()); + self + } + + pub fn set_timestamp_us(mut self, ts_us: i64) -> Self { + self.cxx_handle.pin_mut().set_timestamp_us(ts_us); + self + } + + pub fn set_rotation(mut self, rotation: VideoRotation) -> Self { + self.cxx_handle.pin_mut().set_rotation(rotation.into()); + self + } + + pub fn set_id(mut self, id: u16) -> Self { + self.cxx_handle.pin_mut().set_id(id); + self + } + + pub fn build(mut self) -> VideoFrame { + VideoFrame::new(self.cxx_handle.pin_mut().build()) + } } diff --git a/livekit-webrtc/src/video_frame_buffer.rs b/livekit-webrtc/src/video_frame_buffer.rs index d873b54..d3d1659 100644 --- a/livekit-webrtc/src/video_frame_buffer.rs +++ b/livekit-webrtc/src/video_frame_buffer.rs @@ -6,6 +6,18 @@ use webrtc_sys::video_frame_buffer as vfb_sys; use crate::yuv_helper::{self, ConvertError}; +macro_rules! recursive_cast { + ($ptr:expr $(, $fnc:ident)*) => { + { + let ptr = $ptr; + $( + let ptr = unsafe { vfb_sys::ffi::$fnc(ptr) }; + )* + ptr + } + }; +} + #[derive(Debug)] pub enum VideoFrameBufferType { Native, @@ -94,29 +106,74 @@ impl VideoFrameBuffer { pub(crate) fn new(mut cxx_handle: UniquePtr) -> Self { unsafe { match cxx_handle.buffer_type().into() { - VideoFrameBufferType::Native => Self::Native(NativeBuffer::new(cxx_handle)), + VideoFrameBufferType::Native => Self::Native(NativeBuffer::from(cxx_handle)), VideoFrameBufferType::I420 => { - Self::I420(I420Buffer::new(cxx_handle.pin_mut().get_i420())) + Self::I420(I420Buffer::from(cxx_handle.pin_mut().get_i420())) } VideoFrameBufferType::I420A => { - Self::I420A(I420ABuffer::new(cxx_handle.pin_mut().get_i420a())) + Self::I420A(I420ABuffer::from(cxx_handle.pin_mut().get_i420a())) } VideoFrameBufferType::I422 => { - Self::I422(I422Buffer::new(cxx_handle.pin_mut().get_i422())) + Self::I422(I422Buffer::from(cxx_handle.pin_mut().get_i422())) } VideoFrameBufferType::I444 => { - Self::I444(I444Buffer::new(cxx_handle.pin_mut().get_i444())) + Self::I444(I444Buffer::from(cxx_handle.pin_mut().get_i444())) } VideoFrameBufferType::I010 => { - Self::I010(I010Buffer::new(cxx_handle.pin_mut().get_i010())) + Self::I010(I010Buffer::from(cxx_handle.pin_mut().get_i010())) } VideoFrameBufferType::NV12 => { - Self::NV12(NV12Buffer::new(cxx_handle.pin_mut().get_nv12())) + Self::NV12(NV12Buffer::from(cxx_handle.pin_mut().get_nv12())) } } } } + #[allow(unused_unsafe)] + pub(crate) fn release(self) -> UniquePtr { + unsafe { + match self { + VideoFrameBuffer::Native(native) => native.release(), + VideoFrameBuffer::I420(i420) => UniquePtr::from_raw(recursive_cast!( + i420.release().into_raw(), + i420_to_yuv8, + yuv8_to_yuv, + yuv_to_vfb + ) as *mut _), + VideoFrameBuffer::I420A(i420a) => UniquePtr::from_raw(recursive_cast!( + i420a.release().into_raw(), + i420a_to_yuv8, + yuv8_to_yuv, + yuv_to_vfb + ) as *mut _), + VideoFrameBuffer::I422(i422) => UniquePtr::from_raw(recursive_cast!( + i422.release().into_raw(), + i422_to_yuv8, + yuv8_to_yuv, + yuv_to_vfb + ) as *mut _), + VideoFrameBuffer::I444(i444) => UniquePtr::from_raw(recursive_cast!( + i444.release().into_raw(), + i444_to_yuv8, + yuv8_to_yuv, + yuv_to_vfb + ) as *mut _), + VideoFrameBuffer::I010(i010) => UniquePtr::from_raw(recursive_cast!( + i010.release().into_raw(), + i010_to_yuv16b, + yuv16b_to_yuv, + yuv_to_vfb + ) as *mut _), + VideoFrameBuffer::NV12(nv12) => UniquePtr::from_raw(recursive_cast!( + nv12.release().into_raw(), + nv12_to_biyuv8, + biyuv8_to_biyuv, + biyuv_to_vfb + ) as *mut _), + } + } + } + pub fn to_argb( &self, format: VideoFormatType, @@ -195,18 +252,6 @@ impl VideoFrameBufferTrait for VideoFrameBuffer { ); } -macro_rules! recursive_cast { - ($ptr:expr $(, $fnc:ident)*) => { - { - let ptr = $ptr; - $( - let ptr = unsafe { vfb_sys::ffi::$fnc(ptr) }; - )* - ptr - } - }; -} - macro_rules! impl_video_frame_buffer { ($x:ty $(, $cast:ident)*) => { @@ -241,7 +286,7 @@ macro_rules! impl_video_frame_buffer { as *mut vfb_sys::ffi::VideoFrameBuffer; unsafe { - I420Buffer::new(Pin::new_unchecked(&mut *ptr).to_i420()) + I420Buffer::from(Pin::new_unchecked(&mut *ptr).to_i420()) } } } @@ -456,43 +501,78 @@ impl_biyuv_buffer!(NV12Buffer, nv12_to_biyuv8, biyuv8_to_biyuv); impl_biyuv8_buffer!(NV12Buffer, nv12_to_biyuv8); impl NativeBuffer { - fn new(cxx_handle: UniquePtr) -> Self { + fn from(cxx_handle: UniquePtr) -> Self { Self { cxx_handle } } + + pub(crate) fn release(self) -> UniquePtr { + self.cxx_handle + } } impl I420Buffer { - fn new(cxx_handle: UniquePtr) -> Self { + pub fn new(width: u32, height: u32) -> Self { + Self::from(vfb_sys::ffi::create_i420_buffer( + width as i32, + height as i32, + )) + } + + fn from(cxx_handle: UniquePtr) -> Self { Self { cxx_handle } } + + pub(crate) fn release(self) -> UniquePtr { + self.cxx_handle + } } impl I420ABuffer { - fn new(cxx_handle: UniquePtr) -> Self { + fn from(cxx_handle: UniquePtr) -> Self { Self { cxx_handle } } + + pub(crate) fn release(self) -> UniquePtr { + self.cxx_handle + } } impl I422Buffer { - fn new(cxx_handle: UniquePtr) -> Self { + fn from(cxx_handle: UniquePtr) -> Self { Self { cxx_handle } } + + pub(crate) fn release(self) -> UniquePtr { + self.cxx_handle + } } impl I444Buffer { - fn new(cxx_handle: UniquePtr) -> Self { + fn from(cxx_handle: UniquePtr) -> Self { Self { cxx_handle } } + + pub(crate) fn release(self) -> UniquePtr { + self.cxx_handle + } } impl I010Buffer { - fn new(cxx_handle: UniquePtr) -> Self { + fn from(cxx_handle: UniquePtr) -> Self { Self { cxx_handle } } + + pub(crate) fn release(self) -> UniquePtr { + self.cxx_handle + } } impl NV12Buffer { - fn new(cxx_handle: UniquePtr) -> Self { + fn from(cxx_handle: UniquePtr) -> Self { Self { cxx_handle } } + + pub(crate) fn release(self) -> UniquePtr { + self.cxx_handle + } } diff --git a/livekit-webrtc/src/webrtc.rs b/livekit-webrtc/src/webrtc.rs index 7803dd7..483f5c7 100644 --- a/livekit-webrtc/src/webrtc.rs +++ b/livekit-webrtc/src/webrtc.rs @@ -2,6 +2,10 @@ use cxx::SharedPtr; use webrtc_sys::webrtc as sys_rtc; +pub use sys_rtc::ffi::MediaType; +pub use sys_rtc::ffi::Priority; +pub use sys_rtc::ffi::RtpTransceiverDirection; + #[derive(Clone)] pub struct RTCRuntime { cxx_handle: SharedPtr, diff --git a/livekit/src/room/participant/local_participant.rs b/livekit/src/room/participant/local_participant.rs index 8527be7..bde5c00 100644 --- a/livekit/src/room/participant/local_participant.rs +++ b/livekit/src/room/participant/local_participant.rs @@ -31,6 +31,8 @@ impl LocalParticipant { } } + pub async fn publish_track(&self) {} + pub async fn publish_data( &self, data: &[u8], diff --git a/webrtc-sys/build.rs b/webrtc-sys/build.rs index 36e084b..3c56e31 100644 --- a/webrtc-sys/build.rs +++ b/webrtc-sys/build.rs @@ -137,6 +137,8 @@ fn main() { "src/data_channel.rs", "src/jsep.rs", "src/candidate.rs", + "src/rtp_parameters.rs", + "src/rtp_sender.rs", "src/rtp_receiver.rs", "src/rtp_transceiver.rs", "src/rtc_error.rs", @@ -144,6 +146,7 @@ fn main() { "src/video_frame.rs", "src/video_frame_buffer.rs", "src/yuv_helper.rs", + "src/helper.rs", ]); builder.file("src/peer_connection.cpp"); @@ -153,9 +156,13 @@ fn main() { builder.file("src/jsep.cpp"); builder.file("src/candidate.cpp"); builder.file("src/rtp_receiver.cpp"); + builder.file("src/rtp_sender.cpp"); builder.file("src/rtp_transceiver.cpp"); + builder.file("src/rtp_parameters.cpp"); builder.file("src/rtc_error.cpp"); builder.file("src/webrtc.cpp"); + builder.file("src/video_frame.cpp"); + builder.file("src/video_frame_buffer.cpp"); builder.file("src/video_encoder_factory.cpp"); builder.file("src/video_decoder_factory.cpp"); diff --git a/webrtc-sys/compile_flags.txt b/webrtc-sys/compile_flags.txt index 8c067f7..c8ce65a 100644 --- a/webrtc-sys/compile_flags.txt +++ b/webrtc-sys/compile_flags.txt @@ -4,6 +4,7 @@ -Ilibwebrtc/src -Ilibwebrtc/src/third_party/abseil-cpp -Ilibwebrtc/src/third_party/libc++ +-Ilibwebrtc/src/third_party/libyuv/include -I../target/cxxbridge -DWEBRTC_MAC -DWEBRTC_POSIX diff --git a/webrtc-sys/include/livekit/candidate.h b/webrtc-sys/include/livekit/candidate.h index 83749c1..3d1b6ea 100644 --- a/webrtc-sys/include/livekit/candidate.h +++ b/webrtc-sys/include/livekit/candidate.h @@ -2,13 +2,17 @@ // Created by Théo Monnom on 01/09/2022. // -#ifndef CLIENT_SDK_NATIVE_CANDIDATE_H -#define CLIENT_SDK_NATIVE_CANDIDATE_H +#pragma once #include #include "api/candidate.h" +namespace livekit { +class Candidate; +} +#include "webrtc-sys/src/candidate.rs.h" + // cricket::Candidate namespace livekit { @@ -20,10 +24,8 @@ class Candidate { cricket::Candidate candidate_; }; -static std::unique_ptr _unique_candidate() { +static std::shared_ptr _shared_candidate() { return nullptr; } } // namespace livekit - -#endif // CLIENT_SDK_NATIVE_CANDIDATE_H diff --git a/webrtc-sys/include/livekit/data_channel.h b/webrtc-sys/include/livekit/data_channel.h index 6fcc6df..48a4f0f 100644 --- a/webrtc-sys/include/livekit/data_channel.h +++ b/webrtc-sys/include/livekit/data_channel.h @@ -2,19 +2,22 @@ // Created by Théo Monnom on 01/09/2022. // -#ifndef CLIENT_SDK_NATIVE_DATA_CHANNEL_H -#define CLIENT_SDK_NATIVE_DATA_CHANNEL_H +#pragma once #include #include "api/data_channel_interface.h" +#include "livekit/webrtc.h" #include "rust/cxx.h" -#include "rust_types.h" -#include "webrtc.h" namespace livekit { +class DataChannel; using NativeDataChannelInit = webrtc::DataChannelInit; class NativeDataChannelObserver; +} // namespace livekit +#include "webrtc-sys/src/data_channel.rs.h" + +namespace livekit { class DataChannel { public: @@ -22,8 +25,8 @@ class DataChannel { std::shared_ptr rtc_runtime, rtc::scoped_refptr data_channel); - void register_observer(NativeDataChannelObserver& observer); - void unregister_observer(); + void register_observer(NativeDataChannelObserver& observer) const; + void unregister_observer() const; bool send(const DataBuffer& buffer) const; rust::String label() const; DataState state() const; @@ -57,5 +60,3 @@ class NativeDataChannelObserver : public webrtc::DataChannelObserver { std::unique_ptr create_native_data_channel_observer( rust::Box observer); } // namespace livekit - -#endif // CLIENT_SDK_NATIVE_DATA_CHANNEL_H diff --git a/webrtc-sys/include/livekit/helper.h b/webrtc-sys/include/livekit/helper.h new file mode 100644 index 0000000..5e544b2 --- /dev/null +++ b/webrtc-sys/include/livekit/helper.h @@ -0,0 +1,41 @@ +#pragma once + +#include "rust/cxx.h" + +namespace livekit { +class MediaStream; +class AudioTrack; +class VideoTrack; +class Candidate; +class RtpSender; +class RtpReceiver; +class RtpTransceiver; +} // namespace livekit +#include "webrtc-sys/src/helper.rs.h" + +namespace livekit { + +// Impl not needed +static rust::Vec _vec_media_stream_ptr() { + throw; +} +static rust::Vec _vec_candidate_ptr() { + throw; +} +static rust::Vec _vec_audio_track_ptr() { + throw; +} +static rust::Vec _vec_video_track_ptr() { + throw; +} +static rust::Vec _vec_rtp_sender_ptr() { + throw; +} +static rust::Vec _vec_rtp_receiver_ptr() { + throw; +} +static rust::Vec _vec_rtp_transceiver_ptr() { + throw; +} + +} // namespace livekit diff --git a/webrtc-sys/include/livekit/jsep.h b/webrtc-sys/include/livekit/jsep.h index fa2e03a..708ff9d 100644 --- a/webrtc-sys/include/livekit/jsep.h +++ b/webrtc-sys/include/livekit/jsep.h @@ -2,15 +2,24 @@ // Created by Théo Monnom on 01/09/2022. // -#ifndef CLIENT_SDK_NATIVE_JSEP_H -#define CLIENT_SDK_NATIVE_JSEP_H +#pragma once #include #include "api/jsep.h" #include "api/ref_counted_base.h" +#include "api/set_local_description_observer_interface.h" +#include "api/set_remote_description_observer_interface.h" #include "rust/cxx.h" -#include "rust_types.h" + +namespace livekit { +class IceCandidate; +class SessionDescription; +struct NativeCreateSdpObserverHandle; +struct NativeSetLocalSdpObserverHandle; +struct NativeSetRemoteSdpObserverHandle; +}; // namespace livekit +#include "webrtc-sys/src/jsep.rs.h" namespace livekit { @@ -31,11 +40,11 @@ class IceCandidate { std::unique_ptr ice_candidate_; }; -std::unique_ptr create_ice_candidate(rust::String sdp_mid, +std::shared_ptr create_ice_candidate(rust::String sdp_mid, int sdp_mline_index, rust::String sdp); -static std::unique_ptr _unique_ice_candidate() { +static std::shared_ptr _shared_ice_candidate() { return nullptr; // Ignore } @@ -126,5 +135,3 @@ std::unique_ptr create_native_set_remote_sdp_observer( rust::Box observer); } // namespace livekit - -#endif // CLIENT_SDK_NATIVE_JSEP_H diff --git a/webrtc-sys/include/livekit/media_stream.h b/webrtc-sys/include/livekit/media_stream.h index 603940a..4432349 100644 --- a/webrtc-sys/include/livekit/media_stream.h +++ b/webrtc-sys/include/livekit/media_stream.h @@ -2,78 +2,92 @@ // Created by Théo Monnom on 31/08/2022. // -#ifndef CLIENT_SDK_NATIVE_MEDIA_STREAM_INTERFACE_H -#define CLIENT_SDK_NATIVE_MEDIA_STREAM_INTERFACE_H +#pragma once #include #include "api/media_stream_interface.h" -#include "livekit/rust_types.h" +#include "api/video/video_frame.h" +#include "livekit/helper.h" +#include "livekit/video_frame.h" +#include "media/base/adapted_video_track_source.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/timestamp_aligner.h" #include "rust/cxx.h" namespace livekit { - +class MediaStream; +class MediaStreamTrack; +class VideoTrack; +class AudioTrack; class NativeVideoFrameSink; +class AdaptedVideoTrackSource; +} // namespace livekit +#include "webrtc-sys/src/media_stream.rs.h" + +namespace livekit { class MediaStream { public: explicit MediaStream(rtc::scoped_refptr stream); rust::String id() const; + rust::Vec get_video_tracks() const; + rust::Vec get_audio_tracks() const; + + std::shared_ptr find_audio_track(rust::String track_id) const; + std::shared_ptr find_video_track(rust::String track_id) const; + + bool add_audio_track(std::shared_ptr audio_track) const; + bool add_video_track(std::shared_ptr video_track) const; + bool remove_audio_track(std::shared_ptr audio_track) const; + bool remove_video_track(std::shared_ptr video_track) const; private: rtc::scoped_refptr media_stream_; }; -static std::unique_ptr _unique_media_stream() { - return nullptr; // Ignore -} - class MediaStreamTrack { protected: explicit MediaStreamTrack( rtc::scoped_refptr track); public: - static std::unique_ptr from( + static std::shared_ptr from( rtc::scoped_refptr track); rust::String kind() const; rust::String id() const; bool enabled() const; - bool set_enabled(bool enable); + bool set_enabled(bool enable) const; TrackState state() const; + rtc::scoped_refptr get() const { + return track_; + } + protected: rtc::scoped_refptr track_; }; -static std::unique_ptr _unique_media_stream_track() { - return nullptr; // Ignore -} - class AudioTrack : public MediaStreamTrack { public: explicit AudioTrack(rtc::scoped_refptr track); }; -static std::unique_ptr _unique_audio_track() { - return nullptr; // Ignore -} - class VideoTrack : public MediaStreamTrack { public: explicit VideoTrack(rtc::scoped_refptr track); - void add_sink(NativeVideoFrameSink& sink); - void remove_sink(NativeVideoFrameSink& sink); + void add_sink(NativeVideoFrameSink& sink) const; + void remove_sink(NativeVideoFrameSink& sink) const; - void set_should_receive(bool should_receive); + void set_should_receive(bool should_receive) const; bool should_receive() const; ContentHint content_hint() const; - void set_content_hint(ContentHint hint); + void set_content_hint(ContentHint hint) const; private: webrtc::VideoTrackInterface* track() const { @@ -81,10 +95,6 @@ class VideoTrack : public MediaStreamTrack { } }; -static std::unique_ptr _unique_video_track() { - return nullptr; // Ignore -} - class NativeVideoFrameSink : public rtc::VideoSinkInterface { public: @@ -102,22 +112,61 @@ class NativeVideoFrameSink std::unique_ptr create_native_video_frame_sink( rust::Box observer); -static const MediaStreamTrack* video_to_media(const VideoTrack* track) { - return track; -} +// Native impl of the WebRTC interface +class NativeVideoTrackSource : public rtc::AdaptedVideoTrackSource { + public: + NativeVideoTrackSource(); + ~NativeVideoTrackSource() override; -static const MediaStreamTrack* audio_to_media(const AudioTrack* track) { - return track; -} + bool is_screencast() const override; + absl::optional needs_denoising() const override; + webrtc::MediaSourceInterface::SourceState state() const override; + bool remote() const override; + + bool on_captured_frame(const webrtc::VideoFrame& frame); + + private: + webrtc::Mutex mutex_; + rtc::TimestampAligner timestamp_aligner_; +}; + +class AdaptedVideoTrackSource { + public: + AdaptedVideoTrackSource(rtc::scoped_refptr source); + + bool on_captured_frame(std::unique_ptr frame) + const; // frames pushed from Rust (+interior mutability) + + rtc::scoped_refptr get() const; + + private: + rtc::scoped_refptr source_; +}; + +std::unique_ptr create_adapted_video_track_source(); static const VideoTrack* media_to_video(const MediaStreamTrack* track) { return static_cast(track); } static const AudioTrack* media_to_audio(const MediaStreamTrack* track) { - return static_cast(track); + return static_cast(track); +} + +static std::shared_ptr _shared_media_stream_track() { + return nullptr; // Ignore +} + +static std::shared_ptr _shared_audio_track() { + return nullptr; // Ignore +} + +static std::shared_ptr _shared_video_track() { + return nullptr; // Ignore +} + +static std::shared_ptr _shared_media_stream() { + return nullptr; // Ignore } } // namespace livekit - -#endif // CLIENT_SDK_NATIVE_MEDIA_STREAM_INTERFACE_H diff --git a/webrtc-sys/include/livekit/objc_video_factory.h b/webrtc-sys/include/livekit/objc_video_factory.h index 2a49213..9576b2e 100644 --- a/webrtc-sys/include/livekit/objc_video_factory.h +++ b/webrtc-sys/include/livekit/objc_video_factory.h @@ -1,7 +1,7 @@ -#ifndef OBJC_VIDEO_FACTORY_H -#define OBJC_VIDEO_FACTORY_H +#pragma once #include + #include "api/video_codecs/video_decoder_factory.h" #include "api/video_codecs/video_encoder_factory.h" @@ -11,5 +11,3 @@ std::unique_ptr CreateObjCVideoEncoderFactory(); std::unique_ptr CreateObjCVideoDecoderFactory(); } // namespace livekit - -#endif // OBJC_VIDEO_FACTORY_H diff --git a/webrtc-sys/include/livekit/peer_connection.h b/webrtc-sys/include/livekit/peer_connection.h index e60c202..639ad0f 100644 --- a/webrtc-sys/include/livekit/peer_connection.h +++ b/webrtc-sys/include/livekit/peer_connection.h @@ -2,20 +2,30 @@ // Created by Théo Monnom on 30/08/2022. // -#ifndef CLIENT_SDK_NATIVE_PEER_CONNECTION_H -#define CLIENT_SDK_NATIVE_PEER_CONNECTION_H +#pragma once #include #include "api/peer_connection_interface.h" -#include "data_channel.h" -#include "jsep.h" +#include "livekit/data_channel.h" +#include "livekit/helper.h" +#include "livekit/jsep.h" +#include "livekit/media_stream.h" +#include "livekit/rtp_receiver.h" +#include "livekit/rtp_sender.h" +#include "livekit/rtp_transceiver.h" +#include "livekit/webrtc.h" #include "rust/cxx.h" -#include "rust_types.h" -#include "webrtc.h" namespace livekit { class NativeAddIceCandidateObserver; +class PeerConnection; +class NativeAddIceCandidateObserver; +class NativePeerConnectionObserver; +} // namespace livekit +#include "webrtc-sys/src/peer_connection.rs.h" + +namespace livekit { class PeerConnection { public: @@ -24,23 +34,54 @@ class PeerConnection { rtc::scoped_refptr peer_connection); void create_offer(NativeCreateSdpObserverHandle& observer, - RTCOfferAnswerOptions options); + RTCOfferAnswerOptions options) const; + void create_answer(NativeCreateSdpObserverHandle& observer, - RTCOfferAnswerOptions options); + RTCOfferAnswerOptions options) const; + void set_local_description(std::unique_ptr desc, - NativeSetLocalSdpObserverHandle& observer); + NativeSetLocalSdpObserverHandle& observer) const; + void set_remote_description(std::unique_ptr desc, - NativeSetRemoteSdpObserverHandle& observer); + NativeSetRemoteSdpObserverHandle& observer) const; + std::unique_ptr create_data_channel( rust::String label, - std::unique_ptr init); - void add_ice_candidate(std::unique_ptr candidate, - NativeAddIceCandidateObserver& observer); + std::unique_ptr init) const; + + void add_ice_candidate(std::shared_ptr candidate, + NativeAddIceCandidateObserver& observer) const; + + std::shared_ptr add_track( + std::shared_ptr track, + const rust::Vec& stream_ids) const; + + void remove_track(std::shared_ptr sender) const; + + std::shared_ptr add_transceiver( + std::shared_ptr track, + RtpTransceiverInit init) const; + + std::shared_ptr add_transceiver_for_media( + MediaType media_type, + RtpTransceiverInit init) const; + + rust::Vec get_senders() const; + + rust::Vec get_receivers() const; + + rust::Vec get_transceivers() const; + std::unique_ptr local_description() const; + std::unique_ptr remote_description() const; + SignalingState signaling_state() const; + IceGatheringState ice_gathering_state() const; + IceConnectionState ice_connection_state() const; + void close(); private: @@ -140,5 +181,3 @@ create_native_peer_connection_observer( std::shared_ptr rtc_runtime, rust::Box observer); } // namespace livekit - -#endif // CLIENT_SDK_NATIVE_PEER_CONNECTION_H diff --git a/webrtc-sys/include/livekit/peer_connection_factory.h b/webrtc-sys/include/livekit/peer_connection_factory.h index 729c280..1a9cf78 100644 --- a/webrtc-sys/include/livekit/peer_connection_factory.h +++ b/webrtc-sys/include/livekit/peer_connection_factory.h @@ -2,18 +2,22 @@ // Created by Théo Monnom on 03/08/2022. // -#ifndef PEER_CONNECTION_FACTORY_H -#define PEER_CONNECTION_FACTORY_H +#pragma once #include "api/peer_connection_interface.h" #include "peer_connection.h" -#include "rust_types.h" #include "webrtc.h" namespace livekit { using NativeRTCConfiguration = webrtc::PeerConnectionInterface::RTCConfiguration; +class PeerConnectionFactory; +} // namespace livekit +#include "webrtc-sys/src/peer_connection_factory.rs.h" + +namespace livekit { + class PeerConnectionFactory { public: explicit PeerConnectionFactory(std::shared_ptr rtc_runtime); @@ -33,5 +37,3 @@ std::unique_ptr create_peer_connection_factory( std::unique_ptr create_rtc_configuration( RTCConfiguration conf); } // namespace livekit - -#endif // PEER_CONNECTION_FACTORY_H diff --git a/webrtc-sys/include/livekit/rtc_error.h b/webrtc-sys/include/livekit/rtc_error.h index 88df94c..2fe5d60 100644 --- a/webrtc-sys/include/livekit/rtc_error.h +++ b/webrtc-sys/include/livekit/rtc_error.h @@ -2,13 +2,11 @@ // Created by theom on 04/09/2022. // -#ifndef CLIENT_SDK_NATIVE_RTC_ERROR_H -#define CLIENT_SDK_NATIVE_RTC_ERROR_H +#pragma once #include "api/rtc_error.h" -#include "webrtc-sys/src/rtc_error.rs.h" #include "rust/cxx.h" -#include "rust_types.h" +#include "webrtc-sys/src/rtc_error.rs.h" namespace livekit { @@ -22,5 +20,3 @@ void throw_error(); #endif } // namespace livekit - -#endif // CLIENT_SDK_NATIVE_RTC_ERROR_H diff --git a/webrtc-sys/include/livekit/rtp_parameters.h b/webrtc-sys/include/livekit/rtp_parameters.h new file mode 100644 index 0000000..027c557 --- /dev/null +++ b/webrtc-sys/include/livekit/rtp_parameters.h @@ -0,0 +1,45 @@ +#pragma once + +#include + +#include "api/media_types.h" +#include "api/priority.h" +#include "api/rtp_parameters.h" +#include "api/rtp_transceiver_direction.h" +#include "webrtc-sys/src/rtp_parameters.rs.h" + +namespace livekit { + +webrtc::RtcpFeedback to_native_rtcp_feedback(RtcpFeedback feedback); +webrtc::RtpCodecCapability to_native_rtp_codec_capability( + RtpCodecCapability capability); +webrtc::RtpHeaderExtensionCapability to_native_rtp_header_extension_capability( + RtpHeaderExtensionCapability header); +webrtc::RtpExtension to_native_rtp_extension(RtpExtension ext); +webrtc::RtpFecParameters to_rtp_fec_parameters(RtpFecParameters fec); +webrtc::RtpRtxParameters to_rtp_rtx_parameters(RtpRtxParameters rtx); +webrtc::RtpEncodingParameters to_native_rtp_encoding_paramters( + RtpEncodingParameters parameters); +webrtc::RtpCodecParameters to_native_rtp_codec_parameters( + RtpCodecParameters params); +webrtc::RtpCapabilities to_rtp_capabilities(RtpCapabilities capabilities); +webrtc::RtcpParameters to_native_rtcp_paramaters(RtcpParameters params); +webrtc::RtpParameters to_native_rtp_parameters(RtpParameters params); + +RtcpFeedback to_rust_rtcp_feedback(webrtc::RtcpFeedback feedback); +RtpCodecCapability to_rust_rtp_codec_capability( + webrtc::RtpCodecCapability capability); +RtpHeaderExtensionCapability to_rust_rtp_header_extension_capability( + webrtc::RtpHeaderExtensionCapability header); +RtpExtension to_rust_rtp_extension(webrtc::RtpExtension ext); +RtpFecParameters to_rust_rtp_fec_parameters(webrtc::RtpFecParameters fec); +RtpRtxParameters to_rust_rtp_rtx_parameters(webrtc::RtpRtxParameters param); +RtpEncodingParameters to_rust_rtp_encoding_parameters( + webrtc::RtpEncodingParameters params); +RtpCodecParameters to_rust_rtp_codec_parameters( + webrtc::RtpCodecParameters params); +RtpCapabilities to_rust_rtp_capabilities(webrtc::RtpCapabilities capabilities); +RtcpParameters to_rust_rtcp_parameters(webrtc::RtcpParameters params); +RtpParameters to_rust_rtp_parameters(webrtc::RtpParameters params); + +} // namespace livekit diff --git a/webrtc-sys/include/livekit/rtp_receiver.h b/webrtc-sys/include/livekit/rtp_receiver.h index f541cf5..332989e 100644 --- a/webrtc-sys/include/livekit/rtp_receiver.h +++ b/webrtc-sys/include/livekit/rtp_receiver.h @@ -2,30 +2,51 @@ // Created by Théo Monnom on 01/09/2022. // -#ifndef CLIENT_SDK_NATIVE_RTP_RECEIVER_H -#define CLIENT_SDK_NATIVE_RTP_RECEIVER_H +#pragma once #include #include "api/rtp_receiver_interface.h" +#include "livekit/helper.h" #include "livekit/media_stream.h" +#include "livekit/rtp_parameters.h" +#include "rust/cxx.h" +namespace livekit { +class RtpReceiver; +} +#include "webrtc-sys/src/rtp_receiver.rs.h" namespace livekit { +// TODO(theomonnom): Implement RtpReceiverObserverInterface? +// TODO(theomonnom): RtpSource +// TODO(theomonnom): FrameTransformer & FrameDecryptor interface class RtpReceiver { public: explicit RtpReceiver( rtc::scoped_refptr receiver); - std::unique_ptr track() const; + std::shared_ptr track() const; + + rust::Vec stream_ids() const; + rust::Vec streams() const; + + MediaType media_type() const; + rust::String id() const; + + RtpParameters get_parameters() const; + + // bool set_parameters(RtpParameters parameters) const; // Seems unsupported + + void set_jitter_buffer_minimum_delay(bool is_some, + double delay_seconds) const; private: rtc::scoped_refptr receiver_; }; -static std::unique_ptr _unique_rtp_receiver() { - return nullptr; // Ignore +static std::shared_ptr _shared_rtp_receiver() { + return nullptr; } -} // namespace livekit -#endif // CLIENT_SDK_NATIVE_RTP_RECEIVER_H +} // namespace livekit diff --git a/webrtc-sys/include/livekit/rtp_sender.h b/webrtc-sys/include/livekit/rtp_sender.h new file mode 100644 index 0000000..d143a8e --- /dev/null +++ b/webrtc-sys/include/livekit/rtp_sender.h @@ -0,0 +1,52 @@ +#pragma once + +#include + +#include "api/rtp_sender_interface.h" +#include "livekit/media_stream.h" +#include "livekit/rtc_error.h" +#include "livekit/rtp_parameters.h" +#include "rust/cxx.h" + +namespace livekit { +class RtpSender; +} +#include "webrtc-sys/src/rtp_sender.rs.h" + +namespace livekit { + +// TODO(theomonnom): FrameTransformer & FrameEncryptor interface +class RtpSender { + public: + explicit RtpSender(rtc::scoped_refptr sender); + + bool set_track(std::shared_ptr track) const; + + std::shared_ptr track() const; + + uint32_t ssrc() const; + + MediaType media_type() const; + + rust::String id() const; + + rust::Vec stream_ids() const; + + void set_streams(const rust::Vec& stream_ids) const; + + rust::Vec init_send_encodings() const; + + RtpParameters get_parameters() const; + + void set_parameters(RtpParameters params) const; + + rtc::scoped_refptr get() const { return sender_; } + + private: + rtc::scoped_refptr sender_; +}; + +static std::shared_ptr _shared_rtp_sender() { + return nullptr; // Ignore +} +} // namespace livekit diff --git a/webrtc-sys/include/livekit/rtp_transceiver.h b/webrtc-sys/include/livekit/rtp_transceiver.h index caa95e1..491407f 100644 --- a/webrtc-sys/include/livekit/rtp_transceiver.h +++ b/webrtc-sys/include/livekit/rtp_transceiver.h @@ -2,27 +2,73 @@ // Created by Théo Monnom on 02/09/2022. // -#ifndef CLIENT_SDK_NATIVE_RTP_TRANSCEIVER_H -#define CLIENT_SDK_NATIVE_RTP_TRANSCEIVER_H +#pragma once #include +#include "api/rtp_parameters.h" +#include "api/rtp_transceiver_direction.h" #include "api/rtp_transceiver_interface.h" +#include "livekit/rtc_error.h" +#include "livekit/rtp_parameters.h" +#include "livekit/rtp_receiver.h" +#include "livekit/rtp_sender.h" +#include "rust/cxx.h" namespace livekit { +class RtpTransceiver; +} +#include "webrtc-sys/src/rtp_transceiver.rs.h" + +namespace livekit { + +webrtc::RtpTransceiverInit to_native_rtp_transceiver_init( + RtpTransceiverInit init); class RtpTransceiver { public: explicit RtpTransceiver( rtc::scoped_refptr transceiver); + MediaType media_type() const; + + rust::String mid() const; + + std::shared_ptr sender() const; + + std::shared_ptr receiver() const; + + bool stopped() const; + + bool stopping() const; + + RtpTransceiverDirection direction() const; + + void set_direction(RtpTransceiverDirection direction) const; + + RtpTransceiverDirection current_direction() const; + + RtpTransceiverDirection fired_direction() const; + + void stop_standard() const; + + void set_codec_preferences(rust::Vec codecs) const; + + rust::Vec codec_preferences() const; + + rust::Vec header_extensions_to_offer() const; + + rust::Vec header_extensions_negotiated() const; + + void set_offered_rtp_header_extensions( + rust::Vec header_extensions_to_offer) const; + private: rtc::scoped_refptr transceiver_; }; -static std::unique_ptr _unique_rtp_transceiver() { - return nullptr; // Ignore +static std::shared_ptr _shared_rtp_transceiver() { + return nullptr; } -} // namespace livekit -#endif // CLIENT_SDK_NATIVE_RTP_TRANSCEIVER_H +} // namespace livekit diff --git a/webrtc-sys/include/livekit/rust_types.h b/webrtc-sys/include/livekit/rust_types.h deleted file mode 100644 index f342734..0000000 --- a/webrtc-sys/include/livekit/rust_types.h +++ /dev/null @@ -1,38 +0,0 @@ -// -// Created by Théo Monnom on 30/08/2022. -// - -#ifndef RUST_TYPES_H -#define RUST_TYPES_H - -#include "api/peer_connection_interface.h" - -namespace livekit { -struct RTCConfiguration; -struct PeerConnectionObserverWrapper; -struct CreateSdpObserverWrapper; -struct SetLocalSdpObserverWrapper; -struct SetRemoteSdpObserverWrapper; -struct DataChannelObserverWrapper; -struct AddIceCandidateObserverWrapper; -struct VideoFrameSinkWrapper; - -// Shared types -enum class PeerConnectionState; -enum class SignalingState; -enum class IceConnectionState; -enum class IceGatheringState; -enum class SdpType; -enum class DataState; -enum class TrackState; -enum class ContentHint; -enum class VideoRotation; -enum class VideoFrameBufferType; -struct SdpParseError; -struct RTCOfferAnswerOptions; -struct RTCError; -struct DataChannelInit; -struct DataBuffer; -} // namespace livekit - -#endif // RUST_TYPES_H diff --git a/webrtc-sys/include/livekit/video_decoder_factory.h b/webrtc-sys/include/livekit/video_decoder_factory.h index 36e804d..cc1cc94 100644 --- a/webrtc-sys/include/livekit/video_decoder_factory.h +++ b/webrtc-sys/include/livekit/video_decoder_factory.h @@ -1,8 +1,7 @@ -#ifndef VIDEO_DECODER_FACTORY_H -#define VIDEO_DECODER_FACTORY_H +#pragma once -#include "api/video_codecs/video_decoder_factory.h" #include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_decoder_factory.h" namespace livekit { class VideoDecoderFactory : public webrtc::VideoDecoderFactory { @@ -18,5 +17,3 @@ class VideoDecoderFactory : public webrtc::VideoDecoderFactory { std::vector> factories_; }; } // namespace livekit - -#endif // VIDEO_DECODER_FACTORY_H diff --git a/webrtc-sys/include/livekit/video_encoder_factory.h b/webrtc-sys/include/livekit/video_encoder_factory.h index f0e5d2c..f1d089d 100644 --- a/webrtc-sys/include/livekit/video_encoder_factory.h +++ b/webrtc-sys/include/livekit/video_encoder_factory.h @@ -1,8 +1,7 @@ -#ifndef VIDEO_ENCODER_FACTORY_H -#define VIDEO_ENCODER_FACTORY_H +#pragma once -#include "api/video_codecs/video_encoder_factory.h" #include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_factory.h" namespace livekit { class VideoEncoderFactory : public webrtc::VideoEncoderFactory { @@ -18,5 +17,3 @@ class VideoEncoderFactory : public webrtc::VideoEncoderFactory { std::vector> factories_; }; } // namespace livekit - -#endif // VIDEO_ENCODER_FACTORY_H diff --git a/webrtc-sys/include/livekit/video_frame.h b/webrtc-sys/include/livekit/video_frame.h index 6a247a5..d853d88 100644 --- a/webrtc-sys/include/livekit/video_frame.h +++ b/webrtc-sys/include/livekit/video_frame.h @@ -2,47 +2,59 @@ // Created by theom on 14/11/2022. // -#ifndef LIVEKIT_WEBRTC_VIDEO_FRAME_H -#define LIVEKIT_WEBRTC_VIDEO_FRAME_H +#pragma once #include "api/video/video_frame.h" -#include "livekit/rust_types.h" #include "livekit/video_frame_buffer.h" +#include "rtc_base/checks.h" + +namespace livekit { +class VideoFrame; +class VideoFrameBuilder; +} // namespace livekit +#include "webrtc-sys/src/video_frame.rs.h" namespace livekit { class VideoFrame { public: - explicit VideoFrame(const webrtc::VideoFrame& frame) - : frame_(std::move(frame)) {} + explicit VideoFrame(const webrtc::VideoFrame& frame); - int width() const { return frame_.width(); } - int height() const { return frame_.height(); } - uint32_t size() const { return frame_.size(); } - uint16_t id() const { return frame_.id(); } - int64_t timestamp_us() const { return frame_.timestamp_us(); } - int64_t ntp_time_ms() const { return frame_.ntp_time_ms(); } - uint32_t transport_frame_id() const { return frame_.transport_frame_id(); } - uint32_t timestamp() const { return frame_.timestamp(); } + int width() const; + int height() const; + uint32_t size() const; + uint16_t id() const; + int64_t timestamp_us() const; + int64_t ntp_time_ms() const; + uint32_t transport_frame_id() const; + uint32_t timestamp() const; - VideoRotation rotation() const { - return static_cast(frame_.rotation()); - } + VideoRotation rotation() const; + std::unique_ptr video_frame_buffer() const; - // TODO(theomonnom) This shouldn't create a new shared_ptr at each call - std::unique_ptr video_frame_buffer() const { - return std::make_unique(frame_.video_frame_buffer()); - } + webrtc::VideoFrame get() const; private: webrtc::VideoFrame frame_; }; -static std::unique_ptr _unique_video_frame() { - return nullptr; // Ignore -} +// Allow to create VideoFrames from Rust, +// the builder pattern will be redone in Rust +class VideoFrameBuilder { + public: + VideoFrameBuilder() = default; + // TODO(theomonnom): other setters? + void set_video_frame_buffer(std::unique_ptr buffer); + void set_timestamp_us(int64_t timestamp_us); + void set_rotation(VideoRotation rotation); + void set_id(uint16_t id); + std::unique_ptr build(); + + private: + webrtc::VideoFrame::Builder builder_; +}; + +std::unique_ptr create_video_frame_builder(); } // namespace livekit - -#endif // LIVEKIT_WEBRTC_VIDEO_FRAME_H diff --git a/webrtc-sys/include/livekit/video_frame_buffer.h b/webrtc-sys/include/livekit/video_frame_buffer.h index 723ec7e..e0780fa 100644 --- a/webrtc-sys/include/livekit/video_frame_buffer.h +++ b/webrtc-sys/include/livekit/video_frame_buffer.h @@ -2,75 +2,50 @@ // Created by theom on 14/11/2022. // -#ifndef LIVEKIT_WEBRTC_VIDEO_FRAME_BUFFER_H -#define LIVEKIT_WEBRTC_VIDEO_FRAME_BUFFER_H +#pragma once #include +#include "api/video/i420_buffer.h" #include "api/video/video_frame_buffer.h" -#include "rust_types.h" namespace livekit { - +class VideoFrameBuffer; +class PlanarYuvBuffer; +class PlanarYuv8Buffer; +class PlanarYuv16BBuffer; +class BiplanarYuvBuffer; +class BiplanarYuv8Buffer; class I420Buffer; class I420ABuffer; class I422Buffer; class I444Buffer; class I010Buffer; class NV12Buffer; +} // namespace livekit +#include "webrtc-sys/src/video_frame_buffer.rs.h" + +namespace livekit { class VideoFrameBuffer { public: - explicit VideoFrameBuffer(rtc::scoped_refptr buffer) - : buffer_(std::move(buffer)) {} + explicit VideoFrameBuffer( + rtc::scoped_refptr buffer); - VideoFrameBufferType buffer_type() const { - return static_cast(buffer_->type()); - } + VideoFrameBufferType buffer_type() const; - int width() const { return buffer_->width(); } - int height() const { return buffer_->height(); } + int width() const; + int height() const; - std::unique_ptr to_i420() { - return std::make_unique(buffer_->ToI420()); - } - - // const_cast is valid here because we take the ownership on the rust side - std::unique_ptr get_i420() { - return std::make_unique( - rtc::scoped_refptr( - const_cast(buffer_->GetI420()))); - } - - std::unique_ptr get_i420a() { - return std::make_unique( - rtc::scoped_refptr( - const_cast(buffer_->GetI420A()))); - } - - std::unique_ptr get_i422() { - return std::make_unique( - rtc::scoped_refptr( - const_cast(buffer_->GetI422()))); - } - - std::unique_ptr get_i444() { - return std::make_unique( - rtc::scoped_refptr( - const_cast(buffer_->GetI444()))); - } - - std::unique_ptr get_i010() { - return std::make_unique( - rtc::scoped_refptr( - const_cast(buffer_->GetI010()))); - } - - std::unique_ptr get_nv12() { - return std::make_unique( - rtc::scoped_refptr( - const_cast(buffer_->GetNV12()))); - } + // Takes ownership + std::unique_ptr to_i420(); + std::unique_ptr get_i420(); + std::unique_ptr get_i420a(); + std::unique_ptr get_i422(); + std::unique_ptr get_i444(); + std::unique_ptr get_i010(); + std::unique_ptr get_nv12(); + rtc::scoped_refptr get() const; protected: rtc::scoped_refptr buffer_; @@ -78,120 +53,102 @@ class VideoFrameBuffer { class PlanarYuvBuffer : public VideoFrameBuffer { public: - explicit PlanarYuvBuffer(rtc::scoped_refptr buffer) - : VideoFrameBuffer(buffer) {} + explicit PlanarYuvBuffer(rtc::scoped_refptr buffer); - int chroma_width() const { return buffer()->ChromaWidth(); } - int chroma_height() const { return buffer()->ChromaHeight(); } + int chroma_width() const; + int chroma_height() const; - int stride_y() const { return buffer()->StrideY(); } - int stride_u() const { return buffer()->StrideU(); } - int stride_v() const { return buffer()->StrideV(); } + int stride_y() const; + int stride_u() const; + int stride_v() const; private: - webrtc::PlanarYuvBuffer* buffer() const { - return static_cast(buffer_.get()); - } + webrtc::PlanarYuvBuffer* buffer() const; }; class PlanarYuv8Buffer : public PlanarYuvBuffer { public: - explicit PlanarYuv8Buffer(rtc::scoped_refptr buffer) - : PlanarYuvBuffer(buffer) {} + explicit PlanarYuv8Buffer( + rtc::scoped_refptr buffer); - const uint8_t* data_y() const { return buffer()->DataY(); } - const uint8_t* data_u() const { return buffer()->DataU(); } - const uint8_t* data_v() const { return buffer()->DataV(); } + const uint8_t* data_y() const; + const uint8_t* data_u() const; + const uint8_t* data_v() const; private: - webrtc::PlanarYuv8Buffer* buffer() const { - return static_cast(buffer_.get()); - } + webrtc::PlanarYuv8Buffer* buffer() const; }; class PlanarYuv16BBuffer : public PlanarYuvBuffer { public: explicit PlanarYuv16BBuffer( - rtc::scoped_refptr buffer) - : PlanarYuvBuffer(buffer) {} + rtc::scoped_refptr buffer); - const uint16_t* data_y() const { return buffer()->DataY(); } - const uint16_t* data_u() const { return buffer()->DataU(); } - const uint16_t* data_v() const { return buffer()->DataV(); } + const uint16_t* data_y() const; + const uint16_t* data_u() const; + const uint16_t* data_v() const; private: - webrtc::PlanarYuv16BBuffer* buffer() const { - return static_cast(buffer_.get()); - } + webrtc::PlanarYuv16BBuffer* buffer() const; }; class BiplanarYuvBuffer : public VideoFrameBuffer { public: explicit BiplanarYuvBuffer( - rtc::scoped_refptr buffer) - : VideoFrameBuffer(buffer) {} + rtc::scoped_refptr buffer); - int chroma_width() const { return buffer()->ChromaWidth(); } - int chroma_height() const { return buffer()->ChromaHeight(); } + int chroma_width() const; + int chroma_height() const; - int stride_y() const { return buffer()->StrideY(); } - int stride_uv() const { return buffer()->StrideUV(); } + int stride_y() const; + int stride_uv() const; private: - webrtc::BiplanarYuvBuffer* buffer() const { - return static_cast(buffer_.get()); - } + webrtc::BiplanarYuvBuffer* buffer() const; }; class BiplanarYuv8Buffer : public BiplanarYuvBuffer { public: explicit BiplanarYuv8Buffer( - rtc::scoped_refptr buffer) - : BiplanarYuvBuffer(buffer) {} + rtc::scoped_refptr buffer); - const uint8_t* data_y() const { return buffer()->DataY(); } - const uint8_t* data_uv() const { return buffer()->DataUV(); } + const uint8_t* data_y() const; + const uint8_t* data_uv() const; private: - webrtc::BiplanarYuv8Buffer* buffer() const { - return static_cast(buffer_.get()); - } + webrtc::BiplanarYuv8Buffer* buffer() const; }; +std::unique_ptr create_i420_buffer(int width, int height); + class I420Buffer : public PlanarYuv8Buffer { public: - explicit I420Buffer(rtc::scoped_refptr buffer) - : PlanarYuv8Buffer(buffer) {} + explicit I420Buffer(rtc::scoped_refptr buffer); }; class I420ABuffer : public I420Buffer { public: - explicit I420ABuffer(rtc::scoped_refptr buffer) - : I420Buffer(buffer) {} + explicit I420ABuffer(rtc::scoped_refptr buffer); }; class I422Buffer : public PlanarYuv8Buffer { public: - explicit I422Buffer(rtc::scoped_refptr buffer) - : PlanarYuv8Buffer(buffer) {} + explicit I422Buffer(rtc::scoped_refptr buffer); }; class I444Buffer : public PlanarYuv8Buffer { public: - explicit I444Buffer(rtc::scoped_refptr buffer) - : PlanarYuv8Buffer(buffer) {} + explicit I444Buffer(rtc::scoped_refptr buffer); }; class I010Buffer : public PlanarYuv16BBuffer { public: - explicit I010Buffer(rtc::scoped_refptr buffer) - : PlanarYuv16BBuffer(buffer) {} + explicit I010Buffer(rtc::scoped_refptr buffer); }; class NV12Buffer : public BiplanarYuv8Buffer { public: - explicit NV12Buffer(rtc::scoped_refptr buffer) - : BiplanarYuv8Buffer(buffer) {} + explicit NV12Buffer(rtc::scoped_refptr buffer); }; static const VideoFrameBuffer* yuv_to_vfb(const PlanarYuvBuffer* yuv) { @@ -240,9 +197,7 @@ static const BiplanarYuv8Buffer* nv12_to_biyuv8(const NV12Buffer* nv12) { } static std::unique_ptr _unique_video_frame_buffer() { - return nullptr; // Ignore + return nullptr; } } // namespace livekit - -#endif // LIVEKIT_WEBRTC_VIDEO_FRAME_BUFFER_H diff --git a/webrtc-sys/include/livekit/webrtc.h b/webrtc-sys/include/livekit/webrtc.h index 50fe659..8a372f9 100644 --- a/webrtc-sys/include/livekit/webrtc.h +++ b/webrtc-sys/include/livekit/webrtc.h @@ -2,8 +2,7 @@ // Created by theom on 18/09/2022. // -#ifndef LIVEKIT_WEBRTC_WEBRTC_H -#define LIVEKIT_WEBRTC_WEBRTC_H +#pragma once #include "rtc_base/physical_socket_server.h" #include "rtc_base/ssl_adapter.h" @@ -12,6 +11,11 @@ #include "rtc_base/win32_socket_init.h" #endif +namespace livekit { +class RTCRuntime; +} +#include "webrtc-sys/src/webrtc.rs.h" + namespace livekit { class RTCRuntime { @@ -30,15 +34,14 @@ class RTCRuntime { std::unique_ptr network_thread_; std::unique_ptr worker_thread_; std::unique_ptr signaling_thread_; + #ifdef WEBRTC_WIN rtc::WinsockInitializer winsock_; rtc::PhysicalSocketServer ss_; - rtc::AutoSocketServerThread main_thread_ {&ss_}; + rtc::AutoSocketServerThread main_thread_{&ss_}; #endif }; std::shared_ptr create_rtc_runtime(); } // namespace livekit - -#endif // LIVEKIT_WEBRTC_WEBRTC_H diff --git a/webrtc-sys/include/livekit/yuv_helper.h b/webrtc-sys/include/livekit/yuv_helper.h index 64e02df..beaefc7 100644 --- a/webrtc-sys/include/livekit/yuv_helper.h +++ b/webrtc-sys/include/livekit/yuv_helper.h @@ -2,12 +2,12 @@ // Created by Théo Monnom on 01/12/2022. // -#ifndef CLIENT_SDK_NATIVE_YUV_HELPER_H -#define CLIENT_SDK_NATIVE_YUV_HELPER_H +#pragma once #include #include "api/video/yuv_helper.h" +#include "webrtc-sys/src/yuv_helper.rs.h" namespace livekit { @@ -68,5 +68,3 @@ static void i420_to_rgba(const uint8_t* src_y, } } // namespace livekit - -#endif // CLIENT_SDK_NATIVE_YUV_HELPER_H diff --git a/webrtc-sys/src/candidate.rs b/webrtc-sys/src/candidate.rs index d9e3b46..a61c549 100644 --- a/webrtc-sys/src/candidate.rs +++ b/webrtc-sys/src/candidate.rs @@ -5,6 +5,6 @@ pub mod ffi { type Candidate; // cricket::Candidate - fn _unique_candidate() -> UniquePtr; // Ignore + fn _shared_candidate() -> SharedPtr; } } diff --git a/webrtc-sys/src/data_channel.cpp b/webrtc-sys/src/data_channel.cpp index 30d7e68..60bb078 100644 --- a/webrtc-sys/src/data_channel.cpp +++ b/webrtc-sys/src/data_channel.cpp @@ -6,8 +6,6 @@ #include -#include "webrtc-sys/src/data_channel.rs.h" - namespace livekit { DataChannel::DataChannel( @@ -16,11 +14,11 @@ DataChannel::DataChannel( : rtc_runtime_(std::move(rtc_runtime)), data_channel_(std::move(data_channel)) {} -void DataChannel::register_observer(NativeDataChannelObserver& observer) { +void DataChannel::register_observer(NativeDataChannelObserver& observer) const { data_channel_->RegisterObserver(&observer); } -void DataChannel::unregister_observer() { +void DataChannel::unregister_observer() const { data_channel_->UnregisterObserver(); } diff --git a/webrtc-sys/src/data_channel.rs b/webrtc-sys/src/data_channel.rs index 004be48..b27e0a8 100644 --- a/webrtc-sys/src/data_channel.rs +++ b/webrtc-sys/src/data_channel.rs @@ -1,4 +1,4 @@ -use std::fmt::Debug; +use crate::impl_thread_safety; use std::slice; #[cxx::bridge(namespace = "livekit")] @@ -63,12 +63,12 @@ pub mod ffi { /// SAFETY /// The observer must live as the datachannel uses it unsafe fn register_observer( - self: Pin<&mut DataChannel>, + self: &DataChannel, observer: Pin<&mut NativeDataChannelObserver>, ); - fn unregister_observer(self: Pin<&mut DataChannel>); - fn send(self: &DataChannel, data: &DataBuffer) -> bool; + fn unregister_observer(self: &DataChannel); + fn send(self: &DataChannel, data: &DataBuffer) -> bool; fn label(self: &DataChannel) -> String; fn state(self: &DataChannel) -> DataState; fn close(self: &DataChannel); @@ -82,11 +82,8 @@ pub mod ffi { } } -unsafe impl Send for ffi::DataChannel {} -unsafe impl Sync for ffi::DataChannel {} - -unsafe impl Send for ffi::NativeDataChannelObserver {} -unsafe impl Sync for ffi::NativeDataChannelObserver {} +impl_thread_safety!(ffi::DataChannel, Send + Sync); +impl_thread_safety!(ffi::NativeDataChannelObserver, Send + Sync); // DataChannelObserver diff --git a/webrtc-sys/src/helper.rs b/webrtc-sys/src/helper.rs new file mode 100644 index 0000000..466b473 --- /dev/null +++ b/webrtc-sys/src/helper.rs @@ -0,0 +1,54 @@ +#[cxx::bridge(namespace = "livekit")] +pub mod ffi { + + // Wrapper to opaque C++ objects + // https://github.com/dtolnay/cxx/issues/741 + // Used to allow SharedPtr/UniquePtr type inside a rust::Vec + pub struct MediaStreamPtr { + pub ptr: SharedPtr, + } + + pub struct CandidatePtr { + pub ptr: SharedPtr, + } + + pub struct AudioTrackPtr { + pub ptr: SharedPtr, + } + + pub struct VideoTrackPtr { + pub ptr: SharedPtr, + } + + pub struct RtpSenderPtr { + pub ptr: SharedPtr, + } + + pub struct RtpReceiverPtr { + pub ptr: SharedPtr, + } + + pub struct RtpTransceiverPtr { + pub ptr: SharedPtr, + } + + unsafe extern "C++" { + include!("livekit/helper.h"); + + type MediaStream = crate::media_stream::ffi::MediaStream; + type AudioTrack = crate::media_stream::ffi::AudioTrack; + type VideoTrack = crate::media_stream::ffi::VideoTrack; + type Candidate = crate::candidate::ffi::Candidate; + type RtpSender = crate::rtp_sender::ffi::RtpSender; + type RtpReceiver = crate::rtp_receiver::ffi::RtpReceiver; + type RtpTransceiver = crate::rtp_transceiver::ffi::RtpTransceiver; + + fn _vec_media_stream_ptr() -> Vec; + fn _vec_candidate_ptr() -> Vec; + fn _vec_audio_track_ptr() -> Vec; + fn _vec_video_track_ptr() -> Vec; + fn _vec_rtp_sender_ptr() -> Vec; + fn _vec_rtp_receiver_ptr() -> Vec; + fn _vec_rtp_transceiver_ptr() -> Vec; + } +} diff --git a/webrtc-sys/src/jsep.cpp b/webrtc-sys/src/jsep.cpp index 9bcebf0..777a77b 100644 --- a/webrtc-sys/src/jsep.cpp +++ b/webrtc-sys/src/jsep.cpp @@ -7,8 +7,8 @@ #include #include -#include "webrtc-sys/src/jsep.rs.h" #include "livekit/rtc_error.h" +#include "rtc_base/ref_counted_object.h" namespace livekit { @@ -47,7 +47,7 @@ std::unique_ptr IceCandidate::release() { return std::move(ice_candidate_); } -std::unique_ptr create_ice_candidate(rust::String sdp_mid, +std::shared_ptr create_ice_candidate(rust::String sdp_mid, int sdp_mline_index, rust::String sdp) { webrtc::SdpParseError error; @@ -57,7 +57,7 @@ std::unique_ptr create_ice_candidate(rust::String sdp_mid, throw std::runtime_error(serialize_sdp_error(error)); } - return std::make_unique( + return std::make_shared( std::unique_ptr(ice_rtc)); } diff --git a/webrtc-sys/src/jsep.rs b/webrtc-sys/src/jsep.rs index 72ce2af..10a0b46 100644 --- a/webrtc-sys/src/jsep.rs +++ b/webrtc-sys/src/jsep.rs @@ -1,10 +1,10 @@ +use crate::impl_thread_safety; +use cxx::UniquePtr; use std::error::Error; -use std::fmt::{Debug, Display, Formatter}; +use std::fmt::{Display, Formatter}; use std::mem::ManuallyDrop; use std::str::FromStr; -use cxx::UniquePtr; - use crate::rtc_error::ffi::RTCError; #[cxx::bridge(namespace = "livekit")] @@ -39,11 +39,15 @@ pub mod ffi { fn on_set_remote_description_complete(self: &SetRemoteSdpObserverWrapper, error: RTCError); } - unsafe extern "C++" { - include!("webrtc-sys/src/rtc_error.rs.h"); - include!("livekit/jsep.h"); + extern "C++" { + include!("livekit/rtc_error.h"); type RTCError = crate::rtc_error::ffi::RTCError; + } + + unsafe extern "C++" { + include!("livekit/jsep.h"); + type IceCandidate; type SessionDescription; type NativeCreateSdpObserverHandle; @@ -72,13 +76,13 @@ pub mod ffi { sdp_mid: String, sdp_mline_index: i32, sdp: String, - ) -> Result>; + ) -> Result>; fn create_session_description( sdp_type: SdpType, sdp: String, ) -> Result>; - fn _unique_ice_candidate() -> UniquePtr; // Ignore + fn _shared_ice_candidate() -> SharedPtr; // Ignore fn _unique_session_description() -> UniquePtr; // Ignore } } @@ -95,13 +99,8 @@ impl Display for ffi::SdpParseError { } } -unsafe impl Send for ffi::SessionDescription {} - -unsafe impl Sync for ffi::SessionDescription {} - -unsafe impl Send for ffi::IceCandidate {} - -unsafe impl Sync for ffi::IceCandidate {} +impl_thread_safety!(ffi::SessionDescription, Send + Sync); +impl_thread_safety!(ffi::IceCandidate, Send + Sync); impl ffi::SdpParseError { /// # Safety diff --git a/webrtc-sys/src/lib.rs b/webrtc-sys/src/lib.rs index 0fe1736..19350bd 100644 --- a/webrtc-sys/src/lib.rs +++ b/webrtc-sys/src/lib.rs @@ -1,11 +1,14 @@ pub mod candidate; pub mod data_channel; +pub mod helper; pub mod jsep; pub mod media_stream; pub mod peer_connection; pub mod peer_connection_factory; pub mod rtc_error; +pub mod rtp_parameters; pub mod rtp_receiver; +pub mod rtp_sender; pub mod rtp_transceiver; pub mod video_frame; pub mod video_frame_buffer; diff --git a/webrtc-sys/src/media_stream.cpp b/webrtc-sys/src/media_stream.cpp index ec8d0d1..a23db9c 100644 --- a/webrtc-sys/src/media_stream.cpp +++ b/webrtc-sys/src/media_stream.cpp @@ -4,23 +4,91 @@ #include "livekit/media_stream.h" +#include +#include + #include "api/media_stream_interface.h" -#include "webrtc-sys/src/media_stream.rs.h" +#include "api/video/video_frame.h" +#include "rtc_base/ref_counted_object.h" namespace livekit { +MediaStream::MediaStream( + rtc::scoped_refptr stream) + : media_stream_(std::move(stream)) {} + +rust::String MediaStream::id() const { + return media_stream_->id(); +} + +rust::Vec MediaStream::get_video_tracks() const { + rust::Vec rust; + for (auto video : media_stream_->GetVideoTracks()) + rust.push_back(VideoTrackPtr{std::make_shared(video)}); + + return rust; +} + +rust::Vec MediaStream::get_audio_tracks() const { + rust::Vec rust; + for (auto audio : media_stream_->GetAudioTracks()) + rust.push_back(AudioTrackPtr{std::make_shared(audio)}); + + return rust; +} + +std::shared_ptr MediaStream::find_audio_track( + rust::String track_id) const { + return std::make_shared( + media_stream_->FindAudioTrack(track_id.c_str())); +} + +std::shared_ptr MediaStream::find_video_track( + rust::String track_id) const { + return std::make_shared( + media_stream_->FindVideoTrack(track_id.c_str())); +} + +bool MediaStream::add_audio_track( + std::shared_ptr audio_track) const { + return media_stream_->AddTrack( + rtc::scoped_refptr( + static_cast(audio_track->get().get()))); +} + +bool MediaStream::add_video_track( + std::shared_ptr video_track) const { + return media_stream_->AddTrack( + rtc::scoped_refptr( + static_cast(video_track->get().get()))); +} + +bool MediaStream::remove_audio_track( + std::shared_ptr audio_track) const { + return media_stream_->RemoveTrack( + rtc::scoped_refptr( + static_cast(audio_track->get().get()))); +} + +bool MediaStream::remove_video_track( + std::shared_ptr video_track) const { + return media_stream_->RemoveTrack( + rtc::scoped_refptr( + static_cast(video_track->get().get()))); +} + MediaStreamTrack::MediaStreamTrack( rtc::scoped_refptr track) : track_(std::move(track)) {} -std::unique_ptr MediaStreamTrack::from( +std::shared_ptr MediaStreamTrack::from( rtc::scoped_refptr track) { if (track->kind() == webrtc::MediaStreamTrackInterface::kVideoKind) { - return std::make_unique( + return std::make_shared( rtc::scoped_refptr( static_cast(track.get()))); } else { - return std::make_unique( + return std::make_shared( rtc::scoped_refptr( static_cast(track.get()))); } @@ -38,7 +106,7 @@ bool MediaStreamTrack::enabled() const { return track_->enabled(); } -bool MediaStreamTrack::set_enabled(bool enable) { +bool MediaStreamTrack::set_enabled(bool enable) const { return track_->set_enabled(enable); } @@ -46,29 +114,21 @@ TrackState MediaStreamTrack::state() const { return static_cast(track_->state()); } -MediaStream::MediaStream( - rtc::scoped_refptr stream) - : media_stream_(std::move(stream)) {} - -rust::String MediaStream::id() const { - return media_stream_->id(); -} - AudioTrack::AudioTrack(rtc::scoped_refptr track) : MediaStreamTrack(std::move(track)) {} VideoTrack::VideoTrack(rtc::scoped_refptr track) : MediaStreamTrack(std::move(track)) {} -void VideoTrack::add_sink(NativeVideoFrameSink& sink) { +void VideoTrack::add_sink(NativeVideoFrameSink& sink) const { track()->AddOrUpdateSink(&sink, rtc::VideoSinkWants()); } -void VideoTrack::remove_sink(NativeVideoFrameSink& sink) { +void VideoTrack::remove_sink(NativeVideoFrameSink& sink) const { track()->RemoveSink(&sink); } -void VideoTrack::set_should_receive(bool should_receive) { +void VideoTrack::set_should_receive(bool should_receive) const { track()->set_should_receive(should_receive); } @@ -80,7 +140,7 @@ ContentHint VideoTrack::content_hint() const { return static_cast(track()->content_hint()); } -void VideoTrack::set_content_hint(ContentHint hint) { +void VideoTrack::set_content_hint(ContentHint hint) const { track()->set_content_hint( static_cast(hint)); } @@ -110,4 +170,83 @@ std::unique_ptr create_native_video_frame_sink( return std::make_unique(std::move(observer)); } +NativeVideoTrackSource::NativeVideoTrackSource() + : rtc::AdaptedVideoTrackSource(1) {} + +NativeVideoTrackSource::~NativeVideoTrackSource() {} + +bool NativeVideoTrackSource::is_screencast() const { + return false; +} + +absl::optional NativeVideoTrackSource::needs_denoising() const { + return false; +} + +webrtc::MediaSourceInterface::SourceState NativeVideoTrackSource::state() + const { + // TODO(theomonnom): expose source state to Rust + return SourceState::kLive; +} + +bool NativeVideoTrackSource::remote() const { + return false; +} + +bool NativeVideoTrackSource::on_captured_frame( + const webrtc::VideoFrame& frame) { + webrtc::MutexLock lock(&mutex_); + + int64_t aligned_timestamp_us = timestamp_aligner_.TranslateTimestamp( + frame.timestamp_us(), rtc::TimeMicros()); + + int adapted_width, adapted_height, crop_width, crop_height, crop_x, crop_y; + if (!AdaptFrame(frame.width(), frame.height(), frame.timestamp_us(), + &adapted_width, &adapted_height, &crop_width, &crop_height, + &crop_x, &crop_y)) { + return false; + } + + // TODO(theomonnom): Should this be handled by the users? + rtc::scoped_refptr buffer = + frame.video_frame_buffer(); + if (adapted_width != frame.width() || adapted_height != frame.height()) { + buffer = buffer->CropAndScale(crop_x, crop_y, crop_width, crop_height, + adapted_width, adapted_height); + } + + if (apply_rotation() && frame.rotation() != webrtc::kVideoRotation_0) { + // If the buffer is I420, rtc::AdaptedVideoTrackSource will handle the + // rotation for us. + buffer = buffer->ToI420(); + } + + OnFrame(webrtc::VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_rotation(frame.rotation()) + .set_timestamp_us(aligned_timestamp_us) + .build()); + + return true; +} + +AdaptedVideoTrackSource::AdaptedVideoTrackSource( + rtc::scoped_refptr source) + : source_(source) {} + +bool AdaptedVideoTrackSource::on_captured_frame( + std::unique_ptr frame) const { + return source_->on_captured_frame(frame->get()); +} + +rtc::scoped_refptr AdaptedVideoTrackSource::get() + const { + return source_; +} + +std::unique_ptr create_adapted_video_track_source() { + return std::make_unique( + rtc::make_ref_counted()); +} + } // namespace livekit diff --git a/webrtc-sys/src/media_stream.rs b/webrtc-sys/src/media_stream.rs index 86b8215..4f1fbd0 100644 --- a/webrtc-sys/src/media_stream.rs +++ b/webrtc-sys/src/media_stream.rs @@ -1,6 +1,6 @@ -use cxx::UniquePtr; - +use crate::impl_thread_safety; use crate::video_frame::ffi::VideoFrame; +use cxx::UniquePtr; #[cxx::bridge(namespace = "livekit")] pub mod ffi { @@ -22,51 +22,68 @@ pub mod ffi { } // -1 = optional + #[derive(Debug)] pub struct VideoTrackSourceConstraints { pub min_fps: f64, pub max_fps: f64, } + extern "C++" { + include!("livekit/video_frame.h"); + include!("livekit/helper.h"); + + type VideoFrame = crate::video_frame::ffi::VideoFrame; + type VideoTrackPtr = crate::helper::ffi::VideoTrackPtr; + type AudioTrackPtr = crate::helper::ffi::AudioTrackPtr; + } + unsafe extern "C++" { include!("livekit/media_stream.h"); - include!("livekit/video_frame.h"); type NativeVideoFrameSink; type MediaStreamTrack; type MediaStream; type AudioTrack; type VideoTrack; - type VideoFrame = crate::video_frame::ffi::VideoFrame; + type AdaptedVideoTrackSource; fn id(self: &MediaStream) -> String; + fn get_audio_tracks(self: &MediaStream) -> Vec; + fn get_video_tracks(self: &MediaStream) -> Vec; + fn find_audio_track(self: &MediaStream, track_id: String) -> SharedPtr; + fn find_video_track(self: &MediaStream, track_id: String) -> SharedPtr; + fn add_audio_track(self: &MediaStream, audio_track: SharedPtr) -> bool; + fn add_video_track(self: &MediaStream, video_track: SharedPtr) -> bool; + fn remove_audio_track(self: &MediaStream, audio_track: SharedPtr) -> bool; + fn remove_video_track(self: &MediaStream, video_track: SharedPtr) -> bool; fn kind(self: &MediaStreamTrack) -> String; fn id(self: &MediaStreamTrack) -> String; fn enabled(self: &MediaStreamTrack) -> bool; - fn set_enabled(self: Pin<&mut MediaStreamTrack>, enable: bool) -> bool; + fn set_enabled(self: &MediaStreamTrack, enable: bool) -> bool; fn state(self: &MediaStreamTrack) -> TrackState; - unsafe fn add_sink(self: Pin<&mut VideoTrack>, sink: Pin<&mut NativeVideoFrameSink>); - unsafe fn remove_sink(self: Pin<&mut VideoTrack>, sink: Pin<&mut NativeVideoFrameSink>); + unsafe fn add_sink(self: &VideoTrack, sink: Pin<&mut NativeVideoFrameSink>); + unsafe fn remove_sink(self: &VideoTrack, sink: Pin<&mut NativeVideoFrameSink>); - fn set_should_receive(self: Pin<&mut VideoTrack>, should_receive: bool); + fn set_should_receive(self: &VideoTrack, should_receive: bool); fn should_receive(self: &VideoTrack) -> bool; fn content_hint(self: &VideoTrack) -> ContentHint; - fn set_content_hint(self: Pin<&mut VideoTrack>, hint: ContentHint); + fn set_content_hint(self: &VideoTrack, hint: ContentHint); fn create_native_video_frame_sink( observer: Box, ) -> UniquePtr; - unsafe fn video_to_media(track: *const VideoTrack) -> *const MediaStreamTrack; - unsafe fn audio_to_media(track: *const AudioTrack) -> *const MediaStreamTrack; + fn on_captured_frame(self: &AdaptedVideoTrackSource, frame: UniquePtr) -> bool; + unsafe fn media_to_video(track: *const MediaStreamTrack) -> *const VideoTrack; unsafe fn media_to_audio(track: *const MediaStreamTrack) -> *const AudioTrack; - fn _unique_media_stream_track() -> UniquePtr; // Ignore - fn _unique_media_stream() -> UniquePtr; // Ignore - fn _unique_audio_track() -> UniquePtr; // Ignore - fn _unique_video_track() -> UniquePtr; // Ignore + fn _shared_media_stream_track() -> SharedPtr; + fn _shared_audio_track() -> SharedPtr; + fn _shared_video_track() -> SharedPtr; + fn _shared_media_stream() -> SharedPtr; } extern "Rust" { @@ -81,16 +98,11 @@ pub mod ffi { } } -unsafe impl Sync for ffi::MediaStreamTrack {} -unsafe impl Send for ffi::MediaStreamTrack {} -unsafe impl Sync for ffi::MediaStream {} -unsafe impl Send for ffi::MediaStream {} -unsafe impl Send for ffi::AudioTrack {} -unsafe impl Sync for ffi::AudioTrack {} -unsafe impl Send for ffi::VideoTrack {} -unsafe impl Sync for ffi::VideoTrack {} -unsafe impl Send for ffi::NativeVideoFrameSink {} -unsafe impl Sync for ffi::NativeVideoFrameSink {} +impl_thread_safety!(ffi::MediaStreamTrack, Send + Sync); +impl_thread_safety!(ffi::MediaStream, Send + Sync); +impl_thread_safety!(ffi::AudioTrack, Send + Sync); +impl_thread_safety!(ffi::VideoTrack, Send + Sync); +impl_thread_safety!(ffi::NativeVideoFrameSink, Send + Sync); pub trait VideoFrameSink: Send + Sync { fn on_frame(&self, frame: UniquePtr); diff --git a/webrtc-sys/src/peer_connection.cpp b/webrtc-sys/src/peer_connection.cpp index dd512b3..8b815fc 100644 --- a/webrtc-sys/src/peer_connection.cpp +++ b/webrtc-sys/src/peer_connection.cpp @@ -3,10 +3,10 @@ // #include "livekit/peer_connection.h" -#include "livekit/media_stream.h" -#include "webrtc-sys/src/peer_connection.rs.h" +#include "livekit/media_stream.h" #include "livekit/rtc_error.h" +#include "livekit/rtp_transceiver.h" namespace livekit { @@ -32,35 +32,35 @@ PeerConnection::PeerConnection( void PeerConnection::create_offer( NativeCreateSdpObserverHandle& observer_handle, - RTCOfferAnswerOptions options) { + RTCOfferAnswerOptions options) const { peer_connection_->CreateOffer(observer_handle.observer.get(), toNativeOfferAnswerOptions(options)); } void PeerConnection::create_answer( NativeCreateSdpObserverHandle& observer_handle, - RTCOfferAnswerOptions options) { + RTCOfferAnswerOptions options) const { peer_connection_->CreateAnswer(observer_handle.observer.get(), toNativeOfferAnswerOptions(options)); } void PeerConnection::set_local_description( std::unique_ptr desc, - NativeSetLocalSdpObserverHandle& observer) { + NativeSetLocalSdpObserverHandle& observer) const { peer_connection_->SetLocalDescription(desc->clone()->release(), observer.observer); } void PeerConnection::set_remote_description( std::unique_ptr desc, - NativeSetRemoteSdpObserverHandle& observer) { + NativeSetRemoteSdpObserverHandle& observer) const { peer_connection_->SetRemoteDescription(desc->clone()->release(), observer.observer); } std::unique_ptr PeerConnection::create_data_channel( rust::String label, - std::unique_ptr init) { + std::unique_ptr init) const { auto result = peer_connection_->CreateDataChannelOrError(label.c_str(), init.get()); @@ -71,9 +71,76 @@ std::unique_ptr PeerConnection::create_data_channel( return std::make_unique(rtc_runtime_, result.value()); } +std::shared_ptr PeerConnection::add_track( + std::shared_ptr track, + const rust::Vec& stream_ids) const { + std::vector std_stream_ids(stream_ids.begin(), stream_ids.end()); + auto result = peer_connection_->AddTrack(track->get(), std_stream_ids); + if (!result.ok()) { + throw std::runtime_error(serialize_error(to_error(result.error()))); + } + + return std::make_shared(result.value()); +} + +void PeerConnection::remove_track(std::shared_ptr sender) const { + auto error = peer_connection_->RemoveTrackOrError(sender->get()); + if (!error.ok()) + throw std::runtime_error(serialize_error(to_error(error))); +} + +std::shared_ptr PeerConnection::add_transceiver( + std::shared_ptr track, + RtpTransceiverInit init) const { + auto result = peer_connection_->AddTransceiver( + track->get(), to_native_rtp_transceiver_init(init)); + if (result.ok()) + throw std::runtime_error(serialize_error(to_error(result.error()))); + + return std::make_shared(result.value()); +} + +std::shared_ptr PeerConnection::add_transceiver_for_media( + MediaType media_type, + RtpTransceiverInit init) const { + auto result = peer_connection_->AddTransceiver( + static_cast(media_type), + to_native_rtp_transceiver_init(init)); + + if (result.ok()) + throw std::runtime_error(serialize_error(to_error(result.error()))); + + return std::make_shared(result.value()); +} + +rust::Vec PeerConnection::get_senders() const { + rust::Vec vec; + for (auto sender : peer_connection_->GetSenders()) + vec.push_back(RtpSenderPtr{std::make_shared(sender)}); + + return vec; +} + +rust::Vec PeerConnection::get_receivers() const { + rust::Vec vec; + for (auto receiver : peer_connection_->GetReceivers()) + vec.push_back(RtpReceiverPtr{std::make_shared(receiver)}); + + return vec; +} + +rust::Vec PeerConnection::get_transceivers() const { + rust::Vec vec; + for (auto transceiver : peer_connection_->GetTransceivers()) + vec.push_back( + RtpTransceiverPtr{std::make_shared(transceiver)}); + + return vec; +} + void PeerConnection::add_ice_candidate( - std::unique_ptr candidate, - NativeAddIceCandidateObserver& observer) { + std::shared_ptr candidate, + NativeAddIceCandidateObserver& observer) const { peer_connection_->AddIceCandidate( candidate->release(), [&](const webrtc::RTCError& err) { observer.OnComplete(to_error(err)); }); diff --git a/webrtc-sys/src/peer_connection.rs b/webrtc-sys/src/peer_connection.rs index cae7e1d..e4bc8d6 100644 --- a/webrtc-sys/src/peer_connection.rs +++ b/webrtc-sys/src/peer_connection.rs @@ -1,21 +1,19 @@ -use std::fmt::Debug; -use std::mem::ManuallyDrop; - -use cxx::UniquePtr; - use crate::candidate::ffi::Candidate; use crate::data_channel::ffi::DataChannel; +use crate::impl_thread_safety; use crate::jsep::ffi::IceCandidate; use crate::media_stream::ffi::MediaStream; use crate::rtc_error::ffi::RTCError; use crate::rtp_receiver::ffi::RtpReceiver; use crate::rtp_transceiver::ffi::RtpTransceiver; +use cxx::{SharedPtr, UniquePtr}; +use std::mem::ManuallyDrop; #[cxx::bridge(namespace = "livekit")] pub mod ffi { struct CandidatePair { - local: UniquePtr, - remote: UniquePtr, + local: SharedPtr, + remote: SharedPtr, } struct CandidatePairChangeEvent { @@ -80,39 +78,44 @@ pub mod ffi { use_obsolete_sctp_sdp: bool, } - // Wrapper to opaque C++ objects - // https://github.com/dtolnay/cxx/issues/741 - struct MediaStreamPtr { - pub ptr: UniquePtr, - } - - struct CandidatePtr { - pub ptr: UniquePtr, - } - - unsafe extern "C++" { - include!("livekit/peer_connection.h"); - include!("livekit/jsep.h"); - include!("livekit/data_channel.h"); - include!("livekit/rtp_receiver.h"); - include!("livekit/rtp_transceiver.h"); - include!("livekit/media_stream.h"); + extern "C++" { + include!("livekit/rtc_error.h"); + include!("livekit/helper.h"); include!("livekit/candidate.h"); - include!("webrtc-sys/src/rtc_error.rs.h"); + include!("livekit/media_stream.h"); + include!("livekit/rtp_transceiver.h"); + include!("livekit/rtp_sender.h"); + include!("livekit/rtp_receiver.h"); + include!("livekit/data_channel.h"); + include!("livekit/jsep.h"); + include!("livekit/webrtc.h"); + type MediaStreamPtr = crate::helper::ffi::MediaStreamPtr; + type CandidatePtr = crate::helper::ffi::CandidatePtr; + type RtpSenderPtr = crate::helper::ffi::RtpSenderPtr; + type RtpReceiverPtr = crate::helper::ffi::RtpReceiverPtr; + type RtpTransceiverPtr = crate::helper::ffi::RtpTransceiverPtr; type RTCError = crate::rtc_error::ffi::RTCError; type Candidate = crate::candidate::ffi::Candidate; type IceCandidate = crate::jsep::ffi::IceCandidate; type DataChannel = crate::data_channel::ffi::DataChannel; + type RtpSender = crate::rtp_sender::ffi::RtpSender; type RtpReceiver = crate::rtp_receiver::ffi::RtpReceiver; type RtpTransceiver = crate::rtp_transceiver::ffi::RtpTransceiver; + type RtpTransceiverInit = crate::rtp_transceiver::ffi::RtpTransceiverInit; type MediaStream = crate::media_stream::ffi::MediaStream; + type MediaStreamTrack = crate::media_stream::ffi::MediaStreamTrack; type NativeCreateSdpObserverHandle = crate::jsep::ffi::NativeCreateSdpObserverHandle; type NativeSetLocalSdpObserverHandle = crate::jsep::ffi::NativeSetLocalSdpObserverHandle; type NativeSetRemoteSdpObserverHandle = crate::jsep::ffi::NativeSetRemoteSdpObserverHandle; type NativeDataChannelInit = crate::data_channel::ffi::NativeDataChannelInit; type SessionDescription = crate::jsep::ffi::SessionDescription; + type MediaType = crate::webrtc::ffi::MediaType; type RTCRuntime = crate::webrtc::ffi::RTCRuntime; + } + + unsafe extern "C++" { + include!("livekit/peer_connection.h"); type NativeAddIceCandidateObserver; type NativePeerConnectionObserver; @@ -121,7 +124,7 @@ pub mod ffi { /// SAFETY /// The observer must live as long as the operation ends unsafe fn create_offer( - self: Pin<&mut PeerConnection>, + self: &PeerConnection, observer: Pin<&mut NativeCreateSdpObserverHandle>, options: RTCOfferAnswerOptions, ); @@ -129,7 +132,7 @@ pub mod ffi { /// SAFETY /// The observer must live as long as the operation ends unsafe fn create_answer( - self: Pin<&mut PeerConnection>, + self: &PeerConnection, observer: Pin<&mut NativeCreateSdpObserverHandle>, options: RTCOfferAnswerOptions, ); @@ -137,7 +140,7 @@ pub mod ffi { /// SAFETY /// The observer must live as long as the operation ends unsafe fn set_local_description( - self: Pin<&mut PeerConnection>, + self: &PeerConnection, desc: UniquePtr, observer: Pin<&mut NativeSetLocalSdpObserverHandle>, ); @@ -145,20 +148,46 @@ pub mod ffi { /// SAFETY /// The observer must live as long as the operation ends unsafe fn set_remote_description( - self: Pin<&mut PeerConnection>, + self: &PeerConnection, desc: UniquePtr, observer: Pin<&mut NativeSetRemoteSdpObserverHandle>, ); + fn add_track( + self: &PeerConnection, + track: SharedPtr, + stream_ids: &Vec, + ) -> Result>; + + fn remove_track(self: &PeerConnection, sender: SharedPtr) -> Result<()>; + + fn add_transceiver( + self: &PeerConnection, + track: SharedPtr, + init: RtpTransceiverInit, + ) -> Result>; + + fn add_transceiver_for_media( + self: &PeerConnection, + media_type: MediaType, + init: RtpTransceiverInit, + ) -> Result>; + + fn get_senders(self: &PeerConnection) -> Vec; + + fn get_receivers(self: &PeerConnection) -> Vec; + + fn get_transceivers(self: &PeerConnection) -> Vec; + fn create_data_channel( - self: Pin<&mut PeerConnection>, + self: &PeerConnection, label: String, init: UniquePtr, ) -> Result>; fn add_ice_candidate( - self: Pin<&mut PeerConnection>, - candidate: UniquePtr, + self: &PeerConnection, + candidate: SharedPtr, observer: Pin<&mut NativeAddIceCandidateObserver>, ); @@ -194,8 +223,8 @@ pub mod ffi { type PeerConnectionObserverWrapper; fn on_signaling_change(self: &PeerConnectionObserverWrapper, new_state: SignalingState); - fn on_add_stream(self: &PeerConnectionObserverWrapper, stream: UniquePtr); - fn on_remove_stream(self: &PeerConnectionObserverWrapper, stream: UniquePtr); + fn on_add_stream(self: &PeerConnectionObserverWrapper, stream: SharedPtr); + fn on_remove_stream(self: &PeerConnectionObserverWrapper, stream: SharedPtr); fn on_data_channel( self: &PeerConnectionObserverWrapper, data_channel: UniquePtr, @@ -220,7 +249,7 @@ pub mod ffi { ); fn on_ice_candidate( self: &PeerConnectionObserverWrapper, - candidate: UniquePtr, + candidate: SharedPtr, ); fn on_ice_candidate_error( self: &PeerConnectionObserverWrapper, @@ -244,39 +273,22 @@ pub mod ffi { ); fn on_add_track( self: &PeerConnectionObserverWrapper, - receiver: UniquePtr, + receiver: SharedPtr, streams: Vec, ); - fn on_track(self: &PeerConnectionObserverWrapper, transceiver: UniquePtr); - fn on_remove_track(self: &PeerConnectionObserverWrapper, receiver: UniquePtr); + fn on_track(self: &PeerConnectionObserverWrapper, transceiver: SharedPtr); + fn on_remove_track(self: &PeerConnectionObserverWrapper, receiver: SharedPtr); fn on_interesting_usage(self: &PeerConnectionObserverWrapper, usage_pattern: i32); } } // https://webrtc.github.io/webrtc-org/native-code/native-apis/ -unsafe impl Send for ffi::PeerConnection {} - -unsafe impl Sync for ffi::PeerConnection {} - -unsafe impl Send for ffi::NativePeerConnectionObserver {} - -unsafe impl Sync for ffi::NativePeerConnectionObserver {} - -unsafe impl Sync for ffi::NativeAddIceCandidateObserver {} - -unsafe impl Send for ffi::NativeAddIceCandidateObserver {} - -unsafe impl Sync for ffi::NativeSetRemoteSdpObserverHandle {} - -unsafe impl Send for ffi::NativeSetRemoteSdpObserverHandle {} - -unsafe impl Sync for ffi::NativeSetLocalSdpObserverHandle {} - -unsafe impl Send for ffi::NativeSetLocalSdpObserverHandle {} - -unsafe impl Sync for ffi::NativeCreateSdpObserverHandle {} - -unsafe impl Send for ffi::NativeCreateSdpObserverHandle {} +impl_thread_safety!(ffi::PeerConnection, Send + Sync); +impl_thread_safety!(ffi::NativePeerConnectionObserver, Send + Sync); +impl_thread_safety!(ffi::NativeAddIceCandidateObserver, Send + Sync); +impl_thread_safety!(ffi::NativeSetRemoteSdpObserverHandle, Send + Sync); +impl_thread_safety!(ffi::NativeSetLocalSdpObserverHandle, Send + Sync); +impl_thread_safety!(ffi::NativeCreateSdpObserverHandle, Send + Sync); impl Default for ffi::RTCOfferAnswerOptions { /* @@ -311,8 +323,8 @@ impl AddIceCandidateObserverWrapper { pub trait PeerConnectionObserver: Send + Sync { fn on_signaling_change(&self, new_state: ffi::SignalingState); - fn on_add_stream(&self, stream: UniquePtr); - fn on_remove_stream(&self, stream: UniquePtr); + fn on_add_stream(&self, stream: SharedPtr); + fn on_remove_stream(&self, stream: SharedPtr); fn on_data_channel(&self, data_channel: UniquePtr); fn on_renegotiation_needed(&self); fn on_negotiation_needed_event(&self, event: u32); @@ -320,7 +332,7 @@ pub trait PeerConnectionObserver: Send + Sync { fn on_standardized_ice_connection_change(&self, new_state: ffi::IceConnectionState); fn on_connection_change(&self, new_state: ffi::PeerConnectionState); fn on_ice_gathering_change(&self, new_state: ffi::IceGatheringState); - fn on_ice_candidate(&self, candidate: UniquePtr); + fn on_ice_candidate(&self, candidate: SharedPtr); fn on_ice_candidate_error( &self, address: String, @@ -329,12 +341,12 @@ pub trait PeerConnectionObserver: Send + Sync { error_code: i32, error_text: String, ); - fn on_ice_candidates_removed(&self, removed: Vec>); + fn on_ice_candidates_removed(&self, removed: Vec>); fn on_ice_connection_receiving_change(&self, receiving: bool); fn on_ice_selected_candidate_pair_changed(&self, event: ffi::CandidatePairChangeEvent); - fn on_add_track(&self, receiver: UniquePtr, streams: Vec>); - fn on_track(&self, transceiver: UniquePtr); - fn on_remove_track(&self, receiver: UniquePtr); + fn on_add_track(&self, receiver: SharedPtr, streams: Vec>); + fn on_track(&self, transceiver: SharedPtr); + fn on_remove_track(&self, receiver: SharedPtr); fn on_interesting_usage(&self, usage_pattern: i32); } @@ -356,13 +368,13 @@ impl PeerConnectionObserverWrapper { } } - fn on_add_stream(&self, stream: UniquePtr) { + fn on_add_stream(&self, stream: SharedPtr) { unsafe { (*self.observer).on_add_stream(stream); } } - fn on_remove_stream(&self, stream: UniquePtr) { + fn on_remove_stream(&self, stream: SharedPtr) { unsafe { (*self.observer).on_remove_stream(stream); } @@ -410,7 +422,7 @@ impl PeerConnectionObserverWrapper { } } - fn on_ice_candidate(&self, candidate: UniquePtr) { + fn on_ice_candidate(&self, candidate: SharedPtr) { unsafe { (*self.observer).on_ice_candidate(candidate); } @@ -453,7 +465,7 @@ impl PeerConnectionObserverWrapper { } } - fn on_add_track(&self, receiver: UniquePtr, streams: Vec) { + fn on_add_track(&self, receiver: SharedPtr, streams: Vec) { let mut vec = Vec::new(); for v in streams { @@ -465,13 +477,13 @@ impl PeerConnectionObserverWrapper { } } - fn on_track(&self, transceiver: UniquePtr) { + fn on_track(&self, transceiver: SharedPtr) { unsafe { (*self.observer).on_track(transceiver); } } - fn on_remove_track(&self, receiver: UniquePtr) { + fn on_remove_track(&self, receiver: SharedPtr) { unsafe { (*self.observer).on_remove_track(receiver); } diff --git a/webrtc-sys/src/peer_connection_factory.cpp b/webrtc-sys/src/peer_connection_factory.cpp index eb8bbd2..fd96597 100644 --- a/webrtc-sys/src/peer_connection_factory.cpp +++ b/webrtc-sys/src/peer_connection_factory.cpp @@ -16,7 +16,6 @@ #include "livekit/video_decoder_factory.h" #include "livekit/video_encoder_factory.h" #include "media/engine/webrtc_media_engine.h" -#include "webrtc-sys/src/peer_connection_factory.rs.h" namespace livekit { diff --git a/webrtc-sys/src/peer_connection_factory.rs b/webrtc-sys/src/peer_connection_factory.rs index 2850983..f74b2db 100644 --- a/webrtc-sys/src/peer_connection_factory.rs +++ b/webrtc-sys/src/peer_connection_factory.rs @@ -1,3 +1,5 @@ +use crate::impl_thread_safety; + #[cxx::bridge(namespace = "livekit")] pub mod ffi { #[derive(Debug, Clone)] @@ -55,6 +57,4 @@ pub mod ffi { } } -unsafe impl Send for ffi::PeerConnectionFactory {} - -unsafe impl Sync for ffi::PeerConnectionFactory {} +impl_thread_safety!(ffi::PeerConnectionFactory, Send + Sync); diff --git a/webrtc-sys/src/rtc_error.cpp b/webrtc-sys/src/rtc_error.cpp index 7488112..5b0d400 100644 --- a/webrtc-sys/src/rtc_error.cpp +++ b/webrtc-sys/src/rtc_error.cpp @@ -53,4 +53,4 @@ void throw_error() { } #endif -} // namespace livekit \ No newline at end of file +} // namespace livekit diff --git a/webrtc-sys/src/rtp_parameters.cpp b/webrtc-sys/src/rtp_parameters.cpp new file mode 100644 index 0000000..9bf43f1 --- /dev/null +++ b/webrtc-sys/src/rtp_parameters.cpp @@ -0,0 +1,446 @@ +#include "livekit/rtp_parameters.h" + +namespace livekit { + +webrtc::RtcpFeedback to_native_rtcp_feedback(RtcpFeedback feedback) { + webrtc::RtcpFeedback native; + native.type = static_cast(feedback.feedback_type); + if (feedback.has_message_type) + native.message_type = + static_cast(feedback.message_type); + + return native; +} + +webrtc::RtpCodecCapability to_native_rtp_codec_capability( + RtpCodecCapability capability) { + webrtc::RtpCodecCapability native; + // native.mime_type(); IGNORED + + native.name = capability.name.c_str(); + native.kind = static_cast(capability.kind); + + if (capability.has_clock_rate) + native.clock_rate = native.clock_rate; + + if (capability.has_preferred_payload_type) + native.preferred_payload_type = capability.preferred_payload_type; + + if (capability.has_max_ptime) + native.max_ptime = capability.max_ptime; + + if (capability.has_ptime) + native.ptime = capability.ptime; + + if (capability.has_num_channels) + native.num_channels = capability.num_channels; + + for (auto feedback : capability.rtcp_feedback) + native.rtcp_feedback.push_back(to_native_rtcp_feedback(feedback)); + + for (auto pair : capability.parameters) + native.parameters.insert(std::pair(pair.key, pair.value)); + + for (auto pair : capability.options) + native.options.insert(std::pair(pair.key, pair.value)); + + native.max_temporal_layer_extensions = + capability.max_temporal_layer_extensions; + + native.max_spatial_layer_extensions = capability.max_spatial_layer_extensions; + + native.svc_multi_stream_support = capability.svc_multi_stream_support; + + return native; +} + +webrtc::RtpHeaderExtensionCapability to_native_rtp_header_extension_capability( + RtpHeaderExtensionCapability header) { + webrtc::RtpHeaderExtensionCapability native; + native.uri = header.uri.c_str(); + + if (header.has_preferred_id) + native.preferred_id = header.preferred_id; + + native.preferred_encrypt = header.preferred_encrypt; + native.direction = + static_cast(header.direction); + + return native; +} + +webrtc::RtpExtension to_native_rtp_extension(RtpExtension ext) { + webrtc::RtpExtension native; + native.uri = ext.uri.c_str(); + native.id = ext.id; + native.encrypt = ext.encrypt; + return native; +} + +webrtc::RtpFecParameters to_rtp_fec_parameters(RtpFecParameters fec) { + webrtc::RtpFecParameters native; + + if (fec.has_ssrc) + native.ssrc = fec.ssrc; + + native.mechanism = static_cast(fec.mechanism); + return native; +} + +webrtc::RtpRtxParameters to_rtp_rtx_parameters(RtpRtxParameters rtx) { + webrtc::RtpRtxParameters native; + + if (rtx.has_ssrc) + native.ssrc = rtx.ssrc; + return native; +} + +webrtc::RtpEncodingParameters to_native_rtp_encoding_paramters( + RtpEncodingParameters parameters) { + webrtc::RtpEncodingParameters native; + native.rid = parameters.rid.c_str(); + + if (parameters.has_ssrc) + native.ssrc = parameters.ssrc; + + native.active = parameters.active; + if (parameters.has_max_framerate) + native.max_framerate = parameters.max_framerate; + + native.adaptive_ptime = parameters.adaptive_ptime; + if (parameters.has_max_bitrate_bps) + native.max_bitrate_bps = parameters.max_bitrate_bps; + + if (parameters.has_min_bitrate_bps) + native.min_bitrate_bps = parameters.min_bitrate_bps; + + native.bitrate_priority = parameters.bitrate_priority; + native.network_priority = + static_cast(parameters.network_priority); + + if (parameters.has_scalability_mode) + native.scalability_mode = parameters.scalability_mode.c_str(); + + if (parameters.has_num_temporal_layers) + native.num_temporal_layers = parameters.num_temporal_layers; + + if (parameters.has_scale_resolution_down_by) + native.scale_resolution_down_by = parameters.scale_resolution_down_by; + return native; +} + +webrtc::RtpCodecParameters to_native_rtp_codec_parameters( + RtpCodecParameters params) { + webrtc::RtpCodecParameters native; + native.name = params.name.c_str(); + native.kind = static_cast(params.kind); + native.payload_type = params.payload_type; + + for (auto pair : params.parameters) + native.parameters.insert(std::pair(pair.key, pair.value)); + + for (auto feedback : params.rtcp_feedback) + native.rtcp_feedback.push_back(to_native_rtcp_feedback(feedback)); + + if (params.has_num_channels) + native.num_channels = params.num_channels; + + if (params.has_ptime) + native.ptime = params.ptime; + + if (params.has_max_ptime) + native.max_ptime = params.max_ptime; + + if (params.has_clock_rate) + native.clock_rate = params.clock_rate; + + return native; +} + +webrtc::RtpCapabilities to_rtp_capabilities(RtpCapabilities capabilities) { + webrtc::RtpCapabilities native; + for (auto codec : capabilities.codecs) + native.codecs.push_back(to_native_rtp_codec_capability(codec)); + + for (auto header : capabilities.header_extensions) + native.header_extensions.push_back( + to_native_rtp_header_extension_capability(header)); + + for (auto fec : capabilities.fec) + native.fec.push_back(static_cast(fec)); + + return native; +} + +webrtc::RtcpParameters to_native_rtcp_paramaters(RtcpParameters params) { + webrtc::RtcpParameters native; + if (params.has_ssrc) + native.ssrc = params.ssrc; + + native.mux = params.mux; + native.cname = params.cname.c_str(); + native.reduced_size = params.reduced_size; + return native; +} + +webrtc::RtpParameters to_native_rtp_parameters(RtpParameters params) { + webrtc::RtpParameters native; + native.transaction_id = params.transaction_id.c_str(); + native.mid = params.mid.c_str(); + + for (auto codec : params.codecs) + native.codecs.push_back(to_native_rtp_codec_parameters(codec)); + + for (auto header : params.header_extensions) + native.header_extensions.push_back(to_native_rtp_extension(header)); + + for (auto encoding : params.encodings) + native.encodings.push_back(to_native_rtp_encoding_paramters(encoding)); + + native.rtcp = to_native_rtcp_paramaters(params.rtcp); + + if (params.has_degradation_preference) + native.degradation_preference = static_cast( + params.degradation_preference); + + return native; +} + +RtcpFeedback to_rust_rtcp_feedback(webrtc::RtcpFeedback feedback) { + RtcpFeedback rust; + rust.feedback_type = static_cast(feedback.type); + + if (feedback.message_type.has_value()) { + rust.has_message_type = true; + rust.message_type = + static_cast(feedback.message_type.value()); + } + + return rust; +} + +RtpCodecCapability to_rust_rtp_codec_capability( + webrtc::RtpCodecCapability capability) { + RtpCodecCapability rust; + rust.mime_type = capability.mime_type(); + rust.name = capability.name; + rust.kind = static_cast(capability.kind); + + if (capability.clock_rate.has_value()) { + rust.has_clock_rate = true; + rust.clock_rate = capability.clock_rate.value(); + } + + if (capability.preferred_payload_type.has_value()) { + rust.has_preferred_payload_type = true; + rust.preferred_payload_type = capability.preferred_payload_type.value(); + } + + if (capability.max_ptime.has_value()) { + rust.has_max_ptime = true; + rust.max_ptime = capability.max_ptime.value(); + } + + if (capability.ptime.has_value()) { + rust.has_ptime = true; + rust.ptime = capability.ptime.value(); + } + + if (capability.num_channels.has_value()) { + rust.has_num_channels = true; + rust.num_channels = capability.num_channels.value(); + } + + for (auto feedback : capability.rtcp_feedback) + rust.rtcp_feedback.push_back(to_rust_rtcp_feedback(feedback)); + + for (auto param : capability.parameters) + rust.parameters.push_back(StringKeyValue{param.first, param.second}); + + for (auto option : capability.options) + rust.options.push_back(StringKeyValue{option.first, option.second}); + + rust.max_temporal_layer_extensions = capability.max_temporal_layer_extensions; + rust.max_spatial_layer_extensions = capability.max_spatial_layer_extensions; + rust.svc_multi_stream_support = capability.svc_multi_stream_support; + return rust; +} + +RtpHeaderExtensionCapability to_rust_rtp_header_extension_capability( + webrtc::RtpHeaderExtensionCapability header) { + RtpHeaderExtensionCapability rust; + rust.uri = header.uri; + if (header.preferred_id.has_value()) { + rust.has_preferred_id = true; + rust.preferred_id = header.preferred_id.value(); + } + + rust.preferred_encrypt = header.preferred_encrypt; + rust.direction = static_cast(header.direction); + return rust; +} + +RtpExtension to_rust_rtp_extension(webrtc::RtpExtension ext) { + RtpExtension rust; + rust.uri = ext.uri; + rust.id = ext.id; + rust.encrypt = ext.encrypt; + return rust; +} + +RtpFecParameters to_rust_rtp_fec_parameters(webrtc::RtpFecParameters fec) { + RtpFecParameters rust; + if (fec.ssrc.has_value()) { + rust.has_ssrc = true; + rust.ssrc = fec.ssrc.value(); + } + + rust.mechanism = static_cast(rust.mechanism); + return rust; +} + +RtpRtxParameters to_rust_rtp_rtx_parameters(webrtc::RtpRtxParameters param) { + RtpRtxParameters rust; + if (param.ssrc.has_value()) { + rust.has_ssrc = param.ssrc.has_value(); + rust.ssrc = param.ssrc.value(); + } + return rust; +} + +RtpEncodingParameters to_rust_rtp_encoding_parameters( + webrtc::RtpEncodingParameters params) { + RtpEncodingParameters rust; + if (params.ssrc.has_value()) { + rust.has_ssrc = params.ssrc.has_value(); + rust.ssrc = params.ssrc.value(); + } + + rust.bitrate_priority = params.bitrate_priority; + rust.network_priority = static_cast(params.network_priority); + if (params.max_bitrate_bps.has_value()) { + rust.has_max_bitrate_bps = true; + rust.max_bitrate_bps = params.max_bitrate_bps.value(); + } + + if (params.min_bitrate_bps.has_value()) { + rust.has_min_bitrate_bps = true; + rust.min_bitrate_bps = params.min_bitrate_bps.value(); + } + + if (params.max_framerate.has_value()) { + rust.has_max_framerate = true; + rust.max_framerate = params.max_framerate.value(); + } + + if (params.num_temporal_layers.has_value()) { + rust.has_num_temporal_layers = true; + rust.num_temporal_layers = params.num_temporal_layers.value(); + } + + if (params.scale_resolution_down_by.has_value()) { + rust.has_scale_resolution_down_by = true; + rust.scale_resolution_down_by = params.scale_resolution_down_by.value(); + } + + if (params.scalability_mode.has_value()) { + rust.has_scalability_mode = true; + rust.scalability_mode = params.scalability_mode.value(); + } + + rust.active = params.active; + rust.rid = params.rid; + rust.adaptive_ptime = params.adaptive_ptime; + return rust; +} + +RtpCodecParameters to_rust_rtp_codec_parameters( + webrtc::RtpCodecParameters params) { + RtpCodecParameters rust; + rust.mime_type = params.mime_type(); + rust.name = params.name; + rust.kind = static_cast(params.kind); + rust.payload_type = params.payload_type; + if (params.clock_rate.has_value()) { + rust.has_clock_rate = true; + rust.clock_rate = params.clock_rate.value(); + } + + if (params.num_channels.has_value()) { + rust.has_num_channels = true; + rust.num_channels = params.num_channels.value(); + } + + if (params.max_ptime.has_value()) { + rust.has_max_ptime = true; + rust.max_ptime = params.max_ptime.value(); + } + + if (params.ptime.has_value()) { + rust.has_ptime = true; + rust.ptime = params.ptime.value(); + } + + for (auto feedback : params.rtcp_feedback) + rust.rtcp_feedback.push_back(to_rust_rtcp_feedback(feedback)); + + for (auto pair : params.parameters) + rust.parameters.push_back(StringKeyValue{pair.first, pair.second}); + + return rust; +} + +RtpCapabilities to_rust_rtp_capabilities(webrtc::RtpCapabilities capabilities) { + RtpCapabilities rust; + for (auto codec : capabilities.codecs) + rust.codecs.push_back(to_rust_rtp_codec_capability(codec)); + + for (auto header : capabilities.header_extensions) + rust.header_extensions.push_back( + to_rust_rtp_header_extension_capability(header)); + + for (auto fec : capabilities.fec) + rust.fec.push_back(static_cast(fec)); + + return rust; +} + +RtcpParameters to_rust_rtcp_parameters(webrtc::RtcpParameters params) { + RtcpParameters rust; + if (params.ssrc.has_value()) { + rust.has_ssrc = true; + rust.ssrc = params.ssrc.value(); + } + + rust.cname = params.cname; + rust.reduced_size = params.reduced_size; + rust.mux = params.mux; + return rust; +} + +RtpParameters to_rust_rtp_parameters(webrtc::RtpParameters params) { + RtpParameters rust; + rust.transaction_id = params.transaction_id; + rust.mid = params.mid; + + for (auto codec : params.codecs) + rust.codecs.push_back(to_rust_rtp_codec_parameters(codec)); + + for (auto header : params.header_extensions) + rust.header_extensions.push_back(to_rust_rtp_extension(header)); + + for (auto encoding : params.encodings) + rust.encodings.push_back(to_rust_rtp_encoding_parameters(encoding)); + + rust.rtcp = to_rust_rtcp_parameters(params.rtcp); + + if (params.degradation_preference.has_value()) { + rust.has_degradation_preference = true; + rust.degradation_preference = static_cast( + params.degradation_preference.value()); + } + + return rust; +} + +} // namespace livekit diff --git a/webrtc-sys/src/rtp_parameters.rs b/webrtc-sys/src/rtp_parameters.rs new file mode 100644 index 0000000..cc34365 --- /dev/null +++ b/webrtc-sys/src/rtp_parameters.rs @@ -0,0 +1,194 @@ +use cxx::UniquePtr; + +pub const DEFAULT_BITRATE_PRIORITY: f64 = 1.0; + +#[cxx::bridge(namespace = "livekit")] +pub mod ffi { + + // Used to replace std::map + #[derive(Debug)] + pub struct StringKeyValue { + pub key: String, + pub value: String, + } + + #[repr(i32)] + #[derive(Debug)] + pub enum FecMechanism { + RED, + REDAndULPFEC, + FLEXFEC, + } + + #[repr(i32)] + #[derive(Debug)] + pub enum RtcpFeedbackType { + CCM, + LNTF, + NACK, + REMB, + TransportCC, + } + + #[repr(i32)] + #[derive(Debug)] + pub enum RtcpFeedbackMessageType { + GenericNACK, + PLI, + FIR, + } + + #[repr(i32)] + #[derive(Debug)] + pub enum DegradationPreference { + Disabled, + MaintainFramerate, + MaintainResolution, + Balanced, + } + + #[derive(Debug)] + pub struct RtcpFeedback { + pub feedback_type: RtcpFeedbackType, + pub has_message_type: bool, + pub message_type: RtcpFeedbackMessageType, + } + + #[derive(Debug)] + pub struct RtpCodecCapability { + pub mime_type: String, // filled with mime_type fnc + pub name: String, + pub kind: MediaType, + pub has_clock_rate: bool, + pub clock_rate: i32, + pub has_preferred_payload_type: bool, + pub preferred_payload_type: i32, + pub has_max_ptime: bool, + pub max_ptime: i32, + pub has_ptime: bool, + pub ptime: i32, + pub has_num_channels: bool, + pub num_channels: i32, + pub rtcp_feedback: Vec, + pub parameters: Vec, + pub options: Vec, + pub max_temporal_layer_extensions: i32, + pub max_spatial_layer_extensions: i32, + pub svc_multi_stream_support: bool, + } + + #[derive(Debug)] + pub struct RtpHeaderExtensionCapability { + pub uri: String, + pub has_preferred_id: bool, + pub preferred_id: i32, + pub preferred_encrypt: bool, + pub direction: RtpTransceiverDirection, + } + + #[repr(i32)] + #[derive(Debug)] + pub enum RtpExtensionFilter { + DiscardEncryptedExtension, + PreferEncryptedExtension, + RequireEncryptedExtension, + } + + #[derive(Debug)] + pub struct RtpExtension { + // TODO(theomonnom): export available URI inside api/rtp_parameters.h + pub uri: String, + pub id: i32, + pub encrypt: bool, + } + + #[derive(Debug)] + pub struct RtpFecParameters { + pub has_ssrc: bool, + pub ssrc: u32, + pub mechanism: FecMechanism, + } + + #[derive(Debug)] + pub struct RtpRtxParameters { + pub has_ssrc: bool, + pub ssrc: u32, + } + + #[derive(Debug)] + pub struct RtpEncodingParameters { + pub has_ssrc: bool, + pub ssrc: u32, + pub bitrate_priority: f64, + pub network_priority: Priority, // Todo link type + pub has_max_bitrate_bps: bool, + pub max_bitrate_bps: i32, + pub has_min_bitrate_bps: bool, + pub min_bitrate_bps: i32, + pub has_max_framerate: bool, + pub max_framerate: f64, + pub has_num_temporal_layers: bool, + pub num_temporal_layers: i32, + pub has_scale_resolution_down_by: bool, + pub scale_resolution_down_by: f64, + pub has_scalability_mode: bool, + pub scalability_mode: String, + pub active: bool, + pub rid: String, + pub adaptive_ptime: bool, + } + + #[derive(Debug)] + pub struct RtpCodecParameters { + pub mime_type: String, // filled with mime_type fnc + pub name: String, + pub kind: MediaType, + pub payload_type: i32, + pub has_clock_rate: bool, + pub clock_rate: i32, + pub has_num_channels: bool, + pub num_channels: i32, + pub has_max_ptime: bool, + pub max_ptime: i32, + pub has_ptime: bool, + pub ptime: i32, + pub rtcp_feedback: Vec, + pub parameters: Vec, + } + + #[derive(Debug)] + pub struct RtpCapabilities { + pub codecs: Vec, + pub header_extensions: Vec, + pub fec: Vec, + } + + #[derive(Debug)] + pub struct RtcpParameters { + pub has_ssrc: bool, + pub ssrc: u32, + pub cname: String, + pub reduced_size: bool, + pub mux: bool, + } + + #[derive(Debug)] + pub struct RtpParameters { + pub transaction_id: String, + pub mid: String, + pub codecs: Vec, + pub header_extensions: Vec, + pub encodings: Vec, + pub rtcp: RtcpParameters, + pub has_degradation_preference: bool, + pub degradation_preference: DegradationPreference, + } + + extern "C++" { + include!("webrtc-sys/src/webrtc.rs.h"); + + type Priority = crate::webrtc::ffi::Priority; + type MediaType = crate::webrtc::ffi::MediaType; + type RtpTransceiverDirection = crate::webrtc::ffi::RtpTransceiverDirection; + } +} diff --git a/webrtc-sys/src/rtp_receiver.cpp b/webrtc-sys/src/rtp_receiver.cpp index a7b70c1..b756896 100644 --- a/webrtc-sys/src/rtp_receiver.cpp +++ b/webrtc-sys/src/rtp_receiver.cpp @@ -4,13 +4,48 @@ #include "livekit/rtp_receiver.h" +#include "absl/types/optional.h" + namespace livekit { + RtpReceiver::RtpReceiver( rtc::scoped_refptr receiver) : receiver_(std::move(receiver)) {} -std::unique_ptr RtpReceiver::track() const { +std::shared_ptr RtpReceiver::track() const { return MediaStreamTrack::from(receiver_->track()); } -} // namespace livekit \ No newline at end of file +rust::Vec RtpReceiver::stream_ids() const { + rust::Vec rust; + for (auto id : receiver_->stream_ids()) + rust.push_back(id); + return rust; +} + +rust::Vec RtpReceiver::streams() const { + rust::Vec rust; + for (auto stream : receiver_->streams()) + rust.push_back(MediaStreamPtr{std::make_shared(stream)}); + return rust; +} + +MediaType RtpReceiver::media_type() const { + return static_cast(receiver_->media_type()); +} + +rust::String RtpReceiver::id() const { + return receiver_->id(); +} + +RtpParameters RtpReceiver::get_parameters() const { + return to_rust_rtp_parameters(receiver_->GetParameters()); +} + +void RtpReceiver::set_jitter_buffer_minimum_delay(bool is_some, + double delay_seconds) const { + receiver_->SetJitterBufferMinimumDelay( + is_some ? absl::make_optional(delay_seconds) : absl::nullopt); +} + +} // namespace livekit diff --git a/webrtc-sys/src/rtp_receiver.rs b/webrtc-sys/src/rtp_receiver.rs index a349fa6..9a67775 100644 --- a/webrtc-sys/src/rtp_receiver.rs +++ b/webrtc-sys/src/rtp_receiver.rs @@ -1,18 +1,36 @@ +use crate::impl_thread_safety; + #[cxx::bridge(namespace = "livekit")] pub mod ffi { - unsafe extern "C++" { - include!("livekit/rtp_receiver.h"); + + extern "C++" { + include!("livekit/webrtc.h"); + include!("livekit/rtp_parameters.h"); + include!("livekit/helper.h"); include!("livekit/media_stream.h"); + type MediaType = crate::webrtc::ffi::MediaType; + type RtpParameters = crate::rtp_parameters::ffi::RtpParameters; + type MediaStreamPtr = crate::helper::ffi::MediaStreamPtr; type MediaStreamTrack = crate::media_stream::ffi::MediaStreamTrack; + type MediaStream = crate::media_stream::ffi::MediaStream; + } + + unsafe extern "C++" { + include!("livekit/rtp_receiver.h"); + type RtpReceiver; - fn track(self: &RtpReceiver) -> UniquePtr; + fn track(self: &RtpReceiver) -> SharedPtr; + fn stream_ids(self: &RtpReceiver) -> Vec; + fn streams(self: &RtpReceiver) -> Vec; + fn media_type(self: &RtpReceiver) -> MediaType; + fn id(self: &RtpReceiver) -> String; + fn get_parameters(self: &RtpReceiver) -> RtpParameters; + fn set_jitter_buffer_minimum_delay(self: &RtpReceiver, is_some: bool, delay_seconds: f64); - fn _unique_rtp_receiver() -> UniquePtr; // Ignore + fn _shared_rtp_receiver() -> SharedPtr; } } -unsafe impl Sync for ffi::RtpReceiver {} - -unsafe impl Send for ffi::RtpReceiver {} +impl_thread_safety!(ffi::RtpReceiver, Send + Sync); diff --git a/webrtc-sys/src/rtp_sender.cpp b/webrtc-sys/src/rtp_sender.cpp new file mode 100644 index 0000000..4f0bf27 --- /dev/null +++ b/webrtc-sys/src/rtp_sender.cpp @@ -0,0 +1,58 @@ +#include "livekit/rtp_sender.h" + +namespace livekit { + +RtpSender::RtpSender(rtc::scoped_refptr sender) + : sender_(std::move(sender)) {} + +bool RtpSender::set_track(std::shared_ptr track) const { + return sender_->SetTrack(track->get().get()); +} + +std::shared_ptr RtpSender::track() const { + return MediaStreamTrack::from(sender_->track()); +} + +uint32_t RtpSender::ssrc() const { + return sender_->ssrc(); +} + +MediaType RtpSender::media_type() const { + return static_cast(sender_->media_type()); +} + +rust::String RtpSender::id() const { + return sender_->id(); +} + +rust::Vec RtpSender::stream_ids() const { + rust::Vec vec; + for (auto str : sender_->stream_ids()) + vec.push_back(str); + + return vec; +} + +void RtpSender::set_streams(const rust::Vec& stream_ids) const { + std::vector std_stream_ids(stream_ids.begin(), stream_ids.end()); + sender_->SetStreams(std_stream_ids); +} + +rust::Vec RtpSender::init_send_encodings() const { + rust::Vec encodings; + for (auto encoding : sender_->init_send_encodings()) + encodings.push_back(to_rust_rtp_encoding_parameters(encoding)); + return encodings; +} + +RtpParameters RtpSender::get_parameters() const { + return to_rust_rtp_parameters(sender_->GetParameters()); +} + +void RtpSender::set_parameters(RtpParameters params) const { + auto error = sender_->SetParameters(to_native_rtp_parameters(params)); + if (!error.ok()) + throw std::runtime_error(serialize_error(to_error(error))); +} + +} // namespace livekit diff --git a/webrtc-sys/src/rtp_sender.rs b/webrtc-sys/src/rtp_sender.rs new file mode 100644 index 0000000..01ed519 --- /dev/null +++ b/webrtc-sys/src/rtp_sender.rs @@ -0,0 +1,33 @@ +#[cxx::bridge(namespace = "livekit")] +pub mod ffi { + + extern "C++" { + include!("livekit/webrtc.h"); + include!("livekit/rtp_parameters.h"); + include!("livekit/media_stream.h"); + + type MediaType = crate::webrtc::ffi::MediaType; + type RtpEncodingParameters = crate::rtp_parameters::ffi::RtpEncodingParameters; + type RtpParameters = crate::rtp_parameters::ffi::RtpParameters; + type MediaStreamTrack = crate::media_stream::ffi::MediaStreamTrack; + } + + unsafe extern "C++" { + include!("livekit/rtp_sender.h"); + + type RtpSender; + + fn set_track(self: &RtpSender, track: SharedPtr) -> bool; + fn track(self: &RtpSender) -> SharedPtr; + fn ssrc(self: &RtpSender) -> u32; + fn media_type(self: &RtpSender) -> MediaType; + fn id(self: &RtpSender) -> String; + fn stream_ids(self: &RtpSender) -> Vec; + fn set_streams(self: &RtpSender, stream_ids: &Vec); + fn init_send_encodings(self: &RtpSender) -> Vec; + fn get_parameters(self: &RtpSender) -> RtpParameters; + fn set_parameters(self: &RtpSender, parameters: RtpParameters) -> Result<()>; + + fn _shared_rtp_sender() -> SharedPtr; + } +} diff --git a/webrtc-sys/src/rtp_transceiver.cpp b/webrtc-sys/src/rtp_transceiver.cpp index 84d37d9..a004edb 100644 --- a/webrtc-sys/src/rtp_transceiver.cpp +++ b/webrtc-sys/src/rtp_transceiver.cpp @@ -5,7 +5,128 @@ #include "livekit/rtp_transceiver.h" namespace livekit { + +webrtc::RtpTransceiverInit to_native_rtp_transceiver_init( + RtpTransceiverInit init) { + { + webrtc::RtpTransceiverInit native; + native.direction = + static_cast(init.direction); + native.stream_ids = std::vector(init.stream_ids.begin(), + init.stream_ids.end()); + for (auto encoding : init.send_encodings) + native.send_encodings.push_back( + to_native_rtp_encoding_paramters(encoding)); + return native; + } +} + RtpTransceiver::RtpTransceiver( rtc::scoped_refptr transceiver) : transceiver_(std::move(transceiver)) {} -} // namespace livekit \ No newline at end of file + +MediaType RtpTransceiver::media_type() const { + return static_cast(transceiver_->media_type()); +} + +rust::String RtpTransceiver::mid() const { + // The error/Result is converted into an Option in Rust (Wait for Option + // suport in cxx.rs) (value throws an error if there's no value) + return transceiver_->mid().value(); +} + +std::shared_ptr RtpTransceiver::sender() const { + return std::make_shared(transceiver_->sender()); +} + +std::shared_ptr RtpTransceiver::receiver() const { + return std::make_shared(transceiver_->receiver()); +} + +bool RtpTransceiver::stopped() const { + return transceiver_->stopped(); +} + +bool RtpTransceiver::stopping() const { + return transceiver_->stopping(); +} + +RtpTransceiverDirection RtpTransceiver::direction() const { + return static_cast(transceiver_->direction()); +} + +void RtpTransceiver::set_direction(RtpTransceiverDirection direction) const { + auto error = transceiver_->SetDirectionWithError( + static_cast(direction)); + + if (!error.ok()) { + throw std::runtime_error(serialize_error(to_error(error))); + } +} + +RtpTransceiverDirection RtpTransceiver::current_direction() const { + return static_cast( + transceiver_->current_direction().value()); +} + +RtpTransceiverDirection RtpTransceiver::fired_direction() const { + return static_cast( + transceiver_->fired_direction().value()); +} + +void RtpTransceiver::stop_standard() const { + auto error = transceiver_->StopStandard(); + if (!error.ok()) + throw std::runtime_error(serialize_error(to_error(error))); +} + +void RtpTransceiver::set_codec_preferences( + rust::Vec codecs) const { + std::vector std_codecs; + for (auto codec : codecs) + std_codecs.push_back(to_native_rtp_codec_capability(codec)); + + auto error = transceiver_->SetCodecPreferences(std_codecs); + if (!error.ok()) + throw std::runtime_error(serialize_error(to_error(error))); +} + +rust::Vec RtpTransceiver::codec_preferences() const { + rust::Vec rust; + for (auto codec : transceiver_->codec_preferences()) + rust.push_back(to_rust_rtp_codec_capability(codec)); + + return rust; +} + +rust::Vec +RtpTransceiver::header_extensions_to_offer() const { + rust::Vec rust; + for (auto header : transceiver_->HeaderExtensionsToOffer()) + rust.push_back(to_rust_rtp_header_extension_capability(header)); + + return rust; +} + +rust::Vec +RtpTransceiver::header_extensions_negotiated() const { + rust::Vec rust; + for (auto header : transceiver_->HeaderExtensionsNegotiated()) + rust.push_back(to_rust_rtp_header_extension_capability(header)); + + return rust; +} + +void RtpTransceiver::set_offered_rtp_header_extensions( + rust::Vec header_extensions_to_offer) const { + std::vector headers; + + for (auto header : header_extensions_to_offer) + headers.push_back(to_native_rtp_header_extension_capability(header)); + + auto error = transceiver_->SetOfferedRtpHeaderExtensions(headers); + if (!error.ok()) + throw std::runtime_error(serialize_error(to_error(error))); +} + +} // namespace livekit diff --git a/webrtc-sys/src/rtp_transceiver.rs b/webrtc-sys/src/rtp_transceiver.rs index 1350f72..19b097a 100644 --- a/webrtc-sys/src/rtp_transceiver.rs +++ b/webrtc-sys/src/rtp_transceiver.rs @@ -1,10 +1,62 @@ +use crate::impl_thread_safety; + #[cxx::bridge(namespace = "livekit")] pub mod ffi { + + #[derive(Debug)] + pub struct RtpTransceiverInit { + pub direction: RtpTransceiverDirection, + pub stream_ids: Vec, + pub send_encodings: Vec, + } + + extern "C++" { + include!("livekit/webrtc.h"); + include!("livekit/rtp_parameters.h"); + include!("livekit/rtp_sender.h"); + include!("livekit/rtp_receiver.h"); + + type MediaType = crate::webrtc::ffi::MediaType; + type RtpTransceiverDirection = crate::webrtc::ffi::RtpTransceiverDirection; + type RtpEncodingParameters = crate::rtp_parameters::ffi::RtpEncodingParameters; + type RtpCodecCapability = crate::rtp_parameters::ffi::RtpCodecCapability; + type RtpHeaderExtensionCapability = + crate::rtp_parameters::ffi::RtpHeaderExtensionCapability; + type RtpSender = crate::rtp_sender::ffi::RtpSender; + type RtpReceiver = crate::rtp_receiver::ffi::RtpReceiver; + } + unsafe extern "C++" { include!("livekit/rtp_transceiver.h"); type RtpTransceiver; - fn _unique_rtp_transceiver() -> UniquePtr; // Ignore + fn media_type(self: &RtpTransceiver) -> MediaType; + fn mid(self: &RtpTransceiver) -> Result; + fn sender(self: &RtpTransceiver) -> SharedPtr; + fn receiver(self: &RtpTransceiver) -> SharedPtr; + fn stopped(self: &RtpTransceiver) -> bool; + fn stopping(self: &RtpTransceiver) -> bool; + fn direction(self: &RtpTransceiver) -> RtpTransceiverDirection; + fn set_direction(self: &RtpTransceiver, direction: RtpTransceiverDirection) -> Result<()>; + fn current_direction(self: &RtpTransceiver) -> Result; + fn fired_direction(self: &RtpTransceiver) -> Result; + fn stop_standard(self: &RtpTransceiver) -> Result<()>; + fn set_codec_preferences( + self: &RtpTransceiver, + codecs: Vec, + ) -> Result<()>; + fn codec_preferences(self: &RtpTransceiver) -> Vec; + fn header_extensions_to_offer(self: &RtpTransceiver) -> Vec; + fn header_extensions_negotiated(self: &RtpTransceiver) + -> Vec; + fn set_offered_rtp_header_extensions( + self: &RtpTransceiver, + headers: Vec, + ) -> Result<()>; + + fn _shared_rtp_transceiver() -> SharedPtr; } } + +impl_thread_safety!(ffi::RtpTransceiver, Send + Sync); diff --git a/webrtc-sys/src/video_frame.cpp b/webrtc-sys/src/video_frame.cpp new file mode 100644 index 0000000..eefdf3f --- /dev/null +++ b/webrtc-sys/src/video_frame.cpp @@ -0,0 +1,74 @@ +#include "livekit/video_frame.h" + +#include + +#include "api/video/video_frame.h" + +namespace livekit { +VideoFrame::VideoFrame(const webrtc::VideoFrame& frame) + : frame_(std::move(frame)) {} + +int VideoFrame::width() const { + return frame_.width(); +} +int VideoFrame::height() const { + return frame_.height(); +} +uint32_t VideoFrame::size() const { + return frame_.size(); +} +uint16_t VideoFrame::id() const { + return frame_.id(); +} +int64_t VideoFrame::timestamp_us() const { + return frame_.timestamp_us(); +} +int64_t VideoFrame::ntp_time_ms() const { + return frame_.ntp_time_ms(); +} +uint32_t VideoFrame::transport_frame_id() const { + return frame_.transport_frame_id(); +} +uint32_t VideoFrame::timestamp() const { + return frame_.timestamp(); +} + +VideoRotation VideoFrame::rotation() const { + return static_cast(frame_.rotation()); +} + +// TODO(theomonnom) This shouldn't create a new shared_ptr at each call +std::unique_ptr VideoFrame::video_frame_buffer() const { + return std::make_unique(frame_.video_frame_buffer()); +} + +webrtc::VideoFrame VideoFrame::get() const { + return frame_; +} + +void VideoFrameBuilder::set_video_frame_buffer( + std::unique_ptr buffer) { + builder_.set_video_frame_buffer(buffer->get()); +} + +void VideoFrameBuilder::set_timestamp_us(int64_t timestamp_us) { + builder_.set_timestamp_us(timestamp_us); +} + +void VideoFrameBuilder::set_rotation(VideoRotation rotation) { + builder_.set_rotation(static_cast(rotation)); +} + +void VideoFrameBuilder::set_id(uint16_t id) { + builder_.set_id(id); +} + +std::unique_ptr VideoFrameBuilder::build() { + return std::make_unique(builder_.build()); +} + +std::unique_ptr create_video_frame_builder() { + return std::make_unique(); +} + +} // namespace livekit diff --git a/webrtc-sys/src/video_frame.rs b/webrtc-sys/src/video_frame.rs index 3247b82..6bf6f48 100644 --- a/webrtc-sys/src/video_frame.rs +++ b/webrtc-sys/src/video_frame.rs @@ -11,12 +11,17 @@ pub mod ffi { VideoRotation270 = 270, } - unsafe extern "C++" { - include!("livekit/video_frame.h"); + extern "C++" { include!("livekit/video_frame_buffer.h"); - type VideoFrame; type VideoFrameBuffer = crate::video_frame_buffer::ffi::VideoFrameBuffer; + } + + unsafe extern "C++" { + include!("livekit/video_frame.h"); + + type VideoFrame; + type VideoFrameBuilder; fn width(self: &VideoFrame) -> i32; fn height(self: &VideoFrame) -> i32; @@ -29,7 +34,16 @@ pub mod ffi { fn rotation(self: &VideoFrame) -> VideoRotation; fn video_frame_buffer(self: &VideoFrame) -> UniquePtr; - fn _unique_video_frame() -> UniquePtr; // Ignore + fn set_video_frame_buffer( + self: Pin<&mut VideoFrameBuilder>, + buffer: UniquePtr, + ); + fn set_timestamp_us(self: Pin<&mut VideoFrameBuilder>, timestamp_us: i64); + fn set_rotation(self: Pin<&mut VideoFrameBuilder>, rotation: VideoRotation); + fn set_id(self: Pin<&mut VideoFrameBuilder>, id: u16); + fn build(self: Pin<&mut VideoFrameBuilder>) -> UniquePtr; + + fn create_video_frame_builder() -> UniquePtr; } } diff --git a/webrtc-sys/src/video_frame_buffer.cpp b/webrtc-sys/src/video_frame_buffer.cpp new file mode 100644 index 0000000..3f3bb62 --- /dev/null +++ b/webrtc-sys/src/video_frame_buffer.cpp @@ -0,0 +1,198 @@ +#include "livekit/video_frame_buffer.h" + +namespace livekit { + +VideoFrameBuffer::VideoFrameBuffer( + rtc::scoped_refptr buffer) + : buffer_(std::move(buffer)) {} + +VideoFrameBufferType VideoFrameBuffer::buffer_type() const { + return static_cast(buffer_->type()); +} + +int VideoFrameBuffer::width() const { + return buffer_->width(); +} + +int VideoFrameBuffer::height() const { + return buffer_->height(); +} + +std::unique_ptr VideoFrameBuffer::to_i420() { + return std::make_unique(buffer_->ToI420()); +} + +// const_cast is valid here because we take the ownership on the rust side +std::unique_ptr VideoFrameBuffer::get_i420() { + return std::make_unique( + rtc::scoped_refptr( + const_cast(buffer_->GetI420()))); +} + +std::unique_ptr VideoFrameBuffer::get_i420a() { + return std::make_unique( + rtc::scoped_refptr( + const_cast(buffer_->GetI420A()))); +} + +std::unique_ptr VideoFrameBuffer::get_i422() { + return std::make_unique( + rtc::scoped_refptr( + const_cast(buffer_->GetI422()))); +} + +std::unique_ptr VideoFrameBuffer::get_i444() { + return std::make_unique( + rtc::scoped_refptr( + const_cast(buffer_->GetI444()))); +} + +std::unique_ptr VideoFrameBuffer::get_i010() { + return std::make_unique( + rtc::scoped_refptr( + const_cast(buffer_->GetI010()))); +} + +std::unique_ptr VideoFrameBuffer::get_nv12() { + return std::make_unique( + rtc::scoped_refptr( + const_cast(buffer_->GetNV12()))); +} + +rtc::scoped_refptr VideoFrameBuffer::get() const { + return buffer_; +} + +PlanarYuvBuffer::PlanarYuvBuffer( + rtc::scoped_refptr buffer) + : VideoFrameBuffer(buffer) {} + +int PlanarYuvBuffer::chroma_width() const { + return buffer()->ChromaWidth(); +} + +int PlanarYuvBuffer::chroma_height() const { + return buffer()->ChromaHeight(); +} + +int PlanarYuvBuffer::stride_y() const { + return buffer()->StrideY(); +} + +int PlanarYuvBuffer::stride_u() const { + return buffer()->StrideU(); +} + +int PlanarYuvBuffer::stride_v() const { + return buffer()->StrideV(); +} + +webrtc::PlanarYuvBuffer* PlanarYuvBuffer::buffer() const { + return static_cast(buffer_.get()); +} + +PlanarYuv8Buffer::PlanarYuv8Buffer( + rtc::scoped_refptr buffer) + : PlanarYuvBuffer(buffer) {} + +const uint8_t* PlanarYuv8Buffer::data_y() const { + return buffer()->DataY(); +} + +const uint8_t* PlanarYuv8Buffer::data_u() const { + return buffer()->DataU(); +} + +const uint8_t* PlanarYuv8Buffer::data_v() const { + return buffer()->DataV(); +} + +webrtc::PlanarYuv8Buffer* PlanarYuv8Buffer::buffer() const { + return static_cast(buffer_.get()); +} + +PlanarYuv16BBuffer::PlanarYuv16BBuffer( + rtc::scoped_refptr buffer) + : PlanarYuvBuffer(buffer) {} + +const uint16_t* PlanarYuv16BBuffer::data_y() const { + return buffer()->DataY(); +} + +const uint16_t* PlanarYuv16BBuffer::data_u() const { + return buffer()->DataU(); +} + +const uint16_t* PlanarYuv16BBuffer::data_v() const { + return buffer()->DataV(); +} + +webrtc::PlanarYuv16BBuffer* PlanarYuv16BBuffer::buffer() const { + return static_cast(buffer_.get()); +} + +BiplanarYuvBuffer::BiplanarYuvBuffer( + rtc::scoped_refptr buffer) + : VideoFrameBuffer(buffer) {} + +int BiplanarYuvBuffer::chroma_width() const { + return buffer()->ChromaWidth(); +} + +int BiplanarYuvBuffer::chroma_height() const { + return buffer()->ChromaHeight(); +} + +int BiplanarYuvBuffer::stride_y() const { + return buffer()->StrideY(); +} + +int BiplanarYuvBuffer::stride_uv() const { + return buffer()->StrideUV(); +} + +webrtc::BiplanarYuvBuffer* BiplanarYuvBuffer::buffer() const { + return static_cast(buffer_.get()); +} + +BiplanarYuv8Buffer::BiplanarYuv8Buffer( + rtc::scoped_refptr buffer) + : BiplanarYuvBuffer(buffer) {} + +const uint8_t* BiplanarYuv8Buffer::data_y() const { + return buffer()->DataY(); +} + +const uint8_t* BiplanarYuv8Buffer::data_uv() const { + return buffer()->DataUV(); +} + +webrtc::BiplanarYuv8Buffer* BiplanarYuv8Buffer::buffer() const { + return static_cast(buffer_.get()); +} + +std::unique_ptr create_i420_buffer(int width, int height) { + return std::make_unique( + webrtc::I420Buffer::Create(width, height)); +} + +I420Buffer::I420Buffer(rtc::scoped_refptr buffer) + : PlanarYuv8Buffer(buffer) {} + +I420ABuffer::I420ABuffer( + rtc::scoped_refptr buffer) + : I420Buffer(buffer) {} + +I422Buffer::I422Buffer(rtc::scoped_refptr buffer) + : PlanarYuv8Buffer(buffer) {} + +I444Buffer::I444Buffer(rtc::scoped_refptr buffer) + : PlanarYuv8Buffer(buffer) {} + +I010Buffer::I010Buffer(rtc::scoped_refptr buffer) + : PlanarYuv16BBuffer(buffer) {} + +NV12Buffer::NV12Buffer(rtc::scoped_refptr buffer) + : BiplanarYuv8Buffer(buffer) {} + +} // namespace livekit diff --git a/webrtc-sys/src/video_frame_buffer.rs b/webrtc-sys/src/video_frame_buffer.rs index 420a63c..489e4d3 100644 --- a/webrtc-sys/src/video_frame_buffer.rs +++ b/webrtc-sys/src/video_frame_buffer.rs @@ -66,6 +66,8 @@ pub mod ffi { fn data_y(self: &BiplanarYuv8Buffer) -> *const u8; fn data_uv(self: &BiplanarYuv8Buffer) -> *const u8; + fn create_i420_buffer(width: i32, height: i32) -> UniquePtr; + unsafe fn yuv_to_vfb(yuv: *const PlanarYuvBuffer) -> *const VideoFrameBuffer; unsafe fn biyuv_to_vfb(yuv: *const BiplanarYuvBuffer) -> *const VideoFrameBuffer; unsafe fn yuv8_to_yuv(yuv8: *const PlanarYuv8Buffer) -> *const PlanarYuvBuffer; diff --git a/webrtc-sys/src/webrtc.rs b/webrtc-sys/src/webrtc.rs index f309843..1fc995f 100644 --- a/webrtc-sys/src/webrtc.rs +++ b/webrtc-sys/src/webrtc.rs @@ -1,5 +1,36 @@ +use crate::impl_thread_safety; + #[cxx::bridge(namespace = "livekit")] pub mod ffi { + + #[derive(Debug)] + #[repr(i32)] + pub enum MediaType { + Audio, + Video, + Data, + Unsupported, + } + + #[derive(Debug)] + #[repr(i32)] + pub enum Priority { + VeryLow, + Low, + Medium, + High, + } + + #[derive(Debug)] + #[repr(i32)] + pub enum RtpTransceiverDirection { + SendRecv, + SendOnly, + RecvOnly, + Inactive, + Stopped, + } + unsafe extern "C++" { include!("livekit/webrtc.h"); @@ -9,6 +40,4 @@ pub mod ffi { } } -unsafe impl Send for ffi::RTCRuntime {} - -unsafe impl Sync for ffi::RTCRuntime {} +impl_thread_safety!(ffi::RTCRuntime, Send + Sync);