From 55bda13069ac8baa924002981670c59f5d715943 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sun, 4 Jun 2023 01:02:00 +0200 Subject: [PATCH] cleanup: webrtc-sys & fix RtcRuntime disposing crashes (#81) --- examples/Cargo.lock | 110 ++++++- examples/wgpu_room/src/sine_track.rs | 11 +- livekit-ffi/src/server/audio_frame.rs | 2 +- livekit-ffi/src/server/video_frame.rs | 2 +- livekit-webrtc/src/audio_stream.rs | 2 +- livekit-webrtc/src/audio_track.rs | 23 ++ livekit-webrtc/src/lib.rs | 3 + livekit-webrtc/src/media_stream.rs | 110 +------ livekit-webrtc/src/media_stream_track.rs | 74 +++++ livekit-webrtc/src/native/audio_source.rs | 22 +- livekit-webrtc/src/native/audio_stream.rs | 39 +-- livekit-webrtc/src/native/audio_track.rs | 18 ++ livekit-webrtc/src/native/data_channel.rs | 25 +- livekit-webrtc/src/native/media_stream.rs | 97 +----- .../src/native/media_stream_track.rs | 67 ++++ livekit-webrtc/src/native/mod.rs | 13 +- livekit-webrtc/src/native/peer_connection.rs | 190 ++++++------ .../src/native/peer_connection_factory.rs | 144 ++++----- livekit-webrtc/src/native/rtp_receiver.rs | 5 +- livekit-webrtc/src/native/rtp_sender.rs | 12 +- livekit-webrtc/src/native/rtp_transceiver.rs | 4 +- livekit-webrtc/src/native/video_source.rs | 13 +- livekit-webrtc/src/native/video_stream.rs | 40 +-- livekit-webrtc/src/native/video_track.rs | 18 ++ livekit-webrtc/src/peer_connection.rs | 3 +- livekit-webrtc/src/peer_connection_factory.rs | 13 +- livekit-webrtc/src/prelude.rs | 9 +- livekit-webrtc/src/rtp_receiver.rs | 3 +- livekit-webrtc/src/rtp_sender.rs | 2 +- livekit-webrtc/src/video_stream.rs | 2 +- livekit-webrtc/src/video_track.rs | 23 ++ .../room/participant/remote_participant.rs | 5 +- livekit/src/room/track/local_audio_track.rs | 29 +- livekit/src/room/track/local_video_track.rs | 29 +- livekit/src/room/track/mod.rs | 34 +- livekit/src/room/track/remote_audio_track.rs | 26 +- livekit/src/room/track/remote_video_track.rs | 26 +- webrtc-sys/build.rs | 8 +- webrtc-sys/include/livekit/audio_track.h | 134 ++++++++ webrtc-sys/include/livekit/data_channel.h | 29 +- webrtc-sys/include/livekit/jsep.h | 62 ++-- webrtc-sys/include/livekit/logsink.h | 46 --- webrtc-sys/include/livekit/media_stream.h | 216 +------------ .../include/livekit/media_stream_track.h | 60 ++++ webrtc-sys/include/livekit/peer_connection.h | 80 ++--- .../include/livekit/peer_connection_factory.h | 22 +- webrtc-sys/include/livekit/rtc_error.h | 4 +- webrtc-sys/include/livekit/rtp_receiver.h | 10 +- webrtc-sys/include/livekit/rtp_sender.h | 8 +- webrtc-sys/include/livekit/rtp_transceiver.h | 4 +- webrtc-sys/include/livekit/video_track.h | 133 ++++++++ webrtc-sys/include/livekit/webrtc.h | 72 ++++- webrtc-sys/src/audio_track.cpp | 143 +++++++++ webrtc-sys/src/audio_track.rs | 71 +++++ webrtc-sys/src/data_channel.cpp | 69 +++-- webrtc-sys/src/data_channel.rs | 48 +-- webrtc-sys/src/jsep.cpp | 61 ++-- webrtc-sys/src/jsep.rs | 80 +---- webrtc-sys/src/lib.rs | 4 +- webrtc-sys/src/logsink.cpp | 40 --- webrtc-sys/src/logsink.rs | 24 -- webrtc-sys/src/media_stream.cpp | 291 +----------------- webrtc-sys/src/media_stream.rs | 173 +---------- webrtc-sys/src/media_stream_track.cpp | 58 ++++ webrtc-sys/src/media_stream_track.rs | 26 ++ webrtc-sys/src/peer_connection.cpp | 156 ++++++---- webrtc-sys/src/peer_connection.rs | 158 +++------- webrtc-sys/src/peer_connection_factory.cpp | 92 +++--- webrtc-sys/src/peer_connection_factory.rs | 39 +-- webrtc-sys/src/rtc_error.cpp | 22 +- webrtc-sys/src/rtc_error.rs | 38 ++- webrtc-sys/src/rtp_parameters.rs | 2 +- webrtc-sys/src/rtp_receiver.cpp | 10 +- webrtc-sys/src/rtp_sender.cpp | 9 +- webrtc-sys/src/rtp_transceiver.cpp | 7 +- webrtc-sys/src/rtp_transceiver.rs | 2 +- webrtc-sys/src/video_track.cpp | 188 +++++++++++ webrtc-sys/src/video_track.rs | 97 ++++++ webrtc-sys/src/webrtc.cpp | 100 +++++- webrtc-sys/src/webrtc.rs | 19 +- 80 files changed, 2209 insertions(+), 1954 deletions(-) create mode 100644 livekit-webrtc/src/audio_track.rs create mode 100644 livekit-webrtc/src/media_stream_track.rs create mode 100644 livekit-webrtc/src/native/audio_track.rs create mode 100644 livekit-webrtc/src/native/media_stream_track.rs create mode 100644 livekit-webrtc/src/native/video_track.rs create mode 100644 livekit-webrtc/src/video_track.rs create mode 100644 webrtc-sys/include/livekit/audio_track.h delete mode 100644 webrtc-sys/include/livekit/logsink.h create mode 100644 webrtc-sys/include/livekit/media_stream_track.h create mode 100644 webrtc-sys/include/livekit/video_track.h create mode 100644 webrtc-sys/src/audio_track.cpp create mode 100644 webrtc-sys/src/audio_track.rs delete mode 100644 webrtc-sys/src/logsink.cpp delete mode 100644 webrtc-sys/src/logsink.rs create mode 100644 webrtc-sys/src/media_stream_track.cpp create mode 100644 webrtc-sys/src/media_stream_track.rs create mode 100644 webrtc-sys/src/video_track.cpp create mode 100644 webrtc-sys/src/video_track.rs diff --git a/examples/Cargo.lock b/examples/Cargo.lock index 7c60bb5..03d805c 100644 --- a/examples/Cargo.lock +++ b/examples/Cargo.lock @@ -841,7 +841,7 @@ dependencies = [ "futures-sink", "nanorand", "pin-project", - "spin", + "spin 0.9.8", ] [[package]] @@ -2352,6 +2352,21 @@ dependencies = [ "winreg", ] +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin 0.5.2", + "untrusted", + "web-sys", + "winapi", +] + [[package]] name = "rustc-demangle" version = "0.1.23" @@ -2378,6 +2393,49 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "rustls" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c911ba11bc8433e811ce56fde130ccf32f5127cab0e0194e9c68c5a5b671791e" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-native-certs" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" +dependencies = [ + "base64", +] + +[[package]] +name = "rustls-webpki" +version = "0.100.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "ryu" version = "1.0.13" @@ -2431,6 +2489,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "sctk-adwaita" version = "0.5.4" @@ -2619,6 +2687,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "spin" version = "0.9.8" @@ -2854,6 +2928,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0d409377ff5b1e3ca6437aa86c1eb7d40c134bfec254e44c830defa92669db5" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.14" @@ -2873,9 +2957,10 @@ checksum = "ec509ac96e9a0c43427c74f003127d953a265737636129424288d27cb5c4b12c" dependencies = [ "futures-util", "log", - "native-tls", + "rustls", + "rustls-native-certs", "tokio", - "tokio-native-tls", + "tokio-rustls", "tungstenite", ] @@ -2998,12 +3083,13 @@ dependencies = [ "http", "httparse", "log", - "native-tls", "rand", + "rustls", "sha1", "thiserror", "url", "utf-8", + "webpki", ] [[package]] @@ -3054,6 +3140,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "url" version = "2.3.1" @@ -3287,6 +3379,16 @@ dependencies = [ "web-sys", ] +[[package]] +name = "webpki" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "webrtc-sys" version = "0.1.1" diff --git a/examples/wgpu_room/src/sine_track.rs b/examples/wgpu_room/src/sine_track.rs index 4148790..86288ca 100644 --- a/examples/wgpu_room/src/sine_track.rs +++ b/examples/wgpu_room/src/sine_track.rs @@ -98,15 +98,20 @@ impl SineTrack { Ok(()) } - async fn track_task(_close_rx: oneshot::Receiver<()>, rtc_source: NativeAudioSource) { + async fn track_task(mut close_rx: oneshot::Receiver<()>, rtc_source: NativeAudioSource) { let mut data = FrameData::default(); let mut interval = tokio::time::interval(Duration::from_millis(10)); let mut samples_10ms = Vec::::new(); loop { - const NUM_CHANNELS: usize = 2; + tokio::select! { + _ = &mut close_rx => { + break; + } + _ = interval.tick() => {} + } - interval.tick().await; + const NUM_CHANNELS: usize = 2; let samples_count_10ms = (data.sample_rate / 100) as usize * NUM_CHANNELS; diff --git a/livekit-ffi/src/server/audio_frame.rs b/livekit-ffi/src/server/audio_frame.rs index bac35de..93ea001 100644 --- a/livekit-ffi/src/server/audio_frame.rs +++ b/livekit-ffi/src/server/audio_frame.rs @@ -4,7 +4,7 @@ use livekit::prelude::*; use livekit::webrtc::audio_frame::AudioFrame; use livekit::webrtc::audio_source::native::NativeAudioSource; use livekit::webrtc::audio_stream::native::NativeAudioStream; -use livekit::webrtc::media_stream::MediaStreamTrack; +use livekit::webrtc::prelude::*; use log::warn; use server::utils; use tokio::sync::oneshot; diff --git a/livekit-ffi/src/server/video_frame.rs b/livekit-ffi/src/server/video_frame.rs index c772dc6..0aedf7c 100644 --- a/livekit-ffi/src/server/video_frame.rs +++ b/livekit-ffi/src/server/video_frame.rs @@ -1,7 +1,7 @@ use crate::{proto, server, FfiError, FfiHandleId, FfiResult}; use futures_util::StreamExt; use livekit::prelude::*; -use livekit::webrtc::media_stream::MediaStreamTrack; +use livekit::webrtc::prelude::*; use livekit::webrtc::video_frame::{BoxVideoFrameBuffer, VideoFrame}; use livekit::webrtc::video_source::native::NativeVideoSource; use livekit::webrtc::video_stream::native::NativeVideoStream; diff --git a/livekit-webrtc/src/audio_stream.rs b/livekit-webrtc/src/audio_stream.rs index 8feeefa..a1949cc 100644 --- a/livekit-webrtc/src/audio_stream.rs +++ b/livekit-webrtc/src/audio_stream.rs @@ -4,7 +4,7 @@ use crate::imp::audio_stream as stream_imp; pub mod native { use super::stream_imp; use crate::audio_frame::AudioFrame; - use crate::media_stream::RtcAudioTrack; + use crate::audio_track::RtcAudioTrack; use futures::stream::Stream; use std::fmt::{Debug, Formatter}; use std::pin::Pin; diff --git a/livekit-webrtc/src/audio_track.rs b/livekit-webrtc/src/audio_track.rs new file mode 100644 index 0000000..7b07c62 --- /dev/null +++ b/livekit-webrtc/src/audio_track.rs @@ -0,0 +1,23 @@ +use crate::imp::audio_track as imp_at; +use crate::media_stream_track::media_stream_track; +use crate::media_stream_track::RtcTrackState; +use std::fmt::Debug; + +#[derive(Clone)] +pub struct RtcAudioTrack { + pub(crate) handle: imp_at::RtcAudioTrack, +} + +impl RtcAudioTrack { + media_stream_track!(); +} + +impl Debug for RtcAudioTrack { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RtcAudioTrack") + .field("id", &self.id()) + .field("enabled", &self.enabled()) + .field("state", &self.state()) + .finish() + } +} diff --git a/livekit-webrtc/src/lib.rs b/livekit-webrtc/src/lib.rs index dac2efe..a4e3219 100644 --- a/livekit-webrtc/src/lib.rs +++ b/livekit-webrtc/src/lib.rs @@ -29,9 +29,11 @@ pub struct RtcError { pub mod audio_frame; pub mod audio_source; pub mod audio_stream; +pub mod audio_track; pub mod data_channel; pub mod ice_candidate; pub mod media_stream; +pub mod media_stream_track; pub mod peer_connection; pub mod peer_connection_factory; pub mod prelude; @@ -43,6 +45,7 @@ pub mod session_description; pub mod video_frame; pub mod video_source; pub mod video_stream; +pub mod video_track; #[cfg(not(target_arch = "wasm32"))] pub mod native { diff --git a/livekit-webrtc/src/media_stream.rs b/livekit-webrtc/src/media_stream.rs index 675155c..5a92c1e 100644 --- a/livekit-webrtc/src/media_stream.rs +++ b/livekit-webrtc/src/media_stream.rs @@ -1,13 +1,8 @@ +use crate::audio_track::RtcAudioTrack; use crate::imp::media_stream as imp_ms; -use livekit_protocol::enum_dispatch; +use crate::video_track::RtcVideoTrack; use std::fmt::Debug; -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub enum RtcTrackState { - Live, - Ended, -} - #[derive(Clone)] pub struct MediaStream { pub(crate) handle: imp_ms::MediaStream, @@ -36,104 +31,3 @@ impl Debug for MediaStream { .finish() } } - -#[derive(Clone)] -pub struct RtcVideoTrack { - pub(crate) handle: imp_ms::RtcVideoTrack, -} - -#[derive(Clone)] -pub struct RtcAudioTrack { - pub(crate) handle: imp_ms::RtcAudioTrack, -} - -#[derive(Debug, Clone)] -pub enum MediaStreamTrack { - Video(RtcVideoTrack), - Audio(RtcAudioTrack), -} - -#[cfg(not(target_arch = "wasm32"))] -impl MediaStreamTrack { - enum_dispatch!( - [Video, Audio]; - pub(crate) fn sys_handle(self: &Self) -> cxx::SharedPtr; - ); -} - -impl MediaStreamTrack { - enum_dispatch!( - [Video, Audio]; - pub fn id(self: &Self) -> String; - pub fn enabled(self: &Self) -> bool; - pub fn set_enabled(self: &Self, enabled: bool) -> bool; - pub fn state(self: &Self) -> RtcTrackState; - ); -} - -macro_rules! media_stream_track { - () => { - pub fn id(&self) -> String { - self.handle.id() - } - - pub fn enabled(&self) -> bool { - self.handle.enabled() - } - - pub fn set_enabled(&self, enabled: bool) -> bool { - self.handle.set_enabled(enabled) - } - - pub fn state(&self) -> RtcTrackState { - self.handle.state().into() - } - - #[cfg(not(target_arch = "wasm32"))] - pub(crate) fn sys_handle( - &self, - ) -> cxx::SharedPtr { - self.handle.sys_handle() - } - }; -} - -impl RtcVideoTrack { - media_stream_track!(); -} - -impl RtcAudioTrack { - media_stream_track!(); -} - -impl Debug for RtcAudioTrack { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("RtcAudioTrack") - .field("id", &self.id()) - .field("enabled", &self.enabled()) - .field("state", &self.state()) - .finish() - } -} - -impl Debug for RtcVideoTrack { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("RtcVideoTrack") - .field("id", &self.id()) - .field("enabled", &self.enabled()) - .field("state", &self.state()) - .finish() - } -} - -impl From for MediaStreamTrack { - fn from(track: RtcAudioTrack) -> Self { - Self::Audio(track) - } -} - -impl From for MediaStreamTrack { - fn from(track: RtcVideoTrack) -> Self { - Self::Video(track) - } -} diff --git a/livekit-webrtc/src/media_stream_track.rs b/livekit-webrtc/src/media_stream_track.rs new file mode 100644 index 0000000..4579ca9 --- /dev/null +++ b/livekit-webrtc/src/media_stream_track.rs @@ -0,0 +1,74 @@ +use crate::audio_track::RtcAudioTrack; +use crate::video_track::RtcVideoTrack; +use livekit_protocol::enum_dispatch; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum RtcTrackState { + Live, + Ended, +} + +#[derive(Debug, Clone)] +pub enum MediaStreamTrack { + Video(RtcVideoTrack), + Audio(RtcAudioTrack), +} + +#[cfg(not(target_arch = "wasm32"))] +impl MediaStreamTrack { + enum_dispatch!( + [Video, Audio]; + pub(crate) fn sys_handle(self: &Self) -> cxx::SharedPtr; + ); +} + +impl MediaStreamTrack { + enum_dispatch!( + [Video, Audio]; + pub fn id(self: &Self) -> String; + pub fn enabled(self: &Self) -> bool; + pub fn set_enabled(self: &Self, enabled: bool) -> bool; + pub fn state(self: &Self) -> RtcTrackState; + ); +} + +macro_rules! media_stream_track { + () => { + pub fn id(&self) -> String { + self.handle.id() + } + + pub fn enabled(&self) -> bool { + self.handle.enabled() + } + + pub fn set_enabled(&self, enabled: bool) -> bool { + self.handle.set_enabled(enabled) + } + + pub fn state(&self) -> RtcTrackState { + self.handle.state().into() + } + + #[cfg(not(target_arch = "wasm32"))] + pub(crate) fn sys_handle( + &self, + ) -> cxx::SharedPtr { + self.handle.sys_handle() + } + }; +} + +pub(crate) use media_stream_track; + +impl From for MediaStreamTrack { + fn from(track: RtcAudioTrack) -> Self { + Self::Audio(track) + } +} + +impl From for MediaStreamTrack { + fn from(track: RtcVideoTrack) -> Self { + Self::Video(track) + } +} diff --git a/livekit-webrtc/src/native/audio_source.rs b/livekit-webrtc/src/native/audio_source.rs index e25fa3e..3b00c64 100644 --- a/livekit-webrtc/src/native/audio_source.rs +++ b/livekit-webrtc/src/native/audio_source.rs @@ -2,11 +2,11 @@ use crate::audio_frame::AudioFrame; use cxx::SharedPtr; use parking_lot::Mutex; use std::sync::Arc; -use webrtc_sys::media_stream as sys_ms; +use webrtc_sys::audio_track as sys_at; #[derive(Clone)] pub struct NativeAudioSource { - sys_handle: SharedPtr, + sys_handle: SharedPtr, inner: Arc>, } @@ -21,14 +21,14 @@ struct AudioSourceInner { impl Default for NativeAudioSource { fn default() -> Self { Self { - sys_handle: sys_ms::ffi::new_audio_track_source(), + sys_handle: sys_at::ffi::new_audio_track_source(), inner: Default::default(), } } } impl NativeAudioSource { - pub fn sys_handle(&self) -> SharedPtr { + pub fn sys_handle(&self) -> SharedPtr { self.sys_handle.clone() } @@ -70,14 +70,12 @@ impl NativeAudioSource { &frame.data[i..i + samples_10ms] }; - unsafe { - self.sys_handle.on_captured_frame( - data.as_ptr(), - frame.sample_rate as i32, - frame.num_channels as usize, - samples_10ms / frame.num_channels as usize, - ) - } + self.sys_handle.on_captured_frame( + data, + frame.sample_rate as i32, + frame.num_channels as usize, + samples_10ms / frame.num_channels as usize, + ); i += needed_data; } diff --git a/livekit-webrtc/src/native/audio_stream.rs b/livekit-webrtc/src/native/audio_stream.rs index 4f21729..de20ff5 100644 --- a/livekit-webrtc/src/native/audio_stream.rs +++ b/livekit-webrtc/src/native/audio_stream.rs @@ -1,14 +1,15 @@ -use crate::{audio_frame::AudioFrame, media_stream::RtcAudioTrack}; -use cxx::UniquePtr; +use crate::audio_frame::AudioFrame; +use crate::audio_track::RtcAudioTrack; +use cxx::SharedPtr; use futures::stream::Stream; use std::pin::Pin; +use std::sync::Arc; use std::task::{Context, Poll}; use tokio::sync::mpsc; -use webrtc_sys::media_stream as sys_ms; +use webrtc_sys::audio_track as sys_at; pub struct NativeAudioStream { - native_observer: UniquePtr, - _observer: Box, + native_sink: SharedPtr, audio_track: RtcAudioTrack, frame_rx: mpsc::UnboundedReceiver, } @@ -16,21 +17,16 @@ pub struct NativeAudioStream { impl NativeAudioStream { pub fn new(audio_track: RtcAudioTrack) -> Self { let (frame_tx, frame_rx) = mpsc::unbounded_channel(); - let mut observer = Box::new(AudioTrackObserver { frame_tx }); - let mut native_observer = unsafe { - sys_ms::ffi::new_native_audio_sink(Box::new(sys_ms::AudioSinkWrapper::new( - &mut *observer, - ))) - }; + let observer = Arc::new(AudioTrackObserver { frame_tx }); + let native_sink = sys_at::ffi::new_native_audio_sink(Box::new( + sys_at::AudioSinkWrapper::new(observer.clone()), + )); - unsafe { - sys_ms::ffi::media_to_audio(audio_track.sys_handle()) - .add_sink(native_observer.pin_mut()); - } + let audio = unsafe { sys_at::ffi::media_to_audio(audio_track.sys_handle()) }; + audio.add_sink(&native_sink); Self { - native_observer, - _observer: observer, + native_sink, audio_track, frame_rx, } @@ -41,11 +37,10 @@ impl NativeAudioStream { } pub fn close(&mut self) { + let audio = unsafe { sys_at::ffi::media_to_audio(self.audio_track.sys_handle()) }; + audio.remove_sink(&self.native_sink); + self.frame_rx.close(); - unsafe { - sys_ms::ffi::media_to_audio(self.audio_track.sys_handle()) - .remove_sink(self.native_observer.pin_mut()); - } } } @@ -67,7 +62,7 @@ pub struct AudioTrackObserver { frame_tx: mpsc::UnboundedSender, } -impl sys_ms::AudioSink for AudioTrackObserver { +impl sys_at::AudioSink for AudioTrackObserver { fn on_data(&self, data: &[i16], sample_rate: i32, nb_channels: usize, nb_frames: usize) { // TODO(theomonnom): Should we avoid copy here? let _ = self.frame_tx.send(AudioFrame { diff --git a/livekit-webrtc/src/native/audio_track.rs b/livekit-webrtc/src/native/audio_track.rs new file mode 100644 index 0000000..0658bb7 --- /dev/null +++ b/livekit-webrtc/src/native/audio_track.rs @@ -0,0 +1,18 @@ +use super::media_stream_track::impl_media_stream_track; +use crate::media_stream_track::RtcTrackState; +use cxx::SharedPtr; +use sys_at::ffi::audio_to_media; +use webrtc_sys::audio_track as sys_at; + +#[derive(Clone)] +pub struct RtcAudioTrack { + pub(crate) sys_handle: SharedPtr, +} + +impl RtcAudioTrack { + impl_media_stream_track!(audio_to_media); + + pub fn sys_handle(&self) -> SharedPtr { + audio_to_media(self.sys_handle.clone()) + } +} diff --git a/livekit-webrtc/src/native/data_channel.rs b/livekit-webrtc/src/native/data_channel.rs index a0314e7..39110d5 100644 --- a/livekit-webrtc/src/native/data_channel.rs +++ b/livekit-webrtc/src/native/data_channel.rs @@ -39,30 +39,21 @@ impl From for sys_dc::ffi::DataChannelInit { #[derive(Clone)] pub struct DataChannel { - #[allow(dead_code)] - native_observer: SharedPtr, observer: Arc, - pub(crate) sys_handle: SharedPtr, } impl DataChannel { pub fn configure(sys_handle: SharedPtr) -> Self { - unsafe { - let observer = Arc::new(DataChannelObserver::default()); - let dc = Self { - sys_handle: sys_handle.clone(), - native_observer: sys_dc::ffi::create_native_data_channel_observer( - Box::new(sys_dc::DataChannelObserverWrapper::new(observer.clone())), - &*sys_handle as *const _ as *mut _, - ), - observer, - }; + let observer = Arc::new(DataChannelObserver::default()); + let dc = Self { + sys_handle: sys_handle.clone(), + observer: observer.clone(), + }; - dc.sys_handle - .register_observer(&*dc.native_observer as *const _ as *mut _); - dc - } + dc.sys_handle + .register_observer(Box::new(sys_dc::DataChannelObserverWrapper::new(observer))); + dc } pub fn send(&self, data: &[u8], binary: bool) -> Result<(), DataChannelError> { diff --git a/livekit-webrtc/src/native/media_stream.rs b/livekit-webrtc/src/native/media_stream.rs index 069f6e8..767fbb3 100644 --- a/livekit-webrtc/src/native/media_stream.rs +++ b/livekit-webrtc/src/native/media_stream.rs @@ -1,20 +1,9 @@ -use crate::media_stream::{self, MediaStreamTrack, RtcTrackState}; +use crate::audio_track; +use crate::imp::audio_track::RtcAudioTrack; +use crate::imp::video_track::RtcVideoTrack; +use crate::video_track; use cxx::SharedPtr; use webrtc_sys::media_stream as sys_ms; -use webrtc_sys::media_stream::ffi::{ - audio_to_media, media_to_audio, media_to_video, video_to_media, -}; -use webrtc_sys::{MEDIA_TYPE_AUDIO, MEDIA_TYPE_VIDEO}; - -impl From for RtcTrackState { - fn from(state: sys_ms::ffi::TrackState) -> Self { - match state { - sys_ms::ffi::TrackState::Live => RtcTrackState::Live, - sys_ms::ffi::TrackState::Ended => RtcTrackState::Ended, - _ => panic!("unknown TrackState"), - } - } -} #[derive(Clone)] pub struct MediaStream { @@ -26,93 +15,23 @@ impl MediaStream { self.sys_handle.id() } - pub fn audio_tracks(&self) -> Vec { + pub fn audio_tracks(&self) -> Vec { self.sys_handle .get_audio_tracks() .into_iter() - .map(|t| media_stream::RtcAudioTrack { + .map(|t| audio_track::RtcAudioTrack { handle: RtcAudioTrack { sys_handle: t.ptr }, }) .collect() } - pub fn video_tracks(&self) -> Vec { + pub fn video_tracks(&self) -> Vec { self.sys_handle .get_video_tracks() .into_iter() - .map(|t| media_stream::RtcVideoTrack { + .map(|t| video_track::RtcVideoTrack { handle: RtcVideoTrack { sys_handle: t.ptr }, }) .collect() } } - -pub fn new_media_stream_track( - sys_handle: SharedPtr, -) -> MediaStreamTrack { - if sys_handle.kind() == MEDIA_TYPE_AUDIO { - MediaStreamTrack::Audio(media_stream::RtcAudioTrack { - handle: RtcAudioTrack { - sys_handle: media_to_audio(sys_handle), - }, - }) - } else if sys_handle.kind() == MEDIA_TYPE_VIDEO { - MediaStreamTrack::Video(media_stream::RtcVideoTrack { - handle: RtcVideoTrack { - sys_handle: media_to_video(sys_handle), - }, - }) - } else { - panic!("unknown track kind") - } -} - -macro_rules! impl_media_stream_track { - ($cast:ident) => { - pub fn id(&self) -> String { - let ptr = sys_ms::ffi::$cast(self.sys_handle.clone()); - ptr.id() - } - - pub fn enabled(&self) -> bool { - let ptr = sys_ms::ffi::$cast(self.sys_handle.clone()); - ptr.enabled() - } - - pub fn set_enabled(&self, enabled: bool) -> bool { - let ptr = sys_ms::ffi::$cast(self.sys_handle.clone()); - ptr.set_enabled(enabled) - } - - pub fn state(&self) -> RtcTrackState { - let ptr = sys_ms::ffi::$cast(self.sys_handle.clone()); - ptr.state().into() - } - }; -} - -#[derive(Clone)] -pub struct RtcVideoTrack { - pub(crate) sys_handle: SharedPtr, -} - -impl RtcVideoTrack { - impl_media_stream_track!(video_to_media); - - pub fn sys_handle(&self) -> SharedPtr { - video_to_media(self.sys_handle.clone()) - } -} - -#[derive(Clone)] -pub struct RtcAudioTrack { - pub(crate) sys_handle: SharedPtr, -} - -impl RtcAudioTrack { - impl_media_stream_track!(audio_to_media); - - pub fn sys_handle(&self) -> SharedPtr { - audio_to_media(self.sys_handle.clone()) - } -} diff --git a/livekit-webrtc/src/native/media_stream_track.rs b/livekit-webrtc/src/native/media_stream_track.rs new file mode 100644 index 0000000..5811b0a --- /dev/null +++ b/livekit-webrtc/src/native/media_stream_track.rs @@ -0,0 +1,67 @@ +use crate::audio_track; +use crate::imp::audio_track::RtcAudioTrack; +use crate::imp::video_track::RtcVideoTrack; +use crate::media_stream_track::MediaStreamTrack; +use crate::media_stream_track::RtcTrackState; +use crate::video_track; +use cxx::SharedPtr; +use webrtc_sys::audio_track::ffi::media_to_audio; +use webrtc_sys::media_stream_track as sys_mst; +use webrtc_sys::video_track::ffi::media_to_video; +use webrtc_sys::{MEDIA_TYPE_AUDIO, MEDIA_TYPE_VIDEO}; + +impl From for RtcTrackState { + fn from(state: sys_mst::ffi::TrackState) -> Self { + match state { + sys_mst::ffi::TrackState::Live => RtcTrackState::Live, + sys_mst::ffi::TrackState::Ended => RtcTrackState::Ended, + _ => panic!("unknown TrackState"), + } + } +} + +pub fn new_media_stream_track( + sys_handle: SharedPtr, +) -> MediaStreamTrack { + if sys_handle.kind() == MEDIA_TYPE_AUDIO { + MediaStreamTrack::Audio(audio_track::RtcAudioTrack { + handle: RtcAudioTrack { + sys_handle: unsafe { media_to_audio(sys_handle) }, + }, + }) + } else if sys_handle.kind() == MEDIA_TYPE_VIDEO { + MediaStreamTrack::Video(video_track::RtcVideoTrack { + handle: RtcVideoTrack { + sys_handle: unsafe { media_to_video(sys_handle) }, + }, + }) + } else { + panic!("unknown track kind") + } +} + +macro_rules! impl_media_stream_track { + ($cast:expr) => { + pub fn id(&self) -> String { + let ptr = $cast(self.sys_handle.clone()); + ptr.id() + } + + pub fn enabled(&self) -> bool { + let ptr = $cast(self.sys_handle.clone()); + ptr.enabled() + } + + pub fn set_enabled(&self, enabled: bool) -> bool { + let ptr = $cast(self.sys_handle.clone()); + ptr.set_enabled(enabled) + } + + pub fn state(&self) -> RtcTrackState { + let ptr = $cast(self.sys_handle.clone()); + ptr.state().into() + } + }; +} + +pub(super) use impl_media_stream_track; diff --git a/livekit-webrtc/src/native/mod.rs b/livekit-webrtc/src/native/mod.rs index 7db7bd0..59dbbd6 100644 --- a/livekit-webrtc/src/native/mod.rs +++ b/livekit-webrtc/src/native/mod.rs @@ -1,9 +1,11 @@ pub mod audio_resampler; pub mod audio_source; pub mod audio_stream; +pub mod audio_track; pub mod data_channel; pub mod ice_candidate; pub mod media_stream; +pub mod media_stream_track; pub mod peer_connection; pub mod peer_connection_factory; pub mod rtp_parameters; @@ -14,6 +16,7 @@ pub mod session_description; pub mod video_frame; pub mod video_source; pub mod video_stream; +pub mod video_track; pub mod yuv_helper; use crate::MediaType; @@ -21,17 +24,17 @@ use crate::{RtcError, RtcErrorType}; use webrtc_sys::rtc_error as sys_err; use webrtc_sys::webrtc as sys_rtc; -impl From for RtcErrorType { - fn from(value: sys_err::ffi::RTCErrorType) -> Self { +impl From for RtcErrorType { + fn from(value: sys_err::ffi::RtcErrorType) -> Self { match value { - sys_err::ffi::RTCErrorType::InvalidState => Self::InvalidState, + sys_err::ffi::RtcErrorType::InvalidState => Self::InvalidState, _ => Self::Internal, } } } -impl From for RtcError { - fn from(value: sys_err::ffi::RTCError) -> Self { +impl From for RtcError { + fn from(value: sys_err::ffi::RtcError) -> Self { Self { error_type: value.error_type.into(), message: value.message, diff --git a/livekit-webrtc/src/native/peer_connection.rs b/livekit-webrtc/src/native/peer_connection.rs index 67b1f0c..d00eabe 100644 --- a/livekit-webrtc/src/native/peer_connection.rs +++ b/livekit-webrtc/src/native/peer_connection.rs @@ -4,11 +4,13 @@ use crate::ice_candidate::IceCandidate; use crate::imp::data_channel as imp_dc; use crate::imp::ice_candidate as imp_ic; use crate::imp::media_stream as imp_ms; +use crate::imp::media_stream_track as imp_mst; use crate::imp::rtp_receiver as imp_rr; use crate::imp::rtp_sender as imp_rs; use crate::imp::rtp_transceiver as imp_rt; use crate::imp::session_description as imp_sdp; -use crate::media_stream::{MediaStream, MediaStreamTrack}; +use crate::media_stream::MediaStream; +use crate::media_stream_track::MediaStreamTrack; use crate::peer_connection::{ AnswerOptions, IceCandidateError, IceConnectionState, IceGatheringState, OfferOptions, OnConnectionChange, OnDataChannel, OnIceCandidate, OnIceCandidateError, OnIceConnectionChange, @@ -21,17 +23,16 @@ use crate::rtp_transceiver::RtpTransceiver; use crate::rtp_transceiver::RtpTransceiverInit; use crate::MediaType; use crate::{session_description::SessionDescription, RtcError}; -use cxx::{SharedPtr, UniquePtr}; +use cxx::SharedPtr; use futures::channel::oneshot; use parking_lot::Mutex; -use std::mem::ManuallyDrop; use std::sync::Arc; use webrtc_sys::data_channel as sys_dc; use webrtc_sys::jsep as sys_jsep; use webrtc_sys::peer_connection as sys_pc; use webrtc_sys::rtc_error as sys_err; -impl From for sys_pc::ffi::RTCOfferAnswerOptions { +impl From for sys_pc::ffi::RtcOfferAnswerOptions { fn from(options: OfferOptions) -> Self { Self { ice_restart: options.ice_restart, @@ -42,7 +43,7 @@ impl From for sys_pc::ffi::RTCOfferAnswerOptions { } } -impl From for sys_pc::ffi::RTCOfferAnswerOptions { +impl From for sys_pc::ffi::RtcOfferAnswerOptions { fn from(_options: AnswerOptions) -> Self { Self::default() } @@ -111,10 +112,7 @@ impl From for SignalingState { #[derive(Clone)] pub struct PeerConnection { - #[allow(dead_code)] - native_observer: SharedPtr, observer: Arc, - pub(crate) sys_handle: SharedPtr, } @@ -122,12 +120,10 @@ impl PeerConnection { pub fn configure( sys_handle: SharedPtr, observer: Arc, - native_observer: SharedPtr, ) -> Self { Self { sys_handle, observer, - native_observer, } } @@ -135,12 +131,29 @@ impl PeerConnection { &self, options: OfferOptions, ) -> Result { - let (mut native_wrapper, mut sdp_rx, mut err_rx) = create_sdp_observer(); + let (sdp_tx, mut sdp_rx) = oneshot::channel(); + let (err_tx, mut err_rx) = oneshot::channel(); - unsafe { - self.sys_handle - .create_offer(native_wrapper.pin_mut(), options.into()); - } + let ctx = Box::new(sys_pc::AsyncContext(Box::new((sdp_tx, err_tx)))); + type CtxType = ( + oneshot::Sender, + oneshot::Sender, + ); + + self.sys_handle.create_offer( + options.into(), + ctx, + |ctx, sdp| { + let (sdp_tx, _) = *ctx.0.downcast::().unwrap(); + let _ = sdp_tx.send(SessionDescription { + handle: imp_sdp::SessionDescription { sys_handle: sdp }, + }); + }, + |ctx, error| { + let (_, err_tx) = *ctx.0.downcast::().unwrap(); + let _ = err_tx.send(error.into()); + }, + ); futures::select! { sdp = sdp_rx => Ok(sdp.unwrap()), @@ -152,13 +165,29 @@ impl PeerConnection { &self, options: AnswerOptions, ) -> Result { - let (mut native_wrapper, mut sdp_rx, mut err_rx) = create_sdp_observer(); + let (sdp_tx, mut sdp_rx) = oneshot::channel(); + let (err_tx, mut err_rx) = oneshot::channel(); - unsafe { - self.sys_handle - .create_answer(native_wrapper.pin_mut(), options.into()); - } + let ctx = Box::new(sys_pc::AsyncContext(Box::new((sdp_tx, err_tx)))); + type CtxType = ( + oneshot::Sender, + oneshot::Sender, + ); + self.sys_handle.create_answer( + options.into(), + ctx, + |ctx, sdp| { + let (sdp_tx, _) = *ctx.0.downcast::().unwrap(); + let _ = sdp_tx.send(SessionDescription { + handle: imp_sdp::SessionDescription { sys_handle: sdp }, + }); + }, + |ctx, error| { + let (_, err_tx) = *ctx.0.downcast::().unwrap(); + let _ = err_tx.send(error.into()); + }, + ); futures::select! { sdp = sdp_rx => Ok(sdp.unwrap()), err = err_rx => Err(err.unwrap()), @@ -166,54 +195,66 @@ impl PeerConnection { } pub async fn set_local_description(&self, desc: SessionDescription) -> Result<(), RtcError> { - let (tx, rx) = oneshot::channel(); - let wrapper = - sys_jsep::SetLocalSdpObserverWrapper(ManuallyDrop::new(Box::new(move |error| { - let _ = tx.send(if error.ok() { Ok(()) } else { Err(error) }); - }))); + let (tx, rx) = oneshot::channel::>(); + let ctx = Box::new(sys_pc::AsyncContext(Box::new(tx))); - let mut native_wrapper = - sys_jsep::ffi::create_native_set_local_sdp_observer(Box::new(wrapper)); + self.sys_handle + .set_local_description(desc.handle.sys_handle, ctx, |ctx, err| { + let tx = ctx + .0 + .downcast::>>() + .unwrap(); - unsafe { - self.sys_handle - .set_local_description(desc.handle.sys_handle, native_wrapper.pin_mut()); - } + if err.ok() { + let _ = tx.send(Ok(())); + } else { + let _ = tx.send(Err(err.into())); + } + }); - rx.await.unwrap().map_err(Into::into) + rx.await.unwrap() } pub async fn set_remote_description(&self, desc: SessionDescription) -> Result<(), RtcError> { - let (tx, rx) = oneshot::channel(); - let wrapper = - sys_jsep::SetRemoteSdpObserverWrapper(ManuallyDrop::new(Box::new(move |error| { - let _ = tx.send(if error.ok() { Ok(()) } else { Err(error) }); - }))); + let (tx, rx) = oneshot::channel::>(); + let ctx = Box::new(sys_pc::AsyncContext(Box::new(tx))); - let mut native_wrapper = - sys_jsep::ffi::create_native_set_remote_sdp_observer(Box::new(wrapper)); + self.sys_handle + .set_remote_description(desc.handle.sys_handle, ctx, |ctx, err| { + let tx = ctx + .0 + .downcast::>>() + .unwrap(); - unsafe { - self.sys_handle - .set_remote_description(desc.handle.sys_handle, native_wrapper.pin_mut()); - } + if err.ok() { + let _ = tx.send(Ok(())); + } else { + let _ = tx.send(Err(err.into())); + } + }); - rx.await.unwrap().map_err(Into::into) + rx.await.unwrap() } pub async fn add_ice_candidate(&self, candidate: IceCandidate) -> Result<(), RtcError> { - let (tx, rx) = oneshot::channel(); - let observer = - sys_pc::AddIceCandidateObserverWrapper(ManuallyDrop::new(Box::new(|error| { - let _ = tx.send(if error.ok() { Ok(()) } else { Err(error) }); - }))); + let (tx, rx) = oneshot::channel::>(); + let ctx = Box::new(sys_pc::AsyncContext(Box::new(tx))); - let mut native_observer = - sys_pc::ffi::create_native_add_ice_candidate_observer(Box::new(observer)); self.sys_handle - .add_ice_candidate(candidate.handle.sys_handle, native_observer.pin_mut()); + .add_ice_candidate(candidate.handle.sys_handle, ctx, |ctx, err| { + let tx = ctx + .0 + .downcast::>>() + .unwrap(); - rx.await.unwrap().map_err(Into::into) + if err.ok() { + let _ = tx.send(Ok(())); + } else { + let _ = tx.send(Err(err.into())); + } + }); + + rx.await.unwrap() } pub fn create_data_channel( @@ -221,16 +262,15 @@ impl PeerConnection { label: &str, init: DataChannelInit, ) -> Result { - let native_init = sys_dc::ffi::create_data_channel_init(init.into()); let res = self .sys_handle - .create_data_channel(label.to_string(), native_init); + .create_data_channel(label.to_string(), init.into()); match res { Ok(sys_handle) => Ok(DataChannel { handle: imp_dc::DataChannel::configure(sys_handle), }), - Err(e) => Err(unsafe { sys_err::ffi::RTCError::from(e.what()).into() }), + Err(e) => Err(unsafe { sys_err::ffi::RtcError::from(e.what()).into() }), } } @@ -246,7 +286,7 @@ impl PeerConnection { Ok(sys_handle) => Ok(RtpSender { handle: imp_rs::RtpSender { sys_handle }, }), - Err(e) => unsafe { Err(sys_err::ffi::RTCError::from(e.what()).into()) }, + Err(e) => unsafe { Err(sys_err::ffi::RtcError::from(e.what()).into()) }, } } @@ -265,7 +305,7 @@ impl PeerConnection { sys_handle: sys_handle, }, }), - Err(e) => unsafe { Err(sys_err::ffi::RTCError::from(e.what()).into()) }, + Err(e) => unsafe { Err(sys_err::ffi::RtcError::from(e.what()).into()) }, } } @@ -284,7 +324,7 @@ impl PeerConnection { sys_handle: cxx_handle, }, }), - Err(e) => unsafe { Err(sys_err::ffi::RTCError::from(e.what()).into()) }, + Err(e) => unsafe { Err(sys_err::ffi::RtcError::from(e.what()).into()) }, } } @@ -333,7 +373,7 @@ impl PeerConnection { pub fn remove_track(&self, sender: RtpSender) -> Result<(), RtcError> { self.sys_handle .remove_track(sender.handle.sys_handle) - .map_err(|e| unsafe { sys_err::ffi::RTCError::from(e.what()).into() }) + .map_err(|e| unsafe { sys_err::ffi::RtcError::from(e.what()).into() }) } pub fn senders(&self) -> Vec { @@ -409,34 +449,6 @@ impl PeerConnection { } } -fn create_sdp_observer() -> ( - UniquePtr, - oneshot::Receiver, - oneshot::Receiver, -) { - let (sdp_tx, sdp_rx) = oneshot::channel(); - let (err_tx, err_rx) = oneshot::channel(); - - let wrapper = sys_jsep::CreateSdpObserverWrapper { - on_success: ManuallyDrop::new(Box::new(move |session_description| { - let _ = sdp_tx.send(SessionDescription { - handle: imp_sdp::SessionDescription { - sys_handle: session_description, - }, - }); - })), - on_failure: ManuallyDrop::new(Box::new(move |error| { - let _ = err_tx.send(error.into()); - })), - }; - - ( - sys_jsep::ffi::create_native_create_sdp_observer(Box::new(wrapper)), - sdp_rx, - err_rx, - ) -} - #[derive(Default)] pub struct PeerObserver { pub connection_change_handler: Mutex>, @@ -565,7 +577,7 @@ impl sys_pc::PeerConnectionObserver for PeerObserver { handle: imp_ms::MediaStream { sys_handle: s.ptr }, }) .collect(), - track: imp_ms::new_media_stream_track(track), + track: imp_mst::new_media_stream_track(track), transceiver: RtpTransceiver { handle: imp_rt::RtpTransceiver { sys_handle: transceiver, diff --git a/livekit-webrtc/src/native/peer_connection_factory.rs b/livekit-webrtc/src/native/peer_connection_factory.rs index ea435d3..2cf7911 100644 --- a/livekit-webrtc/src/native/peer_connection_factory.rs +++ b/livekit-webrtc/src/native/peer_connection_factory.rs @@ -1,89 +1,61 @@ use crate::audio_source::native::NativeAudioSource; -use crate::imp::media_stream as imp_ms; +use crate::audio_track::RtcAudioTrack; +use crate::imp::audio_track as imp_at; use crate::imp::peer_connection as imp_pc; -use crate::media_stream::{RtcAudioTrack, RtcVideoTrack}; +use crate::imp::video_track as imp_vt; use crate::peer_connection::PeerConnection; use crate::peer_connection_factory::{ ContinualGatheringPolicy, IceServer, IceTransportsType, RtcConfiguration, }; use crate::rtp_parameters::RtpCapabilities; use crate::video_source::native::NativeVideoSource; +use crate::video_track::RtcVideoTrack; use crate::MediaType; use crate::RtcError; use cxx::SharedPtr; use cxx::UniquePtr; use lazy_static::lazy_static; use parking_lot::Mutex; -use std::sync::{Arc, Weak}; -use webrtc_sys::logsink as sys_ls; +use std::sync::Arc; use webrtc_sys::peer_connection as sys_pc; use webrtc_sys::peer_connection_factory as sys_pcf; use webrtc_sys::rtc_error as sys_err; -use webrtc_sys::webrtc as sys_webrtc; +use webrtc_sys::webrtc as sys_rtc; lazy_static! { - static ref RTC_RUNTIME: Mutex> = Mutex::new(Weak::new()); -} - -pub struct RtcRuntime { - pub(crate) sys_handle: SharedPtr, - _logsink: UniquePtr, -} - -impl RtcRuntime { - pub fn instance() -> Arc { - let mut lk_runtime_ref = RTC_RUNTIME.lock(); - if let Some(lk_runtime) = lk_runtime_ref.upgrade() { - lk_runtime - } else { - log::trace!("RtcRuntime::new()"); - let new_runtime = Arc::new(Self { - sys_handle: sys_webrtc::ffi::create_rtc_runtime(), - _logsink: sys_ls::ffi::new_log_sink(|msg, severity| { - // Forward logs from webrtc to rust log crate - let msg = msg - .strip_suffix("\r\n") - .or(msg.strip_suffix("\n")) - .unwrap_or(&msg); - - let lvl = match severity { - sys_ls::ffi::LoggingSeverity::Verbose => log::Level::Trace, - sys_ls::ffi::LoggingSeverity::Info => log::Level::Debug, // Translte webrtc - // info to debug log level to avoid polluting the user logs - sys_ls::ffi::LoggingSeverity::Warning => log::Level::Warn, - sys_ls::ffi::LoggingSeverity::Error => log::Level::Error, - _ => log::Level::Debug, - }; - - log::log!(target: "libwebrtc", lvl, "{}", msg); - }), - }); - *lk_runtime_ref = Arc::downgrade(&new_runtime); - new_runtime - } - } -} - -impl Drop for RtcRuntime { - fn drop(&mut self) { - log::trace!("RtcRuntime::drop()"); - } + static ref LOG_SINK: Mutex>> = Default::default(); } #[derive(Clone)] pub struct PeerConnectionFactory { sys_handle: SharedPtr, - - #[allow(unused)] - runtime: Arc, } impl Default for PeerConnectionFactory { fn default() -> Self { - let runtime = RtcRuntime::instance(); + let mut log_sink = LOG_SINK.lock(); + if log_sink.is_none() { + *log_sink = Some(sys_rtc::ffi::new_log_sink(|msg, severity| { + let msg = msg + .strip_suffix("\r\n") + .or(msg.strip_suffix("\n")) + .unwrap_or(&msg); + + let lvl = match severity { + sys_rtc::ffi::LoggingSeverity::Verbose => log::Level::Trace, + sys_rtc::ffi::LoggingSeverity::Info => log::Level::Debug, // Translte webrtc + // info to debug log level to avoid polluting the user logs + sys_rtc::ffi::LoggingSeverity::Warning => log::Level::Warn, + sys_rtc::ffi::LoggingSeverity::Error => log::Level::Error, + _ => log::Level::Debug, + }; + + log::log!(target: "libwebrtc", lvl, "{}", msg); + })); + } + Self { - sys_handle: sys_pcf::ffi::create_peer_connection_factory(runtime.sys_handle.clone()), - runtime, + sys_handle: sys_pcf::ffi::create_peer_connection_factory(), } } } @@ -93,35 +65,26 @@ impl PeerConnectionFactory { &self, config: RtcConfiguration, ) -> Result { - let native_config = sys_pcf::ffi::create_rtc_configuration(config.into()); + let observer = Arc::new(imp_pc::PeerObserver::default()); + let native_observer = sys_pc::ffi::create_native_peer_connection_observer(Box::new( + sys_pc::PeerConnectionObserverWrapper::new(observer.clone()), + )); - unsafe { - let observer = Arc::new(imp_pc::PeerObserver::default()); - let native_observer = sys_pc::ffi::create_native_peer_connection_observer( - self.runtime.sys_handle.clone(), - Box::new(sys_pc::PeerConnectionObserverWrapper::new(observer.clone())), - ); + let res = self + .sys_handle + .create_peer_connection(config.into(), native_observer); - let res = self - .sys_handle - .create_peer_connection(native_config, &*native_observer as *const _ as *mut _); - - match res { - Ok(sys_handle) => Ok(PeerConnection { - handle: imp_pc::PeerConnection::configure( - sys_handle, - observer, - native_observer, - ), - }), - Err(e) => Err(sys_err::ffi::RTCError::from(e.what()).into()), - } + match res { + Ok(sys_handle) => Ok(PeerConnection { + handle: imp_pc::PeerConnection::configure(sys_handle, observer), + }), + Err(e) => Err(unsafe { sys_err::ffi::RtcError::from(e.what()).into() }), } } pub fn create_video_track(&self, label: &str, source: NativeVideoSource) -> RtcVideoTrack { RtcVideoTrack { - handle: imp_ms::RtcVideoTrack { + handle: imp_vt::RtcVideoTrack { sys_handle: self .sys_handle .create_video_track(label.to_string(), source.handle.sys_handle()), @@ -131,7 +94,7 @@ impl PeerConnectionFactory { pub fn create_audio_track(&self, label: &str, source: NativeAudioSource) -> RtcAudioTrack { RtcAudioTrack { - handle: imp_ms::RtcAudioTrack { + handle: imp_at::RtcAudioTrack { sys_handle: self .sys_handle .create_audio_track(label.to_string(), source.handle.sys_handle()), @@ -153,9 +116,9 @@ impl PeerConnectionFactory { } // Conversions -impl From for sys_pcf::ffi::ICEServer { +impl From for sys_pcf::ffi::IceServer { fn from(value: IceServer) -> Self { - sys_pcf::ffi::ICEServer { + sys_pcf::ffi::IceServer { urls: value.urls, username: value.username, password: value.password, @@ -187,7 +150,7 @@ impl From for sys_pcf::ffi::IceTransportsType { } } -impl From for sys_pcf::ffi::RTCConfiguration { +impl From for sys_pcf::ffi::RtcConfiguration { fn from(value: RtcConfiguration) -> Self { Self { ice_servers: value.ice_servers.into_iter().map(Into::into).collect(), @@ -196,3 +159,18 @@ impl From for sys_pcf::ffi::RTCConfiguration { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_peer_connection_factory() { + let _ = env_logger::builder().is_test(true).try_init(); + + let factory = PeerConnectionFactory::default(); + let source = NativeVideoSource::default(); + let _track = factory.create_video_track("test", source); + drop(factory); + } +} diff --git a/livekit-webrtc/src/native/rtp_receiver.rs b/livekit-webrtc/src/native/rtp_receiver.rs index 478eb4f..553098d 100644 --- a/livekit-webrtc/src/native/rtp_receiver.rs +++ b/livekit-webrtc/src/native/rtp_receiver.rs @@ -1,5 +1,6 @@ -use super::media_stream::new_media_stream_track; -use crate::{media_stream::MediaStreamTrack, rtp_parameters::RtpParameters}; +use crate::imp::media_stream_track::new_media_stream_track; +use crate::media_stream_track::MediaStreamTrack; +use crate::rtp_parameters::RtpParameters; use cxx::SharedPtr; use webrtc_sys::rtp_receiver as sys_rr; diff --git a/livekit-webrtc/src/native/rtp_sender.rs b/livekit-webrtc/src/native/rtp_sender.rs index d806f6b..7df6aca 100644 --- a/livekit-webrtc/src/native/rtp_sender.rs +++ b/livekit-webrtc/src/native/rtp_sender.rs @@ -1,9 +1,9 @@ -use super::media_stream::new_media_stream_track; -use crate::{ - media_stream::MediaStreamTrack, rtp_parameters::RtpParameters, RtcError, RtcErrorType, -}; +use super::media_stream_track::new_media_stream_track; +use crate::media_stream_track::MediaStreamTrack; +use crate::{rtp_parameters::RtpParameters, RtcError, RtcErrorType}; use cxx::SharedPtr; -use webrtc_sys::{rtc_error::ffi::RTCError, rtp_sender as sys_rs}; +use webrtc_sys::rtc_error as sys_err; +use webrtc_sys::rtp_sender as sys_rs; #[derive(Clone)] pub struct RtpSender { @@ -41,6 +41,6 @@ impl RtpSender { pub fn set_parameters(&self, parameters: RtpParameters) -> Result<(), RtcError> { self.sys_handle .set_parameters(parameters.into()) - .map_err(|e| unsafe { RTCError::from(e.what()).into() }) + .map_err(|e| unsafe { sys_err::ffi::RtcError::from(e.what()).into() }) } } diff --git a/livekit-webrtc/src/native/rtp_transceiver.rs b/livekit-webrtc/src/native/rtp_transceiver.rs index 90eb75d..1de02c8 100644 --- a/livekit-webrtc/src/native/rtp_transceiver.rs +++ b/livekit-webrtc/src/native/rtp_transceiver.rs @@ -83,12 +83,12 @@ impl RtpTransceiver { pub fn set_codec_preferences(&self, codecs: Vec) -> Result<(), RtcError> { self.sys_handle .set_codec_preferences(codecs.into_iter().map(Into::into).collect()) - .map_err(|e| unsafe { sys_err::ffi::RTCError::from(e.what()).into() }) + .map_err(|e| unsafe { sys_err::ffi::RtcError::from(e.what()).into() }) } pub fn stop(&self) -> Result<(), RtcError> { self.sys_handle .stop_standard() - .map_err(|e| unsafe { sys_err::ffi::RTCError::from(e.what()).into() }) + .map_err(|e| unsafe { sys_err::ffi::RtcError::from(e.what()).into() }) } } diff --git a/livekit-webrtc/src/native/video_source.rs b/livekit-webrtc/src/native/video_source.rs index 88db0b1..102b1f0 100644 --- a/livekit-webrtc/src/native/video_source.rs +++ b/livekit-webrtc/src/native/video_source.rs @@ -1,23 +1,23 @@ use crate::video_frame::{VideoFrame, VideoFrameBuffer}; use cxx::SharedPtr; -use webrtc_sys::media_stream as ms_sys; use webrtc_sys::video_frame as vf_sys; +use webrtc_sys::video_track as vt_sys; #[derive(Clone)] pub struct NativeVideoSource { - sys_handle: SharedPtr, + sys_handle: SharedPtr, } impl Default for NativeVideoSource { fn default() -> Self { Self { - sys_handle: ms_sys::ffi::new_adapted_video_track_source(), + sys_handle: vt_sys::ffi::new_video_track_source(), } } } impl NativeVideoSource { - pub fn sys_handle(&self) -> SharedPtr { + pub fn sys_handle(&self) -> SharedPtr { self.sys_handle.clone() } @@ -27,8 +27,7 @@ impl NativeVideoSource { builder .pin_mut() .set_video_frame_buffer(frame.buffer.as_ref().sys_handle()); - - let frame = builder.pin_mut().build(); - self.sys_handle.on_captured_frame(&frame); + self.sys_handle + .on_captured_frame(&builder.pin_mut().build()); } } diff --git a/livekit-webrtc/src/native/video_stream.rs b/livekit-webrtc/src/native/video_stream.rs index 08ef3c4..52321b4 100644 --- a/livekit-webrtc/src/native/video_stream.rs +++ b/livekit-webrtc/src/native/video_stream.rs @@ -1,16 +1,16 @@ use super::video_frame::new_video_frame_buffer; -use crate::media_stream::RtcVideoTrack; use crate::video_frame::{BoxVideoFrame, VideoFrame}; -use cxx::UniquePtr; +use crate::video_track::RtcVideoTrack; +use cxx::{SharedPtr, UniquePtr}; use futures::stream::Stream; use std::pin::Pin; +use std::sync::Arc; use std::task::{Context, Poll}; use tokio::sync::mpsc; -use webrtc_sys::media_stream as sys_ms; +use webrtc_sys::video_track as sys_vt; pub struct NativeVideoStream { - native_observer: UniquePtr, - _observer: Box, + native_sink: SharedPtr, video_track: RtcVideoTrack, frame_rx: mpsc::UnboundedReceiver, } @@ -18,21 +18,16 @@ pub struct NativeVideoStream { impl NativeVideoStream { pub fn new(video_track: RtcVideoTrack) -> Self { let (frame_tx, frame_rx) = mpsc::unbounded_channel(); - let mut observer = Box::new(VideoTrackObserver { frame_tx }); - let mut native_observer = unsafe { - sys_ms::ffi::new_native_video_frame_sink(Box::new(sys_ms::VideoFrameSinkWrapper::new( - &mut *observer, - ))) - }; + let observer = Arc::new(VideoTrackObserver { frame_tx }); + let native_sink = sys_vt::ffi::new_native_video_sink(Box::new( + sys_vt::VideoSinkWrapper::new(observer.clone()), + )); - unsafe { - sys_ms::ffi::media_to_video(video_track.sys_handle()) - .add_sink(native_observer.pin_mut()); - } + let video = unsafe { sys_vt::ffi::media_to_video(video_track.sys_handle()) }; + video.add_sink(&native_sink); Self { - native_observer, - _observer: observer, + native_sink, video_track, frame_rx, } @@ -43,11 +38,10 @@ impl NativeVideoStream { } pub fn close(&mut self) { + let video = unsafe { sys_vt::ffi::media_to_video(self.video_track.sys_handle()) }; + video.remove_sink(&self.native_sink); + self.frame_rx.close(); - unsafe { - sys_ms::ffi::media_to_video(self.video_track.sys_handle()) - .remove_sink(self.native_observer.pin_mut()); - } } } @@ -69,7 +63,7 @@ struct VideoTrackObserver { frame_tx: mpsc::UnboundedSender, } -impl sys_ms::VideoFrameSink for VideoTrackObserver { +impl sys_vt::VideoSink for VideoTrackObserver { fn on_frame(&self, frame: UniquePtr) { let _ = self.frame_tx.send(VideoFrame { rotation: frame.rotation().into(), @@ -80,5 +74,5 @@ impl sys_ms::VideoFrameSink for VideoTrackObserver { fn on_discarded_frame(&self) {} - fn on_constraints_changed(&self, _constraints: sys_ms::ffi::VideoTrackSourceConstraints) {} + fn on_constraints_changed(&self, _constraints: sys_vt::ffi::VideoTrackSourceConstraints) {} } diff --git a/livekit-webrtc/src/native/video_track.rs b/livekit-webrtc/src/native/video_track.rs new file mode 100644 index 0000000..1ba2ec4 --- /dev/null +++ b/livekit-webrtc/src/native/video_track.rs @@ -0,0 +1,18 @@ +use super::media_stream_track::impl_media_stream_track; +use crate::media_stream_track::RtcTrackState; +use cxx::SharedPtr; +use sys_vt::ffi::video_to_media; +use webrtc_sys::video_track as sys_vt; + +#[derive(Clone)] +pub struct RtcVideoTrack { + pub(crate) sys_handle: SharedPtr, +} + +impl RtcVideoTrack { + impl_media_stream_track!(video_to_media); + + pub fn sys_handle(&self) -> SharedPtr { + video_to_media(self.sys_handle.clone()) + } +} diff --git a/livekit-webrtc/src/peer_connection.rs b/livekit-webrtc/src/peer_connection.rs index 45f62b3..e43af53 100644 --- a/livekit-webrtc/src/peer_connection.rs +++ b/livekit-webrtc/src/peer_connection.rs @@ -3,7 +3,8 @@ use std::fmt::Debug; use crate::data_channel::{DataChannel, DataChannelInit}; use crate::ice_candidate::IceCandidate; use crate::imp::peer_connection as imp_pc; -use crate::media_stream::{MediaStream, MediaStreamTrack}; +use crate::media_stream::MediaStream; +use crate::media_stream_track::MediaStreamTrack; use crate::rtp_receiver::RtpReceiver; use crate::rtp_sender::RtpSender; use crate::rtp_transceiver::{RtpTransceiver, RtpTransceiverInit}; diff --git a/livekit-webrtc/src/peer_connection_factory.rs b/livekit-webrtc/src/peer_connection_factory.rs index 6d1c6c5..df9f02d 100644 --- a/livekit-webrtc/src/peer_connection_factory.rs +++ b/livekit-webrtc/src/peer_connection_factory.rs @@ -33,6 +33,16 @@ pub struct RtcConfiguration { pub ice_transport_type: IceTransportsType, } +impl Default for RtcConfiguration { + fn default() -> Self { + Self { + ice_servers: vec![], + continual_gathering_policy: ContinualGatheringPolicy::GatherOnce, + ice_transport_type: IceTransportsType::All, + } + } +} + #[derive(Clone, Default)] pub struct PeerConnectionFactory { pub(crate) handle: imp_pcf::PeerConnectionFactory, @@ -64,8 +74,9 @@ impl PeerConnectionFactory { pub mod native { use super::PeerConnectionFactory; use crate::audio_source::native::NativeAudioSource; - use crate::media_stream::{RtcAudioTrack, RtcVideoTrack}; + use crate::audio_track::RtcAudioTrack; use crate::video_source::native::NativeVideoSource; + use crate::video_track::RtcVideoTrack; pub trait PeerConnectionFactoryExt { fn create_video_track(&self, label: &str, source: NativeVideoSource) -> RtcVideoTrack; diff --git a/livekit-webrtc/src/prelude.rs b/livekit-webrtc/src/prelude.rs index 08fe57a..5aedb53 100644 --- a/livekit-webrtc/src/prelude.rs +++ b/livekit-webrtc/src/prelude.rs @@ -1,11 +1,11 @@ pub use crate::audio_frame::AudioFrame; +pub use crate::audio_track::RtcAudioTrack; pub use crate::data_channel::{ DataBuffer, DataChannel, DataChannelError, DataChannelInit, DataState, }; pub use crate::ice_candidate::IceCandidate; -pub use crate::media_stream::{ - MediaStream, MediaStreamTrack, RtcAudioTrack, RtcTrackState, RtcVideoTrack, -}; +pub use crate::media_stream::MediaStream; +pub use crate::media_stream_track::{MediaStreamTrack, RtcTrackState}; pub use crate::peer_connection::{ AnswerOptions, IceConnectionState, IceGatheringState, OfferOptions, PeerConnection, PeerConnectionState, SignalingState, @@ -22,4 +22,5 @@ pub use crate::video_frame::{ BoxVideoFrame, I010Buffer, I420ABuffer, I420Buffer, I422Buffer, I444Buffer, NV12Buffer, VideoFormatType, VideoFrame, VideoFrameBuffer, VideoFrameBufferType, VideoRotation, }; -pub use crate::{RtcError, RtcErrorType}; +pub use crate::video_track::RtcVideoTrack; +pub use crate::{MediaType, RtcError, RtcErrorType}; diff --git a/livekit-webrtc/src/rtp_receiver.rs b/livekit-webrtc/src/rtp_receiver.rs index acaf71a..a345c86 100644 --- a/livekit-webrtc/src/rtp_receiver.rs +++ b/livekit-webrtc/src/rtp_receiver.rs @@ -1,7 +1,8 @@ use std::fmt::Debug; use crate::{ - imp::rtp_receiver as imp_rr, media_stream::MediaStreamTrack, rtp_parameters::RtpParameters, + imp::rtp_receiver as imp_rr, media_stream_track::MediaStreamTrack, + rtp_parameters::RtpParameters, }; #[derive(Clone)] diff --git a/livekit-webrtc/src/rtp_sender.rs b/livekit-webrtc/src/rtp_sender.rs index 6d15392..7249471 100644 --- a/livekit-webrtc/src/rtp_sender.rs +++ b/livekit-webrtc/src/rtp_sender.rs @@ -1,7 +1,7 @@ use std::fmt::Debug; use crate::{ - imp::rtp_sender as imp_rs, media_stream::MediaStreamTrack, rtp_parameters::RtpParameters, + imp::rtp_sender as imp_rs, media_stream_track::MediaStreamTrack, rtp_parameters::RtpParameters, RtcError, }; diff --git a/livekit-webrtc/src/video_stream.rs b/livekit-webrtc/src/video_stream.rs index f80b9ff..05675df 100644 --- a/livekit-webrtc/src/video_stream.rs +++ b/livekit-webrtc/src/video_stream.rs @@ -6,8 +6,8 @@ use crate::imp::video_stream as stream_imp; #[cfg(not(target_arch = "wasm32"))] pub mod native { use super::stream_imp; - use crate::media_stream::RtcVideoTrack; use crate::video_frame::BoxVideoFrame; + use crate::video_track::RtcVideoTrack; use futures::stream::Stream; use std::fmt::Debug; use std::pin::Pin; diff --git a/livekit-webrtc/src/video_track.rs b/livekit-webrtc/src/video_track.rs new file mode 100644 index 0000000..c00f59d --- /dev/null +++ b/livekit-webrtc/src/video_track.rs @@ -0,0 +1,23 @@ +use crate::imp::video_track as imp_vt; +use crate::media_stream_track::media_stream_track; +use crate::media_stream_track::RtcTrackState; +use std::fmt::Debug; + +#[derive(Clone)] +pub struct RtcVideoTrack { + pub(crate) handle: imp_vt::RtcVideoTrack, +} + +impl RtcVideoTrack { + media_stream_track!(); +} + +impl Debug for RtcVideoTrack { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RtcVideoTrack") + .field("id", &self.id()) + .field("enabled", &self.enabled()) + .field("state", &self.state()) + .finish() + } +} diff --git a/livekit/src/room/participant/remote_participant.rs b/livekit/src/room/participant/remote_participant.rs index a52babe..cec9555 100644 --- a/livekit/src/room/participant/remote_participant.rs +++ b/livekit/src/room/participant/remote_participant.rs @@ -2,9 +2,8 @@ use super::{ConnectionQuality, ParticipantInner}; use crate::prelude::*; use crate::track::TrackError; use livekit_protocol as proto; -use livekit_webrtc as rtc; +use livekit_webrtc::prelude::*; use parking_lot::RwLockReadGuard; -use rtc::prelude::MediaStreamTrack; use std::collections::{HashMap, HashSet}; use std::fmt::Debug; use std::sync::Arc; @@ -67,7 +66,7 @@ impl RemoteParticipant { pub(crate) async fn add_subscribed_media_track( &self, sid: TrackSid, - media_track: rtc::media_stream::MediaStreamTrack, + media_track: MediaStreamTrack, ) { let wait_publication = { let participant = self.clone(); diff --git a/livekit/src/room/track/local_audio_track.rs b/livekit/src/room/track/local_audio_track.rs index 2158d1d..602eda1 100644 --- a/livekit/src/room/track/local_audio_track.rs +++ b/livekit/src/room/track/local_audio_track.rs @@ -4,9 +4,8 @@ use crate::prelude::*; use crate::rtc_engine::lk_runtime::LkRuntime; use crate::webrtc::peer_connection_factory::native::PeerConnectionFactoryExt; use livekit_protocol as proto; -use livekit_webrtc as rtc; +use livekit_webrtc::prelude::*; use parking_lot::Mutex; -use rtc::audio_source::native::NativeAudioSource; use std::fmt::Debug; use std::sync::Arc; use tokio::sync::mpsc; @@ -35,7 +34,7 @@ impl Debug for LocalAudioTrack { impl LocalAudioTrack { pub(crate) fn new( name: String, - rtc_track: rtc::media_stream::RtcAudioTrack, + rtc_track: RtcAudioTrack, capture_options: AudioCaptureOptions, ) -> Self { Self { @@ -44,7 +43,7 @@ impl LocalAudioTrack { "unknown".to_string().into(), // sid name, TrackKind::Audio, - rtc::media_stream::MediaStreamTrack::Audio(rtc_track), + MediaStreamTrack::Audio(rtc_track), ), capture_options: Mutex::new(capture_options), }), @@ -102,14 +101,11 @@ impl LocalAudioTrack { } #[inline] - pub fn rtc_track(&self) -> rtc::media_stream::RtcAudioTrack { - if let rtc::media_stream::MediaStreamTrack::Audio(audio) = - self.inner.track_inner.rtc_track() - { - audio - } else { - unreachable!() + pub fn rtc_track(&self) -> RtcAudioTrack { + if let MediaStreamTrack::Audio(audio) = self.inner.track_inner.rtc_track() { + return audio; } + unreachable!() } #[inline] @@ -123,15 +119,12 @@ impl LocalAudioTrack { } #[inline] - pub(crate) fn transceiver(&self) -> Option { + pub(crate) fn transceiver(&self) -> Option { self.inner.track_inner.transceiver() } #[inline] - pub(crate) fn update_transceiver( - &self, - transceiver: Option, - ) { + pub(crate) fn update_transceiver(&self, transceiver: Option) { self.inner.track_inner.update_transceiver(transceiver) } @@ -145,11 +138,11 @@ impl LocalAudioTrack { pub fn create_audio_track( name: &str, options: AudioCaptureOptions, - source: NativeAudioSource, + source: livekit_webrtc::audio_source::native::NativeAudioSource, ) -> LocalAudioTrack { let rtc_track = LkRuntime::instance() .pc_factory() - .create_audio_track(&rtc::native::create_random_uuid(), source); + .create_audio_track(&livekit_webrtc::native::create_random_uuid(), source); Self::new(name.to_string(), rtc_track, options) } diff --git a/livekit/src/room/track/local_video_track.rs b/livekit/src/room/track/local_video_track.rs index 05b6ceb..1c9f2cf 100644 --- a/livekit/src/room/track/local_video_track.rs +++ b/livekit/src/room/track/local_video_track.rs @@ -2,10 +2,9 @@ use super::TrackInner; use crate::rtc_engine::lk_runtime::LkRuntime; use crate::{options::VideoCaptureOptions, prelude::*}; use livekit_protocol as proto; -use livekit_webrtc as rtc; use livekit_webrtc::peer_connection_factory::native::PeerConnectionFactoryExt; +use livekit_webrtc::prelude::*; use parking_lot::Mutex; -use rtc::video_source::native::NativeVideoSource; use std::fmt::Debug; use std::sync::Arc; use tokio::sync::mpsc; @@ -34,7 +33,7 @@ impl Debug for LocalVideoTrack { impl LocalVideoTrack { pub fn new( name: String, - rtc_track: rtc::media_stream::RtcVideoTrack, + rtc_track: RtcVideoTrack, capture_options: VideoCaptureOptions, ) -> Self { Self { @@ -43,7 +42,7 @@ impl LocalVideoTrack { "unknown".to_string().into(), // sid name, TrackKind::Video, - rtc::media_stream::MediaStreamTrack::Video(rtc_track), + MediaStreamTrack::Video(rtc_track), ), capture_options: Mutex::new(capture_options), }), @@ -101,14 +100,11 @@ impl LocalVideoTrack { } #[inline] - pub fn rtc_track(&self) -> rtc::media_stream::RtcVideoTrack { - if let rtc::media_stream::MediaStreamTrack::Video(video) = - self.inner.track_inner.rtc_track() - { - video - } else { - unreachable!() + pub fn rtc_track(&self) -> RtcVideoTrack { + if let MediaStreamTrack::Video(video) = self.inner.track_inner.rtc_track() { + return video; } + unreachable!() } #[inline] @@ -122,15 +118,12 @@ impl LocalVideoTrack { } #[inline] - pub(crate) fn transceiver(&self) -> Option { + pub(crate) fn transceiver(&self) -> Option { self.inner.track_inner.transceiver() } #[inline] - pub(crate) fn update_transceiver( - &self, - transceiver: Option, - ) { + pub(crate) fn update_transceiver(&self, transceiver: Option) { self.inner.track_inner.update_transceiver(transceiver) } @@ -144,11 +137,11 @@ impl LocalVideoTrack { pub fn create_video_track( name: &str, options: VideoCaptureOptions, - source: NativeVideoSource, + source: livekit_webrtc::video_source::native::NativeVideoSource, ) -> LocalVideoTrack { let rtc_track = LkRuntime::instance() .pc_factory() - .create_video_track(&rtc::native::create_random_uuid(), source); + .create_video_track(&livekit_webrtc::native::create_random_uuid(), source); Self::new(name.to_string(), rtc_track, options) } diff --git a/livekit/src/room/track/mod.rs b/livekit/src/room/track/mod.rs index 46504d2..d84ad93 100644 --- a/livekit/src/room/track/mod.rs +++ b/livekit/src/room/track/mod.rs @@ -2,9 +2,8 @@ use crate::prelude::*; use livekit_protocol as proto; use livekit_protocol::enum_dispatch; use livekit_protocol::observer::Dispatcher; -use livekit_webrtc as rtc; +use livekit_webrtc::prelude::*; use parking_lot::Mutex; -use rtc::MediaType; use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; use thiserror::Error; use tokio::sync::mpsc; @@ -103,8 +102,8 @@ macro_rules! track_dispatch { pub fn register_observer(self: &Self) -> mpsc::UnboundedReceiver; pub fn is_remote(self: &Self) -> bool; - pub(crate) fn transceiver(self: &Self) -> Option; - pub(crate) fn update_transceiver(self: &Self, transceiver: Option) -> (); + pub(crate) fn transceiver(self: &Self) -> Option; + pub(crate) fn update_transceiver(self: &Self, transceiver: Option) -> (); pub(crate) fn update_info(self: &Self, info: proto::TrackInfo) -> (); ); }; @@ -114,7 +113,7 @@ impl Track { track_dispatch!([LocalAudio, LocalVideo, RemoteAudio, RemoteVideo]); #[inline] - pub fn rtc_track(&self) -> rtc::media_stream::MediaStreamTrack { + pub fn rtc_track(&self) -> MediaStreamTrack { match self { Self::LocalAudio(track) => track.rtc_track().into(), Self::LocalVideo(track) => track.rtc_track().into(), @@ -128,7 +127,7 @@ impl LocalTrack { track_dispatch!([Audio, Video]); #[inline] - pub fn rtc_track(&self) -> rtc::media_stream::MediaStreamTrack { + pub fn rtc_track(&self) -> MediaStreamTrack { match self { Self::Audio(track) => track.rtc_track().into(), Self::Video(track) => track.rtc_track().into(), @@ -140,7 +139,7 @@ impl RemoteTrack { track_dispatch!([Audio, Video]); #[inline] - pub fn rtc_track(&self) -> rtc::media_stream::MediaStreamTrack { + pub fn rtc_track(&self) -> MediaStreamTrack { match self { Self::Audio(track) => track.rtc_track().into(), Self::Video(track) => track.rtc_track().into(), @@ -152,7 +151,7 @@ impl VideoTrack { track_dispatch!([Local, Remote]); #[inline] - pub fn rtc_track(&self) -> rtc::media_stream::RtcVideoTrack { + pub fn rtc_track(&self) -> RtcVideoTrack { match self { Self::Local(track) => track.rtc_track(), Self::Remote(track) => track.rtc_track(), @@ -164,7 +163,7 @@ impl AudioTrack { track_dispatch!([Local, Remote]); #[inline] - pub fn rtc_track(&self) -> rtc::media_stream::RtcAudioTrack { + pub fn rtc_track(&self) -> RtcAudioTrack { match self { Self::Local(track) => track.rtc_track().into(), Self::Remote(track) => track.rtc_track().into(), @@ -180,18 +179,13 @@ pub(crate) struct TrackInner { pub source: AtomicU8, // TrackSource pub stream_state: AtomicU8, // StreamState pub muted: AtomicBool, - pub rtc_track: rtc::media_stream::MediaStreamTrack, - pub transceiver: Mutex>, + pub rtc_track: MediaStreamTrack, + pub transceiver: Mutex>, pub dispatcher: Dispatcher, } impl TrackInner { - pub fn new( - sid: TrackSid, - name: String, - kind: TrackKind, - rtc_track: rtc::media_stream::MediaStreamTrack, - ) -> Self { + pub fn new(sid: TrackSid, name: String, kind: TrackKind, rtc_track: MediaStreamTrack) -> Self { Self { sid: Mutex::new(sid), name: Mutex::new(name), @@ -261,7 +255,7 @@ impl TrackInner { self.dispatcher.dispatch(&event); } - pub fn rtc_track(&self) -> rtc::media_stream::MediaStreamTrack { + pub fn rtc_track(&self) -> MediaStreamTrack { self.rtc_track.clone() } @@ -269,11 +263,11 @@ impl TrackInner { self.dispatcher.register() } - pub fn transceiver(&self) -> Option { + pub fn transceiver(&self) -> Option { self.transceiver.lock().clone() } - pub fn update_transceiver(&self, transceiver: Option) { + pub fn update_transceiver(&self, transceiver: Option) { *self.transceiver.lock() = transceiver; } diff --git a/livekit/src/room/track/remote_audio_track.rs b/livekit/src/room/track/remote_audio_track.rs index eb88acd..c5ca90f 100644 --- a/livekit/src/room/track/remote_audio_track.rs +++ b/livekit/src/room/track/remote_audio_track.rs @@ -1,7 +1,7 @@ use super::TrackInner; use crate::prelude::*; use livekit_protocol as proto; -use livekit_webrtc as rtc; +use livekit_webrtc::prelude::*; use std::fmt::Debug; use std::sync::Arc; use tokio::sync::mpsc; @@ -22,17 +22,13 @@ impl Debug for RemoteAudioTrack { } impl RemoteAudioTrack { - pub(crate) fn new( - sid: TrackSid, - name: String, - rtc_track: rtc::media_stream::RtcAudioTrack, - ) -> Self { + pub(crate) fn new(sid: TrackSid, name: String, rtc_track: RtcAudioTrack) -> Self { Self { inner: Arc::new(TrackInner::new( sid, name, TrackKind::Audio, - rtc::media_stream::MediaStreamTrack::Audio(rtc_track), + MediaStreamTrack::Audio(rtc_track), )), } } @@ -83,12 +79,11 @@ impl RemoteAudioTrack { } #[inline] - pub fn rtc_track(&self) -> rtc::media_stream::RtcAudioTrack { - if let rtc::media_stream::MediaStreamTrack::Audio(audio) = self.inner.rtc_track() { - audio - } else { - unreachable!() + pub fn rtc_track(&self) -> RtcAudioTrack { + if let MediaStreamTrack::Audio(audio) = self.inner.rtc_track() { + return audio; } + unreachable!() } #[inline] @@ -103,16 +98,13 @@ impl RemoteAudioTrack { #[allow(dead_code)] #[inline] - pub(crate) fn transceiver(&self) -> Option { + pub(crate) fn transceiver(&self) -> Option { self.inner.transceiver() } #[inline] #[allow(dead_code)] - pub(crate) fn update_transceiver( - &self, - transceiver: Option, - ) { + pub(crate) fn update_transceiver(&self, transceiver: Option) { self.inner.update_transceiver(transceiver) } diff --git a/livekit/src/room/track/remote_video_track.rs b/livekit/src/room/track/remote_video_track.rs index c6d9e89..7f6ef96 100644 --- a/livekit/src/room/track/remote_video_track.rs +++ b/livekit/src/room/track/remote_video_track.rs @@ -1,7 +1,7 @@ use super::TrackInner; use crate::prelude::*; use livekit_protocol as proto; -use livekit_webrtc as rtc; +use livekit_webrtc::prelude::*; use std::fmt::Debug; use std::sync::Arc; use tokio::sync::mpsc; @@ -22,17 +22,13 @@ impl Debug for RemoteVideoTrack { } impl RemoteVideoTrack { - pub(crate) fn new( - sid: TrackSid, - name: String, - rtc_track: rtc::media_stream::RtcVideoTrack, - ) -> Self { + pub(crate) fn new(sid: TrackSid, name: String, rtc_track: RtcVideoTrack) -> Self { Self { inner: Arc::new(TrackInner::new( sid, name, TrackKind::Video, - rtc::media_stream::MediaStreamTrack::Video(rtc_track), + MediaStreamTrack::Video(rtc_track), )), } } @@ -83,12 +79,11 @@ impl RemoteVideoTrack { } #[inline] - pub fn rtc_track(&self) -> rtc::media_stream::RtcVideoTrack { - if let rtc::media_stream::MediaStreamTrack::Video(video) = self.inner.rtc_track() { - video - } else { - unreachable!() + pub fn rtc_track(&self) -> RtcVideoTrack { + if let MediaStreamTrack::Video(video) = self.inner.rtc_track() { + return video; } + unreachable!() } #[inline] @@ -103,16 +98,13 @@ impl RemoteVideoTrack { #[allow(dead_code)] #[inline] - pub(crate) fn transceiver(&self) -> Option { + pub(crate) fn transceiver(&self) -> Option { self.inner.transceiver() } #[allow(dead_code)] #[inline] - pub(crate) fn update_transceiver( - &self, - transceiver: Option, - ) { + pub(crate) fn update_transceiver(&self, transceiver: Option) { self.inner.update_transceiver(transceiver) } diff --git a/webrtc-sys/build.rs b/webrtc-sys/build.rs index a5cbfab..468fff1 100644 --- a/webrtc-sys/build.rs +++ b/webrtc-sys/build.rs @@ -136,6 +136,9 @@ fn main() { "src/peer_connection.rs", "src/peer_connection_factory.rs", "src/media_stream.rs", + "src/media_stream_track.rs", + "src/audio_track.rs", + "src/video_track.rs", "src/data_channel.rs", "src/jsep.rs", "src/candidate.rs", @@ -150,13 +153,15 @@ fn main() { "src/yuv_helper.rs", "src/helper.rs", "src/audio_resampler.rs", - "src/logsink.rs", ]); builder.files(&[ "src/peer_connection.cpp", "src/peer_connection_factory.cpp", "src/media_stream.cpp", + "src/media_stream_track.cpp", + "src/audio_track.cpp", + "src/video_track.cpp", "src/data_channel.cpp", "src/jsep.cpp", "src/candidate.cpp", @@ -172,7 +177,6 @@ fn main() { "src/video_decoder_factory.cpp", "src/audio_device.cpp", "src/audio_resampler.cpp", - "src/logsink.cpp", ]); for include in includes { diff --git a/webrtc-sys/include/livekit/audio_track.h b/webrtc-sys/include/livekit/audio_track.h new file mode 100644 index 0000000..34e6f48 --- /dev/null +++ b/webrtc-sys/include/livekit/audio_track.h @@ -0,0 +1,134 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the “License”); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an “AS IS” BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "livekit/helper.h" +#include "livekit/media_stream_track.h" +#include "livekit/webrtc.h" +#include "pc/local_audio_source.h" +#include "rtc_base/synchronization/mutex.h" +#include "rust/cxx.h" + +namespace livekit { +class AudioTrack; +class NativeAudioSink; +class AudioTrackSource; +} // namespace livekit +#include "webrtc-sys/src/audio_track.rs.h" + +namespace livekit { + +class AudioTrack : public MediaStreamTrack { + private: + friend RtcRuntime; + AudioTrack(std::shared_ptr rtc_runtime, + rtc::scoped_refptr track); + + public: + ~AudioTrack(); + + void add_sink(const std::shared_ptr& sink) const; + void remove_sink(const std::shared_ptr& sink) const; + + private: + webrtc::AudioTrackInterface* track() const { + return static_cast(track_.get()); + } + + mutable webrtc::Mutex mutex_; + + // Same for VideoTrack: + // Keep a strong reference to the added sinks, so we don't need to + // manage the lifetime safety on the Rust side + mutable std::vector> sinks_; +}; + +class NativeAudioSink : public webrtc::AudioTrackSinkInterface { + public: + explicit NativeAudioSink(rust::Box observer); + void OnData(const void* audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames) override; + + private: + rust::Box observer_; +}; + +std::shared_ptr new_native_audio_sink( + rust::Box observer); + +class AudioTrackSource { + class InternalSource : public webrtc::LocalAudioSource { + public: + InternalSource(); + + SourceState state() const override; + bool remote() const override; + + const cricket::AudioOptions options() const override; + + void AddSink(webrtc::AudioTrackSinkInterface* sink) override; + void RemoveSink(webrtc::AudioTrackSinkInterface* sink) override; + + // AudioFrame should always contain 10 ms worth of data (see index.md of + // acm) + void on_captured_frame(rust::Slice audio_data, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames); + + private: + webrtc::Mutex mutex_; + std::vector sinks_; + cricket::AudioOptions options_{}; + }; + + public: + AudioTrackSource(); + + void on_captured_frame(rust::Slice audio_data, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames) const; + + rtc::scoped_refptr get() const; + + private: + rtc::scoped_refptr source_; +}; + +std::shared_ptr new_audio_track_source(); + +static std::shared_ptr audio_to_media( + std::shared_ptr track) { + return track; +} + +static std::shared_ptr media_to_audio( + std::shared_ptr track) { + return std::static_pointer_cast(track); +} + +static std::shared_ptr _shared_audio_track() { + return nullptr; // Ignore +} + +} // namespace livekit diff --git a/webrtc-sys/include/livekit/data_channel.h b/webrtc-sys/include/livekit/data_channel.h index 5c48e43..9c64aba 100644 --- a/webrtc-sys/include/livekit/data_channel.h +++ b/webrtc-sys/include/livekit/data_channel.h @@ -17,27 +17,31 @@ #pragma once #include +#include #include "api/data_channel_interface.h" #include "livekit/webrtc.h" +#include "rtc_base/synchronization/mutex.h" #include "rust/cxx.h" namespace livekit { class DataChannel; -using NativeDataChannelInit = webrtc::DataChannelInit; -class NativeDataChannelObserver; } // namespace livekit #include "webrtc-sys/src/data_channel.rs.h" namespace livekit { +class NativeDataChannelObserver; + +webrtc::DataChannelInit to_native_data_channel_init(DataChannelInit init); + class DataChannel { public: explicit DataChannel( - std::shared_ptr rtc_runtime, + std::shared_ptr rtc_runtime, rtc::scoped_refptr data_channel); - void register_observer(NativeDataChannelObserver* observer) const; + void register_observer(rust::Box observer) const; void unregister_observer() const; bool send(const DataBuffer& buffer) const; rust::String label() const; @@ -45,22 +49,20 @@ class DataChannel { void close() const; private: - std::shared_ptr rtc_runtime_; + mutable webrtc::Mutex mutex_; + std::shared_ptr rtc_runtime_; rtc::scoped_refptr data_channel_; + mutable std::unique_ptr observer_; }; -std::unique_ptr create_data_channel_init( - DataChannelInit init); - static std::shared_ptr _shared_data_channel() { return nullptr; // Ignore } class NativeDataChannelObserver : public webrtc::DataChannelObserver { public: - explicit NativeDataChannelObserver( - rust::Box observer, - DataChannel* dc); + NativeDataChannelObserver(rust::Box observer, + const DataChannel* dc); ~NativeDataChannelObserver(); @@ -70,10 +72,7 @@ class NativeDataChannelObserver : public webrtc::DataChannelObserver { private: rust::Box observer_; - DataChannel* dc_; + const DataChannel* dc_; }; -std::shared_ptr create_native_data_channel_observer( - rust::Box observer, - DataChannel* dc); } // namespace livekit diff --git a/webrtc-sys/include/livekit/jsep.h b/webrtc-sys/include/livekit/jsep.h index f720753..826d4ac 100644 --- a/webrtc-sys/include/livekit/jsep.h +++ b/webrtc-sys/include/livekit/jsep.h @@ -22,19 +22,19 @@ #include "api/ref_counted_base.h" #include "api/set_local_description_observer_interface.h" #include "api/set_remote_description_observer_interface.h" +#include "livekit/rtc_error.h" #include "rust/cxx.h" namespace livekit { class IceCandidate; class SessionDescription; -struct NativeCreateSdpObserverHandle; -struct NativeSetLocalSdpObserverHandle; -struct NativeSetRemoteSdpObserverHandle; }; // namespace livekit #include "webrtc-sys/src/jsep.rs.h" namespace livekit { +class AsyncContext; + class IceCandidate { public: explicit IceCandidate( @@ -82,69 +82,51 @@ static std::unique_ptr _unique_session_description() { return nullptr; // Ignore } -// SetCreateSdpObserver - class NativeCreateSdpObserver : public webrtc::CreateSessionDescriptionObserver { public: - explicit NativeCreateSdpObserver( - rust::Box observer); + NativeCreateSdpObserver( + rust::Box ctx, + rust::Fn ctx, + std::unique_ptr)> on_success, + rust::Fn ctx, RtcError)> on_error); void OnSuccess(webrtc::SessionDescriptionInterface* desc) override; void OnFailure(webrtc::RTCError error) override; private: - rust::Box observer_; + rust::Box ctx_; + rust::Fn, std::unique_ptr)> + on_success_; + rust::Fn, RtcError)> on_error_; }; -struct NativeCreateSdpObserverHandle { - rtc::scoped_refptr observer; -}; - -std::unique_ptr -create_native_create_sdp_observer(rust::Box observer); - -// SetLocalSdpObserver - class NativeSetLocalSdpObserver : public webrtc::SetLocalDescriptionObserverInterface { public: - explicit NativeSetLocalSdpObserver( - rust::Box observer); + NativeSetLocalSdpObserver( + rust::Box ctx, + rust::Fn, RtcError)> on_complete); void OnSetLocalDescriptionComplete(webrtc::RTCError error) override; private: - rust::Box observer_; + rust::Box ctx_; + rust::Fn, RtcError)> on_complete_; }; -struct NativeSetLocalSdpObserverHandle { - rtc::scoped_refptr observer; -}; - -std::unique_ptr -create_native_set_local_sdp_observer( - rust::Box observer); - -// SetRemoteSdpObserver - class NativeSetRemoteSdpObserver : public webrtc::SetRemoteDescriptionObserverInterface { public: - explicit NativeSetRemoteSdpObserver( - rust::Box observer); + NativeSetRemoteSdpObserver( + rust::Box ctx, + rust::Fn, RtcError)> on_complete); void OnSetRemoteDescriptionComplete(webrtc::RTCError error) override; private: - rust::Box observer_; + rust::Box ctx_; + rust::Fn, RtcError)> on_complete_; }; -struct NativeSetRemoteSdpObserverHandle { - rtc::scoped_refptr observer; -}; - -std::unique_ptr -create_native_set_remote_sdp_observer( - rust::Box observer); } // namespace livekit diff --git a/webrtc-sys/include/livekit/logsink.h b/webrtc-sys/include/livekit/logsink.h deleted file mode 100644 index b6449ee..0000000 --- a/webrtc-sys/include/livekit/logsink.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2023 LiveKit - * - * Licensed under the Apache License, Version 2.0 (the “License”); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an “AS IS” BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "livekit/webrtc.h" -#include "rtc_base/logging.h" -#include "rust/cxx.h" - -namespace livekit { -class LogSink; -} // namespace livekit -#include "webrtc-sys/src/logsink.rs.h" - - -namespace livekit { - -class LogSink : public rtc::LogSink { -public: - LogSink(rust::Fn fnc); - ~LogSink(); - - void OnLogMessage(const std::string& message, rtc::LoggingSeverity severity) override; - void OnLogMessage(const std::string& message) override {} - -private: - rust::Fn fnc_; -}; - -std::unique_ptr new_log_sink(rust::Fn fnc); - -} // namespace livekit - diff --git a/webrtc-sys/include/livekit/media_stream.h b/webrtc-sys/include/livekit/media_stream.h index 6d94bfc..fe64dbe 100644 --- a/webrtc-sys/include/livekit/media_stream.h +++ b/webrtc-sys/include/livekit/media_stream.h @@ -19,27 +19,12 @@ #include #include "api/media_stream_interface.h" -#include "api/video/video_frame.h" -#include "common_audio/resampler/include/push_resampler.h" -#include "common_audio/ring_buffer.h" #include "livekit/helper.h" -#include "livekit/video_frame.h" -#include "media/base/adapted_video_track_source.h" -#include "pc/local_audio_source.h" -#include "rtc_base/synchronization/mutex.h" -#include "rtc_base/timestamp_aligner.h" +#include "livekit/webrtc.h" #include "rust/cxx.h" -#include "system_wrappers/include/clock.h" namespace livekit { class MediaStream; -class MediaStreamTrack; -class VideoTrack; -class AudioTrack; -class NativeVideoFrameSink; -class NativeAudioSink; -class AudioTrackSource; -class AdaptedVideoTrackSource; } // namespace livekit #include "webrtc-sys/src/media_stream.rs.h" @@ -47,7 +32,8 @@ namespace livekit { class MediaStream { public: - explicit MediaStream(rtc::scoped_refptr stream); + MediaStream(std::shared_ptr rtc_runtime, + rtc::scoped_refptr stream); rust::String id() const; rust::Vec get_video_tracks() const; @@ -60,204 +46,10 @@ class MediaStream { bool remove_track(std::shared_ptr track) const; private: + std::shared_ptr rtc_runtime_; rtc::scoped_refptr media_stream_; }; -class MediaStreamTrack { - protected: - explicit MediaStreamTrack( - rtc::scoped_refptr track); - - public: - static std::shared_ptr from( - rtc::scoped_refptr track); - - rust::String kind() const; - rust::String id() const; - - bool enabled() const; - bool set_enabled(bool enable) const; - - TrackState state() const; - - rtc::scoped_refptr get() const { - return track_; - } - - protected: - rtc::scoped_refptr track_; -}; - -class AudioTrack : public MediaStreamTrack { - public: - explicit AudioTrack(rtc::scoped_refptr track); - - void add_sink(NativeAudioSink& sink) const; - void remove_sink(NativeAudioSink& sink) const; - - private: - webrtc::AudioTrackInterface* track() const { - return static_cast(track_.get()); - } -}; - -class NativeAudioSink : public webrtc::AudioTrackSinkInterface { - public: - explicit NativeAudioSink(rust::Box observer); - void OnData(const void* audio_data, - int bits_per_sample, - int sample_rate, - size_t number_of_channels, - size_t number_of_frames) override; - - private: - rust::Box observer_; -}; - -std::unique_ptr new_native_audio_sink( - rust::Box observer); - -class NativeAudioTrackSource : public webrtc::LocalAudioSource { - public: - NativeAudioTrackSource(); - - SourceState state() const override; - bool remote() const override; - - const cricket::AudioOptions options() const override; - - void AddSink(webrtc::AudioTrackSinkInterface* sink) override; - void RemoveSink(webrtc::AudioTrackSinkInterface* sink) override; - - // AudioFrame should always contain 10 ms worth of data (see index.md of acm) - void on_captured_frame(const int16_t* audio_data, - int sample_rate, - size_t number_of_channels, - size_t number_of_frames); - - private: - webrtc::Mutex mutex_; - std::vector sinks_; - cricket::AudioOptions options_{}; -}; - -class AudioTrackSource { - public: - AudioTrackSource(rtc::scoped_refptr source); - - void on_captured_frame(const int16_t* audio_data, - int sample_rate, - size_t number_of_channels, - size_t number_of_frames) const; - - rtc::scoped_refptr get() const; - - private: - rtc::scoped_refptr source_; -}; - -std::shared_ptr new_audio_track_source(); - -class VideoTrack : public MediaStreamTrack { - public: - explicit VideoTrack(rtc::scoped_refptr track); - - void add_sink(NativeVideoFrameSink& sink) const; - void remove_sink(NativeVideoFrameSink& sink) const; - - void set_should_receive(bool should_receive) const; - bool should_receive() const; - ContentHint content_hint() const; - void set_content_hint(ContentHint hint) const; - - private: - webrtc::VideoTrackInterface* track() const { - return static_cast(track_.get()); - } -}; - -class NativeVideoFrameSink - : public rtc::VideoSinkInterface { - public: - explicit NativeVideoFrameSink(rust::Box observer); - - void OnFrame(const webrtc::VideoFrame& frame) override; - void OnDiscardedFrame() override; - void OnConstraintsChanged( - const webrtc::VideoTrackSourceConstraints& constraints) override; - - private: - rust::Box observer_; -}; - -std::unique_ptr new_native_video_frame_sink( - rust::Box observer); - -// Native impl of the WebRTC interface -class NativeVideoTrackSource : public rtc::AdaptedVideoTrackSource { - public: - NativeVideoTrackSource(); - ~NativeVideoTrackSource() override; - - bool is_screencast() const override; - absl::optional needs_denoising() const override; - SourceState state() const override; - bool remote() const override; - - bool on_captured_frame(const webrtc::VideoFrame& frame); - - private: - webrtc::Mutex mutex_; - rtc::TimestampAligner timestamp_aligner_; -}; - -class AdaptedVideoTrackSource { - public: - AdaptedVideoTrackSource(rtc::scoped_refptr source); - - bool on_captured_frame(const std::unique_ptr& frame) - const; // frames pushed from Rust (+interior mutability) - - rtc::scoped_refptr get() const; - - private: - rtc::scoped_refptr source_; -}; - -std::shared_ptr new_adapted_video_track_source(); - -static std::shared_ptr video_to_media( - std::shared_ptr track) { - return track; -} - -static std::shared_ptr audio_to_media( - std::shared_ptr track) { - return track; -} - -static std::shared_ptr media_to_video( - std::shared_ptr track) { - return std::static_pointer_cast(track); -} - -static std::shared_ptr media_to_audio( - std::shared_ptr track) { - return std::static_pointer_cast(track); -} - -static std::shared_ptr _shared_media_stream_track() { - return nullptr; // Ignore -} - -static std::shared_ptr _shared_audio_track() { - return nullptr; // Ignore -} - -static std::shared_ptr _shared_video_track() { - return nullptr; // Ignore -} - static std::shared_ptr _shared_media_stream() { return nullptr; // Ignore } diff --git a/webrtc-sys/include/livekit/media_stream_track.h b/webrtc-sys/include/livekit/media_stream_track.h new file mode 100644 index 0000000..6be6904 --- /dev/null +++ b/webrtc-sys/include/livekit/media_stream_track.h @@ -0,0 +1,60 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the “License”); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an “AS IS” BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "api/media_stream_interface.h" +#include "livekit/helper.h" +#include "livekit/webrtc.h" +#include "rust/cxx.h" + +namespace livekit { +class MediaStreamTrack; +} // namespace livekit +#include "webrtc-sys/src/media_stream_track.rs.h" + +namespace livekit { + +class MediaStreamTrack { + protected: + MediaStreamTrack(std::shared_ptr, + rtc::scoped_refptr track); + + public: + rust::String kind() const; + rust::String id() const; + + bool enabled() const; + bool set_enabled(bool enable) const; + + TrackState state() const; + + rtc::scoped_refptr rtc_track() const { + return track_; + } + + protected: + std::shared_ptr rtc_runtime_; + rtc::scoped_refptr track_; +}; + +static std::shared_ptr _shared_media_stream_track() { + return nullptr; // Ignore +} + +} // namespace livekit diff --git a/webrtc-sys/include/livekit/peer_connection.h b/webrtc-sys/include/livekit/peer_connection.h index 8d2801a..d3d6bf9 100644 --- a/webrtc-sys/include/livekit/peer_connection.h +++ b/webrtc-sys/include/livekit/peer_connection.h @@ -23,46 +23,62 @@ #include "livekit/helper.h" #include "livekit/jsep.h" #include "livekit/media_stream.h" +#include "livekit/rtc_error.h" #include "livekit/rtp_receiver.h" #include "livekit/rtp_sender.h" #include "livekit/rtp_transceiver.h" #include "livekit/webrtc.h" #include "rust/cxx.h" +#include "webrtc-sys/src/data_channel.rs.h" namespace livekit { -class NativeAddIceCandidateObserver; class PeerConnection; -class NativeAddIceCandidateObserver; class NativePeerConnectionObserver; } // namespace livekit #include "webrtc-sys/src/peer_connection.rs.h" namespace livekit { +class PeerConnectionFactory; + class PeerConnection { public: - explicit PeerConnection( - std::shared_ptr rtc_runtime, + PeerConnection( + std::shared_ptr rtc_runtime, + std::unique_ptr observer, rtc::scoped_refptr peer_connection); - void create_offer(NativeCreateSdpObserverHandle& observer, - RTCOfferAnswerOptions options) const; + void create_offer( + RtcOfferAnswerOptions options, + rust::Box ctx, + rust::Fn, + std::unique_ptr)> on_success, + rust::Fn, RtcError)> on_error) const; - void create_answer(NativeCreateSdpObserverHandle& observer, - RTCOfferAnswerOptions options) const; + void create_answer( + RtcOfferAnswerOptions options, + rust::Box ctx, + rust::Fn, + std::unique_ptr)> on_success, + rust::Fn, RtcError)> on_error) const; - void set_local_description(std::unique_ptr desc, - NativeSetLocalSdpObserverHandle& observer) const; + void set_local_description( + std::unique_ptr desc, + rust::Box ctx, + rust::Fn, RtcError)> on_complete) const; - void set_remote_description(std::unique_ptr desc, - NativeSetRemoteSdpObserverHandle& observer) const; + void set_remote_description( + std::unique_ptr desc, + rust::Box ctx, + rust::Fn, RtcError)> on_complete) const; - std::shared_ptr create_data_channel( - rust::String label, - std::unique_ptr init) const; + std::shared_ptr create_data_channel(rust::String label, + DataChannelInit init) const; - void add_ice_candidate(std::shared_ptr candidate, - NativeAddIceCandidateObserver& observer) const; + void add_ice_candidate( + std::shared_ptr candidate, + rust::Box ctx, + rust::Fn, RtcError)> on_complete) const; std::shared_ptr add_track( std::shared_ptr track, @@ -107,7 +123,8 @@ class PeerConnection { void close() const; private: - std::shared_ptr rtc_runtime_; + std::shared_ptr rtc_runtime_; + std::unique_ptr observer_; rtc::scoped_refptr peer_connection_; }; @@ -115,25 +132,9 @@ static std::shared_ptr _shared_peer_connection() { return nullptr; // Ignore } -class NativeAddIceCandidateObserver { - public: - explicit NativeAddIceCandidateObserver( - rust::Box observer); - - void OnComplete(const RTCError& error); - - private: - rust::Box observer_; -}; - -std::unique_ptr -create_native_add_ice_candidate_observer( - rust::Box observer); - class NativePeerConnectionObserver : public webrtc::PeerConnectionObserver { public: - explicit NativePeerConnectionObserver( - std::shared_ptr rtc_runtime, + NativePeerConnectionObserver( rust::Box observer); ~NativePeerConnectionObserver(); @@ -196,12 +197,15 @@ class NativePeerConnectionObserver : public webrtc::PeerConnectionObserver { void OnInterestingUsage(int usage_pattern) override; private: - std::shared_ptr rtc_runtime_; + friend PeerConnectionFactory; + // The RtcRuntime is set inside PeerConnectionFactory, we can simplify that + // once create_native_connection_observer is removed + std::shared_ptr rtc_runtime_; rust::Box observer_; }; -std::shared_ptr +std::unique_ptr create_native_peer_connection_observer( - std::shared_ptr rtc_runtime, rust::Box observer); + } // namespace livekit diff --git a/webrtc-sys/include/livekit/peer_connection_factory.h b/webrtc-sys/include/livekit/peer_connection_factory.h index 884bcef..bb110e5 100644 --- a/webrtc-sys/include/livekit/peer_connection_factory.h +++ b/webrtc-sys/include/livekit/peer_connection_factory.h @@ -21,30 +21,31 @@ #include "peer_connection.h" #include "rtp_parameters.h" #include "rust/cxx.h" +#include "webrtc-sys/src/peer_connection.rs.h" #include "webrtc.h" namespace livekit { -using NativeRTCConfiguration = - webrtc::PeerConnectionInterface::RTCConfiguration; - class PeerConnectionFactory; } // namespace livekit #include "webrtc-sys/src/peer_connection_factory.rs.h" namespace livekit { +webrtc::PeerConnectionInterface::RTCConfiguration to_native_rtc_configuration( + RtcConfiguration config); + class PeerConnectionFactory { public: - explicit PeerConnectionFactory(std::shared_ptr rtc_runtime); + explicit PeerConnectionFactory(std::shared_ptr rtc_runtime); ~PeerConnectionFactory(); std::shared_ptr create_peer_connection( - std::unique_ptr config, - NativePeerConnectionObserver* observer) const; + RtcConfiguration config, + std::unique_ptr observer) const; std::shared_ptr create_video_track( rust::String label, - std::shared_ptr source) const; + std::shared_ptr source) const; std::shared_ptr create_audio_track( rust::String label, @@ -55,12 +56,9 @@ class PeerConnectionFactory { RtpCapabilities get_rtp_receiver_capabilities(MediaType type) const; private: - std::shared_ptr rtc_runtime_; + std::shared_ptr rtc_runtime_; rtc::scoped_refptr peer_factory_; }; -std::shared_ptr create_peer_connection_factory( - std::shared_ptr rtc_runtime); -std::unique_ptr create_rtc_configuration( - RTCConfiguration conf); +std::shared_ptr create_peer_connection_factory(); } // namespace livekit diff --git a/webrtc-sys/include/livekit/rtc_error.h b/webrtc-sys/include/livekit/rtc_error.h index f09eede..1240642 100644 --- a/webrtc-sys/include/livekit/rtc_error.h +++ b/webrtc-sys/include/livekit/rtc_error.h @@ -22,9 +22,9 @@ namespace livekit { -RTCError to_error(const webrtc::RTCError& error); +RtcError to_error(const webrtc::RTCError& error); std::string serialize_error( - const RTCError& error); // to be used inside cxx::Exception msg + const RtcError& error); // to be used inside cxx::Exception msg #ifdef LIVEKIT_TEST rust::String serialize_deserialize(); diff --git a/webrtc-sys/include/livekit/rtp_receiver.h b/webrtc-sys/include/livekit/rtp_receiver.h index aa64136..31ff174 100644 --- a/webrtc-sys/include/livekit/rtp_receiver.h +++ b/webrtc-sys/include/livekit/rtp_receiver.h @@ -22,6 +22,7 @@ #include "livekit/helper.h" #include "livekit/media_stream.h" #include "livekit/rtp_parameters.h" +#include "livekit/webrtc.h" #include "rust/cxx.h" namespace livekit { @@ -35,8 +36,8 @@ namespace livekit { // TODO(theomonnom): FrameTransformer & FrameDecryptor interface class RtpReceiver { public: - explicit RtpReceiver( - rtc::scoped_refptr receiver); + RtpReceiver(std::shared_ptr rtc_runtime, + rtc::scoped_refptr receiver); std::shared_ptr track() const; @@ -53,7 +54,12 @@ class RtpReceiver { void set_jitter_buffer_minimum_delay(bool is_some, double delay_seconds) const; + rtc::scoped_refptr rtc_receiver() const { + return receiver_; + } + private: + std::shared_ptr rtc_runtime_; rtc::scoped_refptr receiver_; }; diff --git a/webrtc-sys/include/livekit/rtp_sender.h b/webrtc-sys/include/livekit/rtp_sender.h index 5d12164..73c3c88 100644 --- a/webrtc-sys/include/livekit/rtp_sender.h +++ b/webrtc-sys/include/livekit/rtp_sender.h @@ -34,7 +34,8 @@ namespace livekit { // TODO(theomonnom): FrameTransformer & FrameEncryptor interface class RtpSender { public: - explicit RtpSender(rtc::scoped_refptr sender); + RtpSender(std::shared_ptr rtc_runtime, + rtc::scoped_refptr sender); bool set_track(std::shared_ptr track) const; @@ -56,9 +57,12 @@ class RtpSender { void set_parameters(RtpParameters params) const; - rtc::scoped_refptr get() const { return sender_; } + rtc::scoped_refptr rtc_sender() const { + return sender_; + } private: + std::shared_ptr rtc_runtime_; rtc::scoped_refptr sender_; }; diff --git a/webrtc-sys/include/livekit/rtp_transceiver.h b/webrtc-sys/include/livekit/rtp_transceiver.h index 4b74e7a..06864c5 100644 --- a/webrtc-sys/include/livekit/rtp_transceiver.h +++ b/webrtc-sys/include/livekit/rtp_transceiver.h @@ -39,7 +39,8 @@ webrtc::RtpTransceiverInit to_native_rtp_transceiver_init( class RtpTransceiver { public: - explicit RtpTransceiver( + RtpTransceiver( + std::shared_ptr rtc_runtime, rtc::scoped_refptr transceiver); MediaType media_type() const; @@ -76,6 +77,7 @@ class RtpTransceiver { rust::Vec header_extensions_to_offer) const; private: + std::shared_ptr rtc_runtime_; rtc::scoped_refptr transceiver_; }; diff --git a/webrtc-sys/include/livekit/video_track.h b/webrtc-sys/include/livekit/video_track.h new file mode 100644 index 0000000..75ce556 --- /dev/null +++ b/webrtc-sys/include/livekit/video_track.h @@ -0,0 +1,133 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the “License”); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an “AS IS” BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "api/media_stream_interface.h" +#include "api/video/video_frame.h" +#include "livekit/helper.h" +#include "livekit/media_stream_track.h" +#include "livekit/video_frame.h" +#include "livekit/webrtc.h" +#include "media/base/adapted_video_track_source.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/timestamp_aligner.h" +#include "rust/cxx.h" + +namespace livekit { +class VideoTrack; +class NativeVideoSink; +class VideoTrackSource; +} // namespace livekit +#include "webrtc-sys/src/video_track.rs.h" + +namespace livekit { + +class VideoTrack : public MediaStreamTrack { + private: + friend RtcRuntime; + VideoTrack(std::shared_ptr rtc_runtime, + rtc::scoped_refptr track); + + public: + ~VideoTrack(); + + void add_sink(const std::shared_ptr& sink) const; + void remove_sink(const std::shared_ptr& sink) const; + + void set_should_receive(bool should_receive) const; + bool should_receive() const; + ContentHint content_hint() const; + void set_content_hint(ContentHint hint) const; + + private: + webrtc::VideoTrackInterface* track() const { + return static_cast(track_.get()); + } + + mutable webrtc::Mutex mutex_; + + // Same for AudioTrack: + // Keep a strong reference to the added sinks, so we don't need to + // manage the lifetime safety on the Rust side + mutable std::vector> sinks_; +}; + +class NativeVideoSink : public rtc::VideoSinkInterface { + public: + explicit NativeVideoSink(rust::Box observer); + + void OnFrame(const webrtc::VideoFrame& frame) override; + void OnDiscardedFrame() override; + void OnConstraintsChanged( + const webrtc::VideoTrackSourceConstraints& constraints) override; + + private: + rust::Box observer_; +}; + +std::shared_ptr new_native_video_sink( + rust::Box observer); + +class VideoTrackSource { + class InternalSource : public rtc::AdaptedVideoTrackSource { + public: + InternalSource(); + ~InternalSource() override; + + bool is_screencast() const override; + absl::optional needs_denoising() const override; + SourceState state() const override; + bool remote() const override; + + bool on_captured_frame(const webrtc::VideoFrame& frame); + + private: + webrtc::Mutex mutex_; + rtc::TimestampAligner timestamp_aligner_; + }; + + public: + VideoTrackSource(); + + bool on_captured_frame(const std::unique_ptr& frame) + const; // frames pushed from Rust (+interior mutability) + + rtc::scoped_refptr get() const; + + private: + rtc::scoped_refptr source_; +}; + +std::shared_ptr new_video_track_source(); + +static std::shared_ptr video_to_media( + std::shared_ptr track) { + return track; +} + +static std::shared_ptr media_to_video( + std::shared_ptr track) { + return std::static_pointer_cast(track); +} + +static std::shared_ptr _shared_video_track() { + return nullptr; // Ignore +} + +} // namespace livekit diff --git a/webrtc-sys/include/livekit/webrtc.h b/webrtc-sys/include/livekit/webrtc.h index 378d8f5..78e28ee 100644 --- a/webrtc-sys/include/livekit/webrtc.h +++ b/webrtc-sys/include/livekit/webrtc.h @@ -16,6 +16,13 @@ #pragma once +#include + +#include "api/media_stream_interface.h" +#include "api/rtp_receiver_interface.h" +#include "api/rtp_sender_interface.h" +#include "livekit/helper.h" +#include "rtc_base/logging.h" #include "rtc_base/physical_socket_server.h" #include "rtc_base/ssl_adapter.h" #include "rust/cxx.h" @@ -25,29 +32,60 @@ #endif namespace livekit { -class RTCRuntime; -} +class RtcRuntime; +class LogSink; +} // namespace livekit #include "webrtc-sys/src/webrtc.rs.h" namespace livekit { -class RTCRuntime { - public: - RTCRuntime(); - ~RTCRuntime(); +class MediaStreamTrack; +class RtpReceiver; +class RtpSender; - RTCRuntime(const RTCRuntime&) = delete; - RTCRuntime& operator=(const RTCRuntime&) = delete; +// Using a shared_ptr in RtcRuntime allows us to keep a strong reference to it +// on resources that depend on it. (e.g: AudioTrack, VideoTrack). +class RtcRuntime : public std::enable_shared_from_this { + public: + [[nodiscard]] static std::shared_ptr create() { + return std::shared_ptr(new RtcRuntime()); + } + + RtcRuntime(const RtcRuntime&) = delete; + RtcRuntime& operator=(const RtcRuntime&) = delete; + ~RtcRuntime(); rtc::Thread* network_thread() const; rtc::Thread* worker_thread() const; rtc::Thread* signaling_thread() const; + std::shared_ptr get_or_create_media_stream_track( + rtc::scoped_refptr track); + + std::shared_ptr get_or_create_audio_track( + rtc::scoped_refptr track); + + std::shared_ptr get_or_create_video_track( + rtc::scoped_refptr track); + private: + RtcRuntime(); + std::unique_ptr network_thread_; std::unique_ptr worker_thread_; std::unique_ptr signaling_thread_; + // Lists used to make sure we don't create multiple wrappers for one + // underlying webrtc object. (e.g: webrtc::VideoTrackInterface should only + // have one livekit::VideoTrack associated with it). + // The only reason we to do that is to allow to add states inside our + // wrappers (e.g: the sinks_ member inside AudioTrack) + webrtc::Mutex mutex_; + std::vector> media_stream_tracks_; + // We don't have additonal state in RtpReceiver and RtpSender atm.. + // std::vector> rtp_receivers_; + // std::vector> rtp_senders_; + #ifdef WEBRTC_WIN rtc::WinsockInitializer winsock_; rtc::PhysicalSocketServer ss_; @@ -55,8 +93,22 @@ class RTCRuntime { #endif }; +class LogSink : public rtc::LogSink { + public: + LogSink(rust::Fn fnc); + ~LogSink(); + + void OnLogMessage(const std::string& message, + rtc::LoggingSeverity severity) override; + void OnLogMessage(const std::string& message) override {} + + private: + rust::Fn fnc_; +}; + +std::unique_ptr new_log_sink( + rust::Fn fnc); + rust::String create_random_uuid(); -std::shared_ptr create_rtc_runtime(); - } // namespace livekit diff --git a/webrtc-sys/src/audio_track.cpp b/webrtc-sys/src/audio_track.cpp new file mode 100644 index 0000000..b8774bb --- /dev/null +++ b/webrtc-sys/src/audio_track.cpp @@ -0,0 +1,143 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the “License”); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an “AS IS” BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "livekit/audio_track.h" + +#include + +#include +#include +#include + +#include "api/media_stream_interface.h" +#include "audio/remix_resample.h" +#include "common_audio/include/audio_util.h" +#include "rtc_base/logging.h" +#include "rtc_base/ref_counted_object.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/time_utils.h" +#include "rust/cxx.h" + +namespace livekit { + +AudioTrack::AudioTrack(std::shared_ptr rtc_runtime, + rtc::scoped_refptr track) + : MediaStreamTrack(rtc_runtime, std::move(track)) {} + +AudioTrack::~AudioTrack() { + webrtc::MutexLock lock(&mutex_); + for (auto& sink : sinks_) { + track()->RemoveSink(sink.get()); + } +} + +void AudioTrack::add_sink(const std::shared_ptr& sink) const { + webrtc::MutexLock lock(&mutex_); + track()->AddSink(sink.get()); + sinks_.push_back(sink); +} + +void AudioTrack::remove_sink( + const std::shared_ptr& sink) const { + webrtc::MutexLock lock(&mutex_); + track()->RemoveSink(sink.get()); + sinks_.erase(std::remove(sinks_.begin(), sinks_.end(), sink), sinks_.end()); +} + +NativeAudioSink::NativeAudioSink(rust::Box observer) + : observer_(std::move(observer)) {} + +void NativeAudioSink::OnData(const void* audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames) { + RTC_CHECK_EQ(16, bits_per_sample); + rust::Slice data(static_cast(audio_data), + number_of_channels * number_of_frames); + observer_->on_data(data, sample_rate, number_of_channels, number_of_frames); +} + +std::shared_ptr new_native_audio_sink( + rust::Box observer) { + return std::make_shared(std::move(observer)); +} + +AudioTrackSource::InternalSource::InternalSource() { + options_.echo_cancellation = false; + options_.auto_gain_control = false; + options_.noise_suppression = false; +} + +webrtc::MediaSourceInterface::SourceState +AudioTrackSource::InternalSource::state() const { + return webrtc::MediaSourceInterface::SourceState::kLive; +} + +bool AudioTrackSource::InternalSource::remote() const { + return false; +} + +const cricket::AudioOptions AudioTrackSource::InternalSource::options() const { + return options_; +} + +void AudioTrackSource::InternalSource::AddSink( + webrtc::AudioTrackSinkInterface* sink) { + webrtc::MutexLock lock(&mutex_); + sinks_.push_back(sink); +} + +void AudioTrackSource::InternalSource::RemoveSink( + webrtc::AudioTrackSinkInterface* sink) { + webrtc::MutexLock lock(&mutex_); + sinks_.erase(std::remove(sinks_.begin(), sinks_.end(), sink), sinks_.end()); +} + +void AudioTrackSource::InternalSource::on_captured_frame( + rust::Slice data, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames) { + webrtc::MutexLock lock(&mutex_); + for (auto sink : sinks_) { + sink->OnData(data.data(), 16, sample_rate, number_of_channels, + number_of_frames); + } +} + +AudioTrackSource::AudioTrackSource() { + source_ = rtc::make_ref_counted(); +} + +void AudioTrackSource::on_captured_frame(rust::Slice audio_data, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames) const { + source_->on_captured_frame(audio_data, sample_rate, number_of_channels, + number_of_frames); +} + +rtc::scoped_refptr AudioTrackSource::get() + const { + return source_; +} + +std::shared_ptr new_audio_track_source() { + return std::make_shared(); +} + +} // namespace livekit diff --git a/webrtc-sys/src/audio_track.rs b/webrtc-sys/src/audio_track.rs new file mode 100644 index 0000000..715579c --- /dev/null +++ b/webrtc-sys/src/audio_track.rs @@ -0,0 +1,71 @@ +use crate::impl_thread_safety; +use std::sync::Arc; + +#[cxx::bridge(namespace = "livekit")] +pub mod ffi { + extern "C++" { + include!("livekit/media_stream_track.h"); + + type MediaStreamTrack = crate::media_stream_track::ffi::MediaStreamTrack; + } + + unsafe extern "C++" { + include!("livekit/audio_track.h"); + + type AudioTrack; + type NativeAudioSink; + type AudioTrackSource; + + fn add_sink(self: &AudioTrack, sink: &SharedPtr); + fn remove_sink(self: &AudioTrack, sink: &SharedPtr); + fn new_native_audio_sink(observer: Box) -> SharedPtr; + + fn on_captured_frame( + self: &AudioTrackSource, + data: &[i16], + sample_rate: i32, + nb_channels: usize, + nb_frames: usize, + ); + fn new_audio_track_source() -> SharedPtr; + + fn audio_to_media(track: SharedPtr) -> SharedPtr; + unsafe fn media_to_audio(track: SharedPtr) -> SharedPtr; + fn _shared_audio_track() -> SharedPtr; + } + + extern "Rust" { + type AudioSinkWrapper; + + fn on_data( + self: &AudioSinkWrapper, + data: &[i16], + sample_rate: i32, + nb_channels: usize, + nb_frames: usize, + ); + } +} + +impl_thread_safety!(ffi::AudioTrack, Send + Sync); +impl_thread_safety!(ffi::NativeAudioSink, Send + Sync); +impl_thread_safety!(ffi::AudioTrackSource, Send + Sync); + +pub trait AudioSink: Send { + fn on_data(&self, data: &[i16], sample_rate: i32, nb_channels: usize, nb_frames: usize); +} + +pub struct AudioSinkWrapper { + observer: Arc, +} + +impl AudioSinkWrapper { + pub fn new(observer: Arc) -> Self { + Self { observer } + } + + fn on_data(&self, data: &[i16], sample_rate: i32, nb_channels: usize, nb_frames: usize) { + self.observer + .on_data(data, sample_rate, nb_channels, nb_frames); + } +} diff --git a/webrtc-sys/src/data_channel.cpp b/webrtc-sys/src/data_channel.cpp index 61aa30c..d6863f7 100644 --- a/webrtc-sys/src/data_channel.cpp +++ b/webrtc-sys/src/data_channel.cpp @@ -18,22 +18,50 @@ #include +#include "rtc_base/synchronization/mutex.h" #include "webrtc-sys/src/data_channel.rs.h" namespace livekit { -DataChannel::DataChannel( - std::shared_ptr rtc_runtime, - rtc::scoped_refptr data_channel) - : rtc_runtime_(std::move(rtc_runtime)), - data_channel_(std::move(data_channel)) {} +webrtc::DataChannelInit to_native_data_channel_init(DataChannelInit init) { + webrtc::DataChannelInit rtc_init{}; + rtc_init.id = init.id; + rtc_init.negotiated = init.negotiated; + rtc_init.ordered = init.ordered; + rtc_init.protocol = init.protocol.c_str(); -void DataChannel::register_observer(NativeDataChannelObserver* observer) const { - data_channel_->RegisterObserver(observer); + if (init.has_max_retransmit_time) + rtc_init.maxRetransmitTime = init.max_retransmit_time; + + if (init.has_max_retransmits) + rtc_init.maxRetransmits = init.max_retransmits; + + if (init.has_priority) + rtc_init.priority = static_cast(init.priority); + + return rtc_init; +} + +DataChannel::DataChannel( + std::shared_ptr rtc_runtime, + rtc::scoped_refptr data_channel) + : rtc_runtime_(rtc_runtime), data_channel_(std::move(data_channel)) {} + +void DataChannel::register_observer( + rust::Box observer) const { + webrtc::MutexLock lock(&mutex_); + + data_channel_->UnregisterObserver(); + + observer_ = + std::make_unique(std::move(observer), this); + data_channel_->RegisterObserver(observer_.get()); } void DataChannel::unregister_observer() const { + webrtc::MutexLock lock(&mutex_); data_channel_->UnregisterObserver(); + observer_ = nullptr; } bool DataChannel::send(const DataBuffer& buffer) const { @@ -53,29 +81,9 @@ void DataChannel::close() const { return data_channel_->Close(); } -std::unique_ptr create_data_channel_init( - DataChannelInit init) { - auto rtc_init = std::make_unique(); - rtc_init->id = init.id; - rtc_init->negotiated = init.negotiated; - rtc_init->ordered = init.ordered; - rtc_init->protocol = init.protocol.c_str(); - - if (init.has_max_retransmit_time) - rtc_init->maxRetransmitTime = init.max_retransmit_time; - - if (init.has_max_retransmits) - rtc_init->maxRetransmits = init.max_retransmits; - - if (init.has_priority) - rtc_init->priority = static_cast(init.priority); - - return rtc_init; -} - NativeDataChannelObserver::NativeDataChannelObserver( rust::Box observer, - DataChannel* dc) + const DataChannel* dc) : observer_(std::move(observer)), dc_(dc) {} NativeDataChannelObserver::~NativeDataChannelObserver() { @@ -99,9 +107,4 @@ void NativeDataChannelObserver::OnBufferedAmountChange( observer_->on_buffered_amount_change(sent_data_size); } -std::shared_ptr create_native_data_channel_observer( - rust::Box observer, - DataChannel* dc) { - return std::make_shared(std::move(observer), dc); -} } // namespace livekit diff --git a/webrtc-sys/src/data_channel.rs b/webrtc-sys/src/data_channel.rs index a5a6cca..976ff3b 100644 --- a/webrtc-sys/src/data_channel.rs +++ b/webrtc-sys/src/data_channel.rs @@ -1,5 +1,4 @@ use crate::impl_thread_safety; -use std::slice; use std::sync::Arc; #[cxx::bridge(namespace = "livekit")] @@ -43,6 +42,22 @@ pub mod ffi { Closed, } + unsafe extern "C++" { + include!("livekit/data_channel.h"); + + type DataChannel; + + fn register_observer(self: &DataChannel, observer: Box); + fn unregister_observer(self: &DataChannel); + + fn send(self: &DataChannel, data: &DataBuffer) -> bool; + fn label(self: &DataChannel) -> String; + fn state(self: &DataChannel) -> DataState; + fn close(self: &DataChannel); + + fn _shared_data_channel() -> SharedPtr; // Ignore + } + extern "Rust" { type DataChannelObserverWrapper; @@ -50,38 +65,9 @@ pub mod ffi { fn on_message(self: &DataChannelObserverWrapper, buffer: DataBuffer); fn on_buffered_amount_change(self: &DataChannelObserverWrapper, sent_data_size: u64); } - - unsafe extern "C++" { - include!("livekit/data_channel.h"); - - type DataChannel; - type NativeDataChannelInit; - type NativeDataChannelObserver; - - /// SAFETY - /// The observer must live as long as the datachannel uses it - unsafe fn register_observer(self: &DataChannel, observer: *mut NativeDataChannelObserver); - - fn unregister_observer(self: &DataChannel); - fn send(self: &DataChannel, data: &DataBuffer) -> bool; - fn label(self: &DataChannel) -> String; - fn state(self: &DataChannel) -> DataState; - fn close(self: &DataChannel); - - fn create_data_channel_init(init: DataChannelInit) -> UniquePtr; - unsafe fn create_native_data_channel_observer( - observer: Box, - dc: *mut DataChannel, - ) -> SharedPtr; - - fn _shared_data_channel() -> SharedPtr; // Ignore - } } impl_thread_safety!(ffi::DataChannel, Send + Sync); -impl_thread_safety!(ffi::NativeDataChannelObserver, Send + Sync); - -// DataChannelObserver pub trait DataChannelObserver: Send + Sync { fn on_state_change(&self, state: ffi::DataState); @@ -104,7 +90,7 @@ impl DataChannelObserverWrapper { fn on_message(&self, buffer: ffi::DataBuffer) { unsafe { - let data = slice::from_raw_parts(buffer.ptr, buffer.len); + let data = std::slice::from_raw_parts(buffer.ptr, buffer.len); self.observer.on_message(data, buffer.binary); } } diff --git a/webrtc-sys/src/jsep.cpp b/webrtc-sys/src/jsep.cpp index aea2103..06b3f77 100644 --- a/webrtc-sys/src/jsep.cpp +++ b/webrtc-sys/src/jsep.cpp @@ -21,6 +21,7 @@ #include "livekit/rtc_error.h" #include "rtc_base/ref_counted_object.h" +#include "rust/cxx.h" namespace livekit { @@ -109,69 +110,43 @@ std::unique_ptr create_session_description( return std::make_unique(std::move(rtc_sdp)); } -// CreateSdpObserver - NativeCreateSdpObserver::NativeCreateSdpObserver( - rust::Box observer) - : observer_(std::move(observer)) {} + rust::Box ctx, + rust::Fn, std::unique_ptr)> + on_success, + rust::Fn, RtcError)> on_error) + : ctx_(std::move(ctx)), on_success_(on_success), on_error_(on_error) {} void NativeCreateSdpObserver::OnSuccess( webrtc::SessionDescriptionInterface* desc) { // We have ownership of desc - observer_->on_success(std::make_unique( - std::unique_ptr(desc))); + on_success_(std::move(ctx_), + std::make_unique( + std::unique_ptr(desc))); } void NativeCreateSdpObserver::OnFailure(webrtc::RTCError error) { - observer_->on_failure(to_error(error)); + on_error_(std::move(ctx_), to_error(error)); } -std::unique_ptr -create_native_create_sdp_observer( - rust::Box observer) { - return std::make_unique( - NativeCreateSdpObserverHandle{ - rtc::make_ref_counted(std::move(observer))}); -} - -// SetLocalSdpObserver - NativeSetLocalSdpObserver::NativeSetLocalSdpObserver( - rust::Box observer) - : observer_(std::move(observer)) {} + rust::Box ctx, + rust::Fn, RtcError)> on_complete) + : ctx_(std::move(ctx)), on_complete_(on_complete) {} void NativeSetLocalSdpObserver::OnSetLocalDescriptionComplete( webrtc::RTCError error) { - observer_->on_set_local_description_complete(to_error(error)); + on_complete_(std::move(ctx_), to_error(error)); } -std::unique_ptr -create_native_set_local_sdp_observer( - rust::Box observer) { - return std::make_unique( - NativeSetLocalSdpObserverHandle{ - rtc::make_ref_counted( - std::move(observer))}); -} - -// SetRemoteSdpObserver - NativeSetRemoteSdpObserver::NativeSetRemoteSdpObserver( - rust::Box observer) - : observer_(std::move(observer)) {} + rust::Box ctx, + rust::Fn, RtcError)> on_complete) + : ctx_(std::move(ctx)), on_complete_(on_complete) {} void NativeSetRemoteSdpObserver::OnSetRemoteDescriptionComplete( webrtc::RTCError error) { - observer_->on_set_remote_description_complete(to_error(error)); -} - -std::unique_ptr -create_native_set_remote_sdp_observer( - rust::Box observer) { - return std::make_unique( - NativeSetRemoteSdpObserverHandle{ - rtc::make_ref_counted( - std::move(observer))}); + on_complete_(std::move(ctx_), to_error(error)); } } // namespace livekit diff --git a/webrtc-sys/src/jsep.rs b/webrtc-sys/src/jsep.rs index a6aed82..77167b2 100644 --- a/webrtc-sys/src/jsep.rs +++ b/webrtc-sys/src/jsep.rs @@ -1,10 +1,6 @@ use crate::impl_thread_safety; -use cxx::UniquePtr; use std::error::Error; use std::fmt::{Display, Formatter}; -use std::mem::ManuallyDrop; - -use crate::rtc_error::ffi::RTCError; #[cxx::bridge(namespace = "livekit")] pub mod ffi { @@ -23,25 +19,10 @@ pub mod ffi { pub description: String, } - extern "Rust" { - type CreateSdpObserverWrapper; - fn on_success( - self: &CreateSdpObserverWrapper, - session_description: UniquePtr, - ); - fn on_failure(self: &CreateSdpObserverWrapper, error: RTCError); - - type SetLocalSdpObserverWrapper; - fn on_set_local_description_complete(self: &SetLocalSdpObserverWrapper, error: RTCError); - - type SetRemoteSdpObserverWrapper; - fn on_set_remote_description_complete(self: &SetRemoteSdpObserverWrapper, error: RTCError); - } - extern "C++" { include!("livekit/rtc_error.h"); - type RTCError = crate::rtc_error::ffi::RTCError; + type RtcError = crate::rtc_error::ffi::RtcError; } unsafe extern "C++" { @@ -49,9 +30,6 @@ pub mod ffi { type IceCandidate; type SessionDescription; - type NativeCreateSdpObserverHandle; - type NativeSetLocalSdpObserverHandle; - type NativeSetRemoteSdpObserverHandle; fn sdp_mid(self: &IceCandidate) -> String; fn sdp_mline_index(self: &IceCandidate) -> i32; @@ -62,21 +40,12 @@ pub mod ffi { fn stringify(self: &SessionDescription) -> String; fn clone(self: &SessionDescription) -> UniquePtr; - fn create_native_create_sdp_observer( - observer: Box, - ) -> UniquePtr; - fn create_native_set_local_sdp_observer( - observer: Box, - ) -> UniquePtr; - fn create_native_set_remote_sdp_observer( - observer: Box, - ) -> UniquePtr; - fn create_ice_candidate( sdp_mid: String, sdp_mline_index: i32, sdp: String, ) -> Result>; + fn create_session_description( sdp_type: SdpType, sdp: String, @@ -115,51 +84,6 @@ impl ffi::SdpParseError { } } -// CreateSdpObserver - -pub struct CreateSdpObserverWrapper { - pub on_success: ManuallyDrop) + Send>>, - pub on_failure: ManuallyDrop>, -} - -impl CreateSdpObserverWrapper { - fn on_success(&self, session_description: UniquePtr) { - unsafe { - std::ptr::read(&*self.on_success)(session_description); - } - } - - fn on_failure(&self, error: RTCError) { - unsafe { - std::ptr::read(&*self.on_failure)(error); - } - } -} - -// SetLocalSdpObserver - -pub struct SetLocalSdpObserverWrapper(pub ManuallyDrop>); - -impl SetLocalSdpObserverWrapper { - fn on_set_local_description_complete(&self, error: RTCError) { - unsafe { - std::ptr::read(&*self.0)(error); - } - } -} - -// SetRemoteSdpObserver - -pub struct SetRemoteSdpObserverWrapper(pub ManuallyDrop>); - -impl SetRemoteSdpObserverWrapper { - fn on_set_remote_description_complete(&self, error: RTCError) { - unsafe { - std::ptr::read(&*self.0)(error); - } - } -} - #[cfg(test)] mod tests { use log::info; diff --git a/webrtc-sys/src/lib.rs b/webrtc-sys/src/lib.rs index bc9607a..d24faea 100644 --- a/webrtc-sys/src/lib.rs +++ b/webrtc-sys/src/lib.rs @@ -1,10 +1,11 @@ pub mod audio_resampler; +pub mod audio_track; pub mod candidate; pub mod data_channel; pub mod helper; pub mod jsep; -pub mod logsink; pub mod media_stream; +pub mod media_stream_track; pub mod peer_connection; pub mod peer_connection_factory; pub mod rtc_error; @@ -14,6 +15,7 @@ pub mod rtp_sender; pub mod rtp_transceiver; pub mod video_frame; pub mod video_frame_buffer; +pub mod video_track; pub mod webrtc; pub mod yuv_helper; diff --git a/webrtc-sys/src/logsink.cpp b/webrtc-sys/src/logsink.cpp deleted file mode 100644 index c543edf..0000000 --- a/webrtc-sys/src/logsink.cpp +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2023 LiveKit - * - * Licensed under the Apache License, Version 2.0 (the “License”); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an “AS IS” BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include "livekit/logsink.h" - -namespace livekit { - -LogSink::LogSink(rust::Fn fnc) : fnc_(fnc) { - rtc::LogMessage::AddLogToStream(this, rtc::LoggingSeverity::LS_VERBOSE); -} - -LogSink::~LogSink() { - rtc::LogMessage::RemoveLogToStream(this); -} - -void LogSink::OnLogMessage(const std::string& message, rtc::LoggingSeverity severity) { - fnc_(rust::String(message), static_cast(severity)); -} - -std::unique_ptr new_log_sink(rust::Fn fnc) { - return std::make_unique(fnc); -} - -} - diff --git a/webrtc-sys/src/logsink.rs b/webrtc-sys/src/logsink.rs deleted file mode 100644 index 6d63f97..0000000 --- a/webrtc-sys/src/logsink.rs +++ /dev/null @@ -1,24 +0,0 @@ -use crate::impl_thread_safety; - -#[cxx::bridge(namespace = "livekit")] -pub mod ffi { - - #[derive(Debug)] - #[repr(i32)] - pub enum LoggingSeverity { - Verbose, - Info, - Warning, - Error, - None, - } - - unsafe extern "C++" { - include!("livekit/logsink.h"); - - type LogSink; - fn new_log_sink(fnc: fn(String, LoggingSeverity)) -> UniquePtr; - } -} - -impl_thread_safety!(ffi::LogSink, Send + Sync); diff --git a/webrtc-sys/src/media_stream.cpp b/webrtc-sys/src/media_stream.cpp index cbd8c96..ed6f78e 100644 --- a/webrtc-sys/src/media_stream.cpp +++ b/webrtc-sys/src/media_stream.cpp @@ -32,8 +32,9 @@ namespace livekit { MediaStream::MediaStream( + std::shared_ptr rtc_runtime, rtc::scoped_refptr stream) - : media_stream_(std::move(stream)) {} + : rtc_runtime_(rtc_runtime), media_stream_(std::move(stream)) {} rust::String MediaStream::id() const { return media_stream_->id(); @@ -42,7 +43,8 @@ rust::String MediaStream::id() const { rust::Vec MediaStream::get_video_tracks() const { rust::Vec rust; for (auto video : media_stream_->GetVideoTracks()) - rust.push_back(VideoTrackPtr{std::make_shared(video)}); + rust.push_back( + VideoTrackPtr{rtc_runtime_->get_or_create_video_track(video)}); return rust; } @@ -50,20 +52,21 @@ rust::Vec MediaStream::get_video_tracks() const { rust::Vec MediaStream::get_audio_tracks() const { rust::Vec rust; for (auto audio : media_stream_->GetAudioTracks()) - rust.push_back(AudioTrackPtr{std::make_shared(audio)}); + rust.push_back( + AudioTrackPtr{rtc_runtime_->get_or_create_audio_track(audio)}); return rust; } std::shared_ptr MediaStream::find_audio_track( rust::String track_id) const { - return std::make_shared( + return rtc_runtime_->get_or_create_audio_track( media_stream_->FindAudioTrack(track_id.c_str())); } std::shared_ptr MediaStream::find_video_track( rust::String track_id) const { - return std::make_shared( + return rtc_runtime_->get_or_create_video_track( media_stream_->FindVideoTrack(track_id.c_str())); } @@ -71,11 +74,13 @@ bool MediaStream::add_track(std::shared_ptr track) const { if (track->kind() == webrtc::MediaStreamTrackInterface::kVideoKind) { return media_stream_->AddTrack( rtc::scoped_refptr( - static_cast(track->get().get()))); + static_cast( + track->rtc_track().get()))); } else { return media_stream_->AddTrack( rtc::scoped_refptr( - static_cast(track->get().get()))); + static_cast( + track->rtc_track().get()))); } } @@ -83,278 +88,14 @@ bool MediaStream::remove_track(std::shared_ptr track) const { if (track->kind() == webrtc::MediaStreamTrackInterface::kVideoKind) { return media_stream_->RemoveTrack( rtc::scoped_refptr( - static_cast(track->get().get()))); + static_cast( + track->rtc_track().get()))); } else { return media_stream_->RemoveTrack( rtc::scoped_refptr( - static_cast(track->get().get()))); + static_cast( + track->rtc_track().get()))); } } -MediaStreamTrack::MediaStreamTrack( - rtc::scoped_refptr track) - : track_(std::move(track)) {} - -std::shared_ptr MediaStreamTrack::from( - rtc::scoped_refptr track) { - if (track->kind() == webrtc::MediaStreamTrackInterface::kVideoKind) { - return std::make_shared( - rtc::scoped_refptr( - static_cast(track.get()))); - } else { - return std::make_shared( - rtc::scoped_refptr( - static_cast(track.get()))); - } -} - -rust::String MediaStreamTrack::kind() const { - return track_->kind(); -} - -rust::String MediaStreamTrack::id() const { - return track_->id(); -} - -bool MediaStreamTrack::enabled() const { - return track_->enabled(); -} - -bool MediaStreamTrack::set_enabled(bool enable) const { - return track_->set_enabled(enable); -} - -TrackState MediaStreamTrack::state() const { - return static_cast(track_->state()); -} - -AudioTrack::AudioTrack(rtc::scoped_refptr track) - : MediaStreamTrack(std::move(track)) {} - -void AudioTrack::add_sink(NativeAudioSink& sink) const { - track()->AddSink(&sink); -} - -void AudioTrack::remove_sink(NativeAudioSink& sink) const { - track()->RemoveSink(&sink); -} - -NativeAudioSink::NativeAudioSink(rust::Box observer) - : observer_(std::move(observer)) {} - -void NativeAudioSink::OnData(const void* audio_data, - int bits_per_sample, - int sample_rate, - size_t number_of_channels, - size_t number_of_frames) { - RTC_CHECK_EQ(16, bits_per_sample); - - observer_->on_data(static_cast(audio_data), sample_rate, - number_of_channels, number_of_frames); -} - -std::unique_ptr new_native_audio_sink( - rust::Box observer) { - return std::make_unique(std::move(observer)); -} - -NativeAudioTrackSource::NativeAudioTrackSource() { - options_.echo_cancellation = false; - options_.auto_gain_control = false; - options_.noise_suppression = false; -} - -webrtc::MediaSourceInterface::SourceState NativeAudioTrackSource::state() - const { - return webrtc::MediaSourceInterface::SourceState::kLive; -} - -bool NativeAudioTrackSource::remote() const { - return false; -} - -const cricket::AudioOptions NativeAudioTrackSource::options() const { - return options_; -} - -void NativeAudioTrackSource::AddSink(webrtc::AudioTrackSinkInterface* sink) { - webrtc::MutexLock lock(&mutex_); - sinks_.push_back(sink); -} - -void NativeAudioTrackSource::RemoveSink(webrtc::AudioTrackSinkInterface* sink) { - webrtc::MutexLock lock(&mutex_); - sinks_.erase(std::remove(sinks_.begin(), sinks_.end(), sink), sinks_.end()); -} - -void NativeAudioTrackSource::on_captured_frame(const int16_t* data, - int sample_rate, - size_t number_of_channels, - size_t number_of_frames) { - webrtc::MutexLock lock(&mutex_); - for (auto sink : sinks_) { - sink->OnData(data, 16, sample_rate, number_of_channels, number_of_frames); - } -} - -AudioTrackSource::AudioTrackSource( - rtc::scoped_refptr source) - : source_(std::move(source)) {} - -void AudioTrackSource::on_captured_frame(const int16_t* audio_data, - int sample_rate, - size_t number_of_channels, - size_t number_of_frames) const { - source_->on_captured_frame(audio_data, sample_rate, number_of_channels, - number_of_frames); -} - -rtc::scoped_refptr AudioTrackSource::get() const { - return source_; -} - -std::shared_ptr new_audio_track_source() { - return std::make_shared( - rtc::make_ref_counted()); -} - -VideoTrack::VideoTrack(rtc::scoped_refptr track) - : MediaStreamTrack(std::move(track)) {} - -void VideoTrack::add_sink(NativeVideoFrameSink& sink) const { - track()->AddOrUpdateSink(&sink, rtc::VideoSinkWants()); -} - -void VideoTrack::remove_sink(NativeVideoFrameSink& sink) const { - track()->RemoveSink(&sink); -} - -void VideoTrack::set_should_receive(bool should_receive) const { - track()->set_should_receive(should_receive); -} - -bool VideoTrack::should_receive() const { - return track()->should_receive(); -} - -ContentHint VideoTrack::content_hint() const { - return static_cast(track()->content_hint()); -} - -void VideoTrack::set_content_hint(ContentHint hint) const { - track()->set_content_hint( - static_cast(hint)); -} - -NativeVideoFrameSink::NativeVideoFrameSink( - rust::Box observer) - : observer_(std::move(observer)) {} - -void NativeVideoFrameSink::OnFrame(const webrtc::VideoFrame& frame) { - observer_->on_frame(std::make_unique(frame)); -} - -void NativeVideoFrameSink::OnDiscardedFrame() { - observer_->on_discarded_frame(); -} - -void NativeVideoFrameSink::OnConstraintsChanged( - const webrtc::VideoTrackSourceConstraints& constraints) { - VideoTrackSourceConstraints cst; - cst.min_fps = constraints.min_fps.value_or(-1); - cst.max_fps = constraints.max_fps.value_or(-1); - observer_->on_constraints_changed(cst); -} - -std::unique_ptr new_native_video_frame_sink( - rust::Box observer) { - return std::make_unique(std::move(observer)); -} - -NativeVideoTrackSource::NativeVideoTrackSource() - : rtc::AdaptedVideoTrackSource(4) {} - -NativeVideoTrackSource::~NativeVideoTrackSource() {} - -bool NativeVideoTrackSource::is_screencast() const { - return false; -} - -absl::optional NativeVideoTrackSource::needs_denoising() const { - return false; -} - -webrtc::MediaSourceInterface::SourceState NativeVideoTrackSource::state() - const { - return SourceState::kLive; -} - -bool NativeVideoTrackSource::remote() const { - return false; -} - -bool NativeVideoTrackSource::on_captured_frame( - const webrtc::VideoFrame& frame) { - webrtc::MutexLock lock(&mutex_); - - int64_t aligned_timestamp_us = timestamp_aligner_.TranslateTimestamp( - frame.timestamp_us(), rtc::TimeMicros()); - - rtc::scoped_refptr buffer = - frame.video_frame_buffer(); - - int adapted_width, adapted_height, crop_width, crop_height, crop_x, crop_y; - if (!AdaptFrame(buffer->width(), buffer->height(), aligned_timestamp_us, - &adapted_width, &adapted_height, &crop_width, &crop_height, - &crop_x, &crop_y)) { - return false; - } - - if (adapted_width != frame.width() || adapted_height != frame.height()) { - buffer = buffer->CropAndScale(crop_x, crop_y, crop_width, crop_height, - adapted_width, adapted_height); - } - - webrtc::VideoRotation rotation = frame.rotation(); - if (apply_rotation() && rotation != webrtc::kVideoRotation_0) { - // If the buffer is I420, rtc::AdaptedVideoTrackSource will handle the - // rotation for us. - buffer = buffer->ToI420(); - } - - OnFrame(webrtc::VideoFrame::Builder() - .set_video_frame_buffer(buffer) - .set_rotation(rotation) - .set_timestamp_us(aligned_timestamp_us) - .build()); - - return true; -} - -AdaptedVideoTrackSource::AdaptedVideoTrackSource( - rtc::scoped_refptr source) - : source_(source) {} - -bool AdaptedVideoTrackSource::on_captured_frame( - const std::unique_ptr& frame) const { - auto rtc_frame = frame->get(); - rtc_frame.set_timestamp_us(rtc::TimeMicros()); - - // auto buffer = webrtc::I420Buffer::Create(1280, 720); - // webrtc::I420Buffer::SetBlack(buffer.get()); - // rtc_frame.set_video_frame_buffer(buffer); - - return source_->on_captured_frame(rtc_frame); -} - -rtc::scoped_refptr AdaptedVideoTrackSource::get() - const { - return source_; -} - -std::shared_ptr new_adapted_video_track_source() { - return std::make_shared( - rtc::make_ref_counted()); -} - } // namespace livekit diff --git a/webrtc-sys/src/media_stream.rs b/webrtc-sys/src/media_stream.rs index 77fcabf..da1c515 100644 --- a/webrtc-sys/src/media_stream.rs +++ b/webrtc-sys/src/media_stream.rs @@ -1,38 +1,16 @@ use crate::impl_thread_safety; -use crate::video_frame::ffi::VideoFrame; -use cxx::UniquePtr; #[cxx::bridge(namespace = "livekit")] pub mod ffi { - - #[derive(Debug)] - #[repr(i32)] - pub enum TrackState { - Live, - Ended, - } - - #[derive(Debug)] - #[repr(i32)] - pub enum ContentHint { - None, - Fluid, - Detailed, - Text, - } - - // -1 = optional - #[derive(Debug)] - pub struct VideoTrackSourceConstraints { - pub min_fps: f64, - pub max_fps: f64, - } - extern "C++" { - include!("livekit/video_frame.h"); include!("livekit/helper.h"); + include!("livekit/media_stream_track.h"); + include!("livekit/audio_track.h"); + include!("livekit/video_track.h"); - type VideoFrame = crate::video_frame::ffi::VideoFrame; + type MediaStreamTrack = crate::media_stream_track::ffi::MediaStreamTrack; + type AudioTrack = crate::audio_track::ffi::AudioTrack; + type VideoTrack = crate::video_track::ffi::VideoTrack; type VideoTrackPtr = crate::helper::ffi::VideoTrackPtr; type AudioTrackPtr = crate::helper::ffi::AudioTrackPtr; } @@ -40,14 +18,7 @@ pub mod ffi { unsafe extern "C++" { include!("livekit/media_stream.h"); - type NativeAudioSink; - type NativeVideoFrameSink; - type MediaStreamTrack; type MediaStream; - type AudioTrack; - type VideoTrack; - type AudioTrackSource; - type AdaptedVideoTrackSource; fn id(self: &MediaStream) -> String; fn get_audio_tracks(self: &MediaStream) -> Vec; @@ -57,140 +28,8 @@ pub mod ffi { fn add_track(self: &MediaStream, audio_track: SharedPtr) -> bool; fn remove_track(self: &MediaStream, audio_track: SharedPtr) -> bool; - fn kind(self: &MediaStreamTrack) -> String; - fn id(self: &MediaStreamTrack) -> String; - fn enabled(self: &MediaStreamTrack) -> bool; - fn set_enabled(self: &MediaStreamTrack, enable: bool) -> bool; - fn state(self: &MediaStreamTrack) -> TrackState; - - unsafe fn add_sink(self: &AudioTrack, sink: Pin<&mut NativeAudioSink>); - unsafe fn remove_sink(self: &AudioTrack, sink: Pin<&mut NativeAudioSink>); - - fn new_native_audio_sink(observer: Box) -> UniquePtr; - - unsafe fn on_captured_frame( - self: &AudioTrackSource, - data: *const i16, - sample_rate: i32, - nb_channels: usize, - nb_frames: usize, - ); - - fn new_audio_track_source() -> SharedPtr; - - unsafe fn add_sink(self: &VideoTrack, sink: Pin<&mut NativeVideoFrameSink>); - unsafe fn remove_sink(self: &VideoTrack, sink: Pin<&mut NativeVideoFrameSink>); - - fn set_should_receive(self: &VideoTrack, should_receive: bool); - fn should_receive(self: &VideoTrack) -> bool; - fn content_hint(self: &VideoTrack) -> ContentHint; - fn set_content_hint(self: &VideoTrack, hint: ContentHint); - - fn new_native_video_frame_sink( - observer: Box, - ) -> UniquePtr; - - fn on_captured_frame(self: &AdaptedVideoTrackSource, frame: &UniquePtr) - -> bool; - - fn new_adapted_video_track_source() -> SharedPtr; - - fn video_to_media(track: SharedPtr) -> SharedPtr; - fn audio_to_media(track: SharedPtr) -> SharedPtr; - fn media_to_video(track: SharedPtr) -> SharedPtr; - fn media_to_audio(track: SharedPtr) -> SharedPtr; - - fn _shared_media_stream_track() -> SharedPtr; - fn _shared_audio_track() -> SharedPtr; - fn _shared_video_track() -> SharedPtr; fn _shared_media_stream() -> SharedPtr; } - - extern "Rust" { - type AudioSinkWrapper; - type VideoFrameSinkWrapper; - - unsafe fn on_data( - self: &AudioSinkWrapper, - data: *const i16, - sample_rate: i32, - nb_channels: usize, - nb_frames: usize, - ); - - fn on_frame(self: &VideoFrameSinkWrapper, frame: UniquePtr); - fn on_discarded_frame(self: &VideoFrameSinkWrapper); - fn on_constraints_changed( - self: &VideoFrameSinkWrapper, - constraints: VideoTrackSourceConstraints, - ); - } } -impl_thread_safety!(ffi::MediaStreamTrack, Send + Sync); impl_thread_safety!(ffi::MediaStream, Send + Sync); -impl_thread_safety!(ffi::AudioTrack, Send + Sync); -impl_thread_safety!(ffi::VideoTrack, Send + Sync); -impl_thread_safety!(ffi::NativeVideoFrameSink, Send + Sync); -impl_thread_safety!(ffi::NativeAudioSink, Send + Sync); -impl_thread_safety!(ffi::AudioTrackSource, Send + Sync); -impl_thread_safety!(ffi::AdaptedVideoTrackSource, Send + Sync); - -pub trait AudioSink: Send { - fn on_data(&self, data: &[i16], sample_rate: i32, nb_channels: usize, nb_frames: usize); -} - -pub struct AudioSinkWrapper { - observer: *mut dyn AudioSink, -} - -impl AudioSinkWrapper { - /// # Safety - /// AudioSink must lives as long as AudioSinkWrapper does - pub unsafe fn new(observer: *mut dyn AudioSink) -> Self { - Self { observer } - } - - fn on_data(&self, data: *const i16, sample_rate: i32, nb_channels: usize, nb_frames: usize) { - unsafe { - let data = std::slice::from_raw_parts(data, nb_channels * nb_frames); - (*self.observer).on_data(data, sample_rate, nb_channels, nb_frames); - } - } -} - -pub trait VideoFrameSink: Send { - fn on_frame(&self, frame: UniquePtr); - fn on_discarded_frame(&self); - fn on_constraints_changed(&self, constraints: ffi::VideoTrackSourceConstraints); -} - -pub struct VideoFrameSinkWrapper { - observer: *mut dyn VideoFrameSink, -} - -impl VideoFrameSinkWrapper { - /// # Safety - /// VideoFrameSink must lives as long as VideoSinkInterfaceWrapper does - pub unsafe fn new(observer: *mut dyn VideoFrameSink) -> Self { - Self { observer } - } - - fn on_frame(&self, frame: UniquePtr) { - unsafe { - (*self.observer).on_frame(frame); - } - } - - fn on_discarded_frame(&self) { - unsafe { - (*self.observer).on_discarded_frame(); - } - } - - fn on_constraints_changed(&self, constraints: ffi::VideoTrackSourceConstraints) { - unsafe { - (*self.observer).on_constraints_changed(constraints); - } - } -} diff --git a/webrtc-sys/src/media_stream_track.cpp b/webrtc-sys/src/media_stream_track.cpp new file mode 100644 index 0000000..dafefca --- /dev/null +++ b/webrtc-sys/src/media_stream_track.cpp @@ -0,0 +1,58 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the “License”); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an “AS IS” BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include "api/media_stream_interface.h" +#include "api/video/video_frame.h" +#include "api/video/video_rotation.h" +#include "audio/remix_resample.h" +#include "common_audio/include/audio_util.h" +#include "livekit/media_stream.h" +#include "rtc_base/logging.h" +#include "rtc_base/ref_counted_object.h" +#include "rtc_base/time_utils.h" + +namespace livekit { + +MediaStreamTrack::MediaStreamTrack( + std::shared_ptr rtc_runtime, + rtc::scoped_refptr track) + : rtc_runtime_(rtc_runtime), track_(std::move(track)) {} + +rust::String MediaStreamTrack::kind() const { + return track_->kind(); +} + +rust::String MediaStreamTrack::id() const { + return track_->id(); +} + +bool MediaStreamTrack::enabled() const { + return track_->enabled(); +} + +bool MediaStreamTrack::set_enabled(bool enable) const { + return track_->set_enabled(enable); +} + +TrackState MediaStreamTrack::state() const { + return static_cast(track_->state()); +} + +} // namespace livekit diff --git a/webrtc-sys/src/media_stream_track.rs b/webrtc-sys/src/media_stream_track.rs new file mode 100644 index 0000000..d7b8446 --- /dev/null +++ b/webrtc-sys/src/media_stream_track.rs @@ -0,0 +1,26 @@ +use crate::impl_thread_safety; + +#[cxx::bridge(namespace = "livekit")] +pub mod ffi { + #[repr(i32)] + pub enum TrackState { + Live, + Ended, + } + + unsafe extern "C++" { + include!("livekit/media_stream_track.h"); + + type MediaStreamTrack; + + fn kind(self: &MediaStreamTrack) -> String; + fn id(self: &MediaStreamTrack) -> String; + fn enabled(self: &MediaStreamTrack) -> bool; + fn set_enabled(self: &MediaStreamTrack, enable: bool) -> bool; + fn state(self: &MediaStreamTrack) -> TrackState; + + fn _shared_media_stream_track() -> SharedPtr; + } +} + +impl_thread_safety!(ffi::MediaStreamTrack, Send + Sync); diff --git a/webrtc-sys/src/peer_connection.cpp b/webrtc-sys/src/peer_connection.cpp index 44ffd32..cb7c8ce 100644 --- a/webrtc-sys/src/peer_connection.cpp +++ b/webrtc-sys/src/peer_connection.cpp @@ -16,14 +16,22 @@ #include "livekit/peer_connection.h" +#include + +#include "api/data_channel_interface.h" +#include "api/scoped_refptr.h" +#include "livekit/data_channel.h" +#include "livekit/jsep.h" #include "livekit/media_stream.h" #include "livekit/rtc_error.h" #include "livekit/rtp_transceiver.h" +#include "webrtc-sys/src/peer_connection.rs.h" +#include "webrtc-sys/src/rtc_error.rs.h" namespace livekit { inline webrtc::PeerConnectionInterface::RTCOfferAnswerOptions -toNativeOfferAnswerOptions(const RTCOfferAnswerOptions& options) { +to_native_offer_answer_options(const RtcOfferAnswerOptions& options) { webrtc::PeerConnectionInterface::RTCOfferAnswerOptions rtc_options; rtc_options.offer_to_receive_video = options.offer_to_receive_video; rtc_options.offer_to_receive_audio = options.offer_to_receive_audio; @@ -37,44 +45,79 @@ toNativeOfferAnswerOptions(const RTCOfferAnswerOptions& options) { } PeerConnection::PeerConnection( - std::shared_ptr rtc_runtime, + std::shared_ptr rtc_runtime, + std::unique_ptr observer, rtc::scoped_refptr peer_connection) - : rtc_runtime_(std::move(rtc_runtime)), + : rtc_runtime_(rtc_runtime), + observer_(std::move(observer)), peer_connection_(std::move(peer_connection)) {} void PeerConnection::create_offer( - NativeCreateSdpObserverHandle& observer_handle, - RTCOfferAnswerOptions options) const { - peer_connection_->CreateOffer(observer_handle.observer.get(), - toNativeOfferAnswerOptions(options)); + RtcOfferAnswerOptions options, + rust::Box ctx, + rust::Fn, std::unique_ptr)> + on_success, + rust::Fn, RtcError)> on_error) const { + rtc::scoped_refptr observer = + rtc::make_ref_counted(std::move(ctx), on_success, + on_error); + + peer_connection_->CreateOffer(observer.get(), + to_native_offer_answer_options(options)); } void PeerConnection::create_answer( - NativeCreateSdpObserverHandle& observer_handle, - RTCOfferAnswerOptions options) const { - peer_connection_->CreateAnswer(observer_handle.observer.get(), - toNativeOfferAnswerOptions(options)); + RtcOfferAnswerOptions options, + rust::Box ctx, + rust::Fn, std::unique_ptr)> + on_success, + rust::Fn, RtcError)> on_error) const { + rtc::scoped_refptr observer = + rtc::make_ref_counted(std::move(ctx), on_success, + on_error); + + peer_connection_->CreateAnswer(observer.get(), + to_native_offer_answer_options(options)); } void PeerConnection::set_local_description( std::unique_ptr desc, - NativeSetLocalSdpObserverHandle& observer) const { - peer_connection_->SetLocalDescription(desc->clone()->release(), - observer.observer); + rust::Box ctx, + rust::Fn, RtcError)> on_complete) const { + rtc::scoped_refptr observer = + rtc::make_ref_counted(std::move(ctx), + on_complete); + + peer_connection_->SetLocalDescription(desc->clone()->release(), observer); } void PeerConnection::set_remote_description( std::unique_ptr desc, - NativeSetRemoteSdpObserverHandle& observer) const { - peer_connection_->SetRemoteDescription(desc->clone()->release(), - observer.observer); + rust::Box ctx, + rust::Fn, RtcError)> on_complete) const { + rtc::scoped_refptr observer = + rtc::make_ref_counted(std::move(ctx), + on_complete); + + peer_connection_->SetRemoteDescription(desc->clone()->release(), observer); +} + +void PeerConnection::add_ice_candidate( + std::shared_ptr candidate, + rust::Box ctx, + rust::Fn, RtcError)> on_complete) const { + peer_connection_->AddIceCandidate( + candidate->release(), [&](const webrtc::RTCError& err) { + on_complete(std::move(ctx), to_error(err)); + }); } std::shared_ptr PeerConnection::create_data_channel( rust::String label, - std::unique_ptr init) const { + DataChannelInit init) const { + webrtc::DataChannelInit rtc_init = to_native_data_channel_init(init); auto result = - peer_connection_->CreateDataChannelOrError(label.c_str(), init.get()); + peer_connection_->CreateDataChannelOrError(label.c_str(), &rtc_init); if (!result.ok()) { throw std::runtime_error(serialize_error(to_error(result.error()))); @@ -87,16 +130,16 @@ std::shared_ptr PeerConnection::add_track( std::shared_ptr track, const rust::Vec& stream_ids) const { std::vector std_stream_ids(stream_ids.begin(), stream_ids.end()); - auto result = peer_connection_->AddTrack(track->get(), std_stream_ids); + auto result = peer_connection_->AddTrack(track->rtc_track(), std_stream_ids); if (!result.ok()) { throw std::runtime_error(serialize_error(to_error(result.error()))); } - return std::make_shared(result.value()); + return std::make_shared(rtc_runtime_, result.value()); } void PeerConnection::remove_track(std::shared_ptr sender) const { - auto error = peer_connection_->RemoveTrackOrError(sender->get()); + auto error = peer_connection_->RemoveTrackOrError(sender->rtc_sender()); if (!error.ok()) throw std::runtime_error(serialize_error(to_error(error))); } @@ -105,11 +148,11 @@ std::shared_ptr PeerConnection::add_transceiver( std::shared_ptr track, RtpTransceiverInit init) const { auto result = peer_connection_->AddTransceiver( - track->get(), to_native_rtp_transceiver_init(init)); + track->rtc_track(), to_native_rtp_transceiver_init(init)); if (!result.ok()) throw std::runtime_error(serialize_error(to_error(result.error()))); - return std::make_shared(result.value()); + return std::make_shared(rtc_runtime_, result.value()); } std::shared_ptr PeerConnection::add_transceiver_for_media( @@ -122,13 +165,14 @@ std::shared_ptr PeerConnection::add_transceiver_for_media( if (!result.ok()) throw std::runtime_error(serialize_error(to_error(result.error()))); - return std::make_shared(result.value()); + return std::make_shared(rtc_runtime_, result.value()); } rust::Vec PeerConnection::get_senders() const { rust::Vec vec; for (auto sender : peer_connection_->GetSenders()) - vec.push_back(RtpSenderPtr{std::make_shared(sender)}); + vec.push_back( + RtpSenderPtr{std::make_shared(rtc_runtime_, sender)}); return vec; } @@ -136,7 +180,8 @@ rust::Vec PeerConnection::get_senders() const { rust::Vec PeerConnection::get_receivers() const { rust::Vec vec; for (auto receiver : peer_connection_->GetReceivers()) - vec.push_back(RtpReceiverPtr{std::make_shared(receiver)}); + vec.push_back( + RtpReceiverPtr{std::make_shared(rtc_runtime_, receiver)}); return vec; } @@ -144,20 +189,12 @@ rust::Vec PeerConnection::get_receivers() const { rust::Vec PeerConnection::get_transceivers() const { rust::Vec vec; for (auto transceiver : peer_connection_->GetTransceivers()) - vec.push_back( - RtpTransceiverPtr{std::make_shared(transceiver)}); + vec.push_back(RtpTransceiverPtr{ + std::make_shared(rtc_runtime_, transceiver)}); return vec; } -void PeerConnection::add_ice_candidate( - std::shared_ptr candidate, - NativeAddIceCandidateObserver& observer) const { - peer_connection_->AddIceCandidate( - candidate->release(), - [&](const webrtc::RTCError& err) { observer.OnComplete(to_error(err)); }); -} - std::unique_ptr PeerConnection::current_local_description() const { auto local_description = peer_connection_->current_local_description(); @@ -233,28 +270,11 @@ void PeerConnection::close() const { peer_connection_->Close(); } -// AddIceCandidateObserver - -NativeAddIceCandidateObserver::NativeAddIceCandidateObserver( - rust::Box observer) - : observer_(std::move(observer)) {} - -void NativeAddIceCandidateObserver::OnComplete(const RTCError& error) { - observer_->on_complete(error); -} - -std::unique_ptr -create_native_add_ice_candidate_observer( - rust::Box observer) { - return std::make_unique(std::move(observer)); -} - // PeerConnectionObserver NativePeerConnectionObserver::NativePeerConnectionObserver( - std::shared_ptr rtc_runtime, rust::Box observer) - : rtc_runtime_(std::move(rtc_runtime)), observer_(std::move(observer)) { + : observer_(std::move(observer)) { RTC_LOG(LS_INFO) << "NativePeerConnectionObserver()"; } @@ -269,12 +289,14 @@ void NativePeerConnectionObserver::OnSignalingChange( void NativePeerConnectionObserver::OnAddStream( rtc::scoped_refptr stream) { - observer_->on_add_stream(std::make_unique(stream)); + observer_->on_add_stream(std::make_unique(rtc_runtime_, stream)); } void NativePeerConnectionObserver::OnRemoveStream( rtc::scoped_refptr stream) { - observer_->on_remove_stream(std::make_unique(stream)); + // Find current MediaStream + // observer_->on_remove_stream(std::make_unique(rtc_runtime_, + // stream)); } void NativePeerConnectionObserver::OnDataChannel( @@ -349,7 +371,7 @@ void NativePeerConnectionObserver::OnIceConnectionReceivingChange( void NativePeerConnectionObserver::OnIceSelectedCandidatePairChanged( const cricket::CandidatePairChangeEvent& event) { - CandidatePairChangeEvent e; + CandidatePairChangeEvent e{}; e.selected_candidate_pair.local = std::make_unique(event.selected_candidate_pair.local); e.selected_candidate_pair.remote = @@ -368,32 +390,34 @@ void NativePeerConnectionObserver::OnAddTrack( rust::Vec vec; for (const auto& item : streams) { - vec.push_back(MediaStreamPtr{std::make_unique(item)}); + vec.push_back( + MediaStreamPtr{std::make_unique(rtc_runtime_, item)}); } - observer_->on_add_track(std::make_unique(receiver), + observer_->on_add_track(std::make_unique(rtc_runtime_, receiver), std::move(vec)); } void NativePeerConnectionObserver::OnTrack( rtc::scoped_refptr transceiver) { - observer_->on_track(std::make_unique(transceiver)); + observer_->on_track( + std::make_unique(rtc_runtime_, transceiver)); } void NativePeerConnectionObserver::OnRemoveTrack( rtc::scoped_refptr receiver) { - observer_->on_remove_track(std::make_unique(receiver)); + observer_->on_remove_track( + std::make_unique(rtc_runtime_, receiver)); } void NativePeerConnectionObserver::OnInterestingUsage(int usage_pattern) { observer_->on_interesting_usage(usage_pattern); } -std::shared_ptr +std::unique_ptr create_native_peer_connection_observer( - std::shared_ptr rtc_runtime, rust::Box observer) { - return std::make_shared(rtc_runtime, - std::move(observer)); + return std::make_unique(std::move(observer)); } + } // namespace livekit diff --git a/webrtc-sys/src/peer_connection.rs b/webrtc-sys/src/peer_connection.rs index 0401201..2dfd0ea 100644 --- a/webrtc-sys/src/peer_connection.rs +++ b/webrtc-sys/src/peer_connection.rs @@ -3,28 +3,26 @@ use crate::data_channel::ffi::DataChannel; use crate::impl_thread_safety; use crate::jsep::ffi::IceCandidate; use crate::media_stream::ffi::MediaStream; -use crate::rtc_error::ffi::RTCError; use crate::rtp_receiver::ffi::RtpReceiver; use crate::rtp_transceiver::ffi::RtpTransceiver; use cxx::SharedPtr; -use std::mem::ManuallyDrop; +use std::any::Any; use std::sync::Arc; #[cxx::bridge(namespace = "livekit")] pub mod ffi { - struct CandidatePair { + pub struct CandidatePair { local: SharedPtr, remote: SharedPtr, } - struct CandidatePairChangeEvent { + pub struct CandidatePairChangeEvent { selected_candidate_pair: CandidatePair, last_data_received_ms: i64, reason: String, estimated_disconnected_time_ms: i64, } - #[derive(Debug)] #[repr(i32)] pub enum PeerConnectionState { New, @@ -35,7 +33,6 @@ pub mod ffi { Closed, } - #[derive(Debug)] #[repr(i32)] pub enum SignalingState { Stable, @@ -46,7 +43,6 @@ pub mod ffi { Closed, } - #[derive(Debug)] #[repr(i32)] pub enum IceConnectionState { IceConnectionNew, @@ -59,7 +55,6 @@ pub mod ffi { IceConnectionMax, } - #[derive(Debug)] #[repr(i32)] pub enum IceGatheringState { IceGatheringNew, @@ -67,8 +62,7 @@ pub mod ffi { IceGatheringComplete, } - #[derive(Debug)] - pub struct RTCOfferAnswerOptions { + pub struct RtcOfferAnswerOptions { offer_to_receive_video: i32, offer_to_receive_audio: i32, voice_activity_detection: bool, @@ -96,133 +90,104 @@ pub mod ffi { type RtpSenderPtr = crate::helper::ffi::RtpSenderPtr; type RtpReceiverPtr = crate::helper::ffi::RtpReceiverPtr; type RtpTransceiverPtr = crate::helper::ffi::RtpTransceiverPtr; - type RTCError = crate::rtc_error::ffi::RTCError; + type RtcError = crate::rtc_error::ffi::RtcError; type Candidate = crate::candidate::ffi::Candidate; type IceCandidate = crate::jsep::ffi::IceCandidate; type DataChannel = crate::data_channel::ffi::DataChannel; + type DataChannelInit = crate::data_channel::ffi::DataChannelInit; type RtpSender = crate::rtp_sender::ffi::RtpSender; type RtpReceiver = crate::rtp_receiver::ffi::RtpReceiver; type RtpTransceiver = crate::rtp_transceiver::ffi::RtpTransceiver; type RtpTransceiverInit = crate::rtp_transceiver::ffi::RtpTransceiverInit; type MediaStream = crate::media_stream::ffi::MediaStream; type MediaStreamTrack = crate::media_stream::ffi::MediaStreamTrack; - type NativeCreateSdpObserverHandle = crate::jsep::ffi::NativeCreateSdpObserverHandle; - type NativeSetLocalSdpObserverHandle = crate::jsep::ffi::NativeSetLocalSdpObserverHandle; - type NativeSetRemoteSdpObserverHandle = crate::jsep::ffi::NativeSetRemoteSdpObserverHandle; - type NativeDataChannelInit = crate::data_channel::ffi::NativeDataChannelInit; type SessionDescription = crate::jsep::ffi::SessionDescription; type MediaType = crate::webrtc::ffi::MediaType; - type RTCRuntime = crate::webrtc::ffi::RTCRuntime; } unsafe extern "C++" { include!("livekit/peer_connection.h"); - type NativeAddIceCandidateObserver; - type NativePeerConnectionObserver; type PeerConnection; - /// SAFETY - /// The observer must live as long as the operation ends - unsafe fn create_offer( - self: &PeerConnection, - observer: Pin<&mut NativeCreateSdpObserverHandle>, - options: RTCOfferAnswerOptions, - ); + // The reason we still expose NativePeerConnectionObserver is because cxx doeesn't support Rust type alias + // So we can't share NativePeerConnectionWrapper in peer_connection_factory.rs + // (It is technically possible to get the Opaque C++ Type, but in this case, we can't use Box) + // We can delete create_native_peer_connection_observer once cxx supports Rust type alias + type NativePeerConnectionObserver; + fn create_native_peer_connection_observer( + observer: Box, + ) -> UniquePtr; - /// SAFETY - /// The observer must live as long as the operation ends - unsafe fn create_answer( + fn create_offer( self: &PeerConnection, - observer: Pin<&mut NativeCreateSdpObserverHandle>, - options: RTCOfferAnswerOptions, + options: RtcOfferAnswerOptions, + ctx: Box, + on_success: fn(ctx: Box, sdp: UniquePtr), + on_error: fn(ctx: Box, error: RtcError), ); - - /// SAFETY - /// The observer must live as long as the operation ends - unsafe fn set_local_description( + fn create_answer( + self: &PeerConnection, + options: RtcOfferAnswerOptions, + ctx: Box, + on_success: fn(ctx: Box, sdp: UniquePtr), + on_error: fn(ctx: Box, error: RtcError), + ); + fn set_local_description( self: &PeerConnection, desc: UniquePtr, - observer: Pin<&mut NativeSetLocalSdpObserverHandle>, + ctx: Box, + on_complete: fn(ctx: Box, error: RtcError), ); - - /// SAFETY - /// The observer must live as long as the operation ends - unsafe fn set_remote_description( + fn set_remote_description( self: &PeerConnection, desc: UniquePtr, - observer: Pin<&mut NativeSetRemoteSdpObserverHandle>, + ctx: Box, + on_complete: fn(ctx: Box, error: RtcError), ); - fn add_track( self: &PeerConnection, track: SharedPtr, stream_ids: &Vec, ) -> Result>; - fn remove_track(self: &PeerConnection, sender: SharedPtr) -> Result<()>; - fn add_transceiver( self: &PeerConnection, track: SharedPtr, init: RtpTransceiverInit, ) -> Result>; - fn add_transceiver_for_media( self: &PeerConnection, media_type: MediaType, init: RtpTransceiverInit, ) -> Result>; - fn get_senders(self: &PeerConnection) -> Vec; - fn get_receivers(self: &PeerConnection) -> Vec; - fn get_transceivers(self: &PeerConnection) -> Vec; - fn create_data_channel( self: &PeerConnection, label: String, - init: UniquePtr, + init: DataChannelInit, ) -> Result>; - fn add_ice_candidate( self: &PeerConnection, candidate: SharedPtr, - observer: Pin<&mut NativeAddIceCandidateObserver>, + ctx: Box, + on_complete: fn(ctx: Box, error: RtcError), ); - fn current_local_description(self: &PeerConnection) -> UniquePtr; - fn current_remote_description(self: &PeerConnection) -> UniquePtr; - fn connection_state(self: &PeerConnection) -> PeerConnectionState; - fn signaling_state(self: &PeerConnection) -> SignalingState; - fn ice_gathering_state(self: &PeerConnection) -> IceGatheringState; - fn ice_connection_state(self: &PeerConnection) -> IceConnectionState; - fn close(self: &PeerConnection); - fn create_native_peer_connection_observer( - rtc_runtime: SharedPtr, - observer: Box, - ) -> SharedPtr; - - fn create_native_add_ice_candidate_observer( - observer: Box, - ) -> UniquePtr; - fn _shared_peer_connection() -> SharedPtr; // Ignore } extern "Rust" { - type AddIceCandidateObserverWrapper; - - fn on_complete(self: &AddIceCandidateObserverWrapper, error: RTCError); - + type AsyncContext; type PeerConnectionObserverWrapper; fn on_signaling_change(self: &PeerConnectionObserverWrapper, new_state: SignalingState); @@ -285,20 +250,16 @@ pub mod ffi { } } +#[repr(transparent)] +pub struct AsyncContext(pub Box); + // https://webrtc.github.io/webrtc-org/native-code/native-apis/ impl_thread_safety!(ffi::PeerConnection, Send + Sync); -impl_thread_safety!(ffi::NativePeerConnectionObserver, Send + Sync); -impl_thread_safety!(ffi::NativeAddIceCandidateObserver, Send + Sync); -impl_thread_safety!(ffi::NativeSetRemoteSdpObserverHandle, Send + Sync); -impl_thread_safety!(ffi::NativeSetLocalSdpObserverHandle, Send + Sync); -impl_thread_safety!(ffi::NativeCreateSdpObserverHandle, Send + Sync); -impl Default for ffi::RTCOfferAnswerOptions { - /* - static const int kUndefined = -1; - static const int kMaxOfferToReceiveMedia = 1; - static const int kOfferToReceiveMediaTrue = 1; - */ +impl Default for ffi::RtcOfferAnswerOptions { + // static const int kUndefined = -1; + // static const int kMaxOfferToReceiveMedia = 1; + // static const int kOfferToReceiveMediaTrue = 1; fn default() -> Self { Self { @@ -314,16 +275,6 @@ impl Default for ffi::RTCOfferAnswerOptions { } } -pub struct AddIceCandidateObserverWrapper(pub ManuallyDrop>); - -impl AddIceCandidateObserverWrapper { - fn on_complete(&self, error: RTCError) { - unsafe { - std::ptr::read(&*self.0)(error); - } - } -} - pub trait PeerConnectionObserver: Send + Sync { fn on_signaling_change(&self, new_state: ffi::SignalingState); fn on_add_stream(&self, stream: SharedPtr); @@ -353,7 +304,8 @@ pub trait PeerConnectionObserver: Send + Sync { fn on_interesting_usage(&self, usage_pattern: i32); } -// Thread safety is handled inside PeerConnectionObserver +// Wrapper for PeerConnectionObserver because cxx doesn't support dyn Trait on c++ +// https://github.com/dtolnay/cxx/issues/665 pub struct PeerConnectionObserverWrapper { observer: Arc, } @@ -420,14 +372,9 @@ impl PeerConnectionObserverWrapper { .on_ice_candidate_error(address, port, url, error_code, error_text); } - fn on_ice_candidates_removed(&self, removed: Vec) { - let mut vec = Vec::new(); - - for v in removed { - vec.push(v.ptr); - } - - self.observer.on_ice_candidates_removed(vec); + fn on_ice_candidates_removed(&self, candidates: Vec) { + self.observer + .on_ice_candidates_removed(candidates.into_iter().map(|v| v.ptr).collect()); } fn on_ice_connection_receiving_change(&self, receiving: bool) { @@ -439,13 +386,8 @@ impl PeerConnectionObserverWrapper { } fn on_add_track(&self, receiver: SharedPtr, streams: Vec) { - let mut vec = Vec::new(); - - for v in streams { - vec.push(v.ptr); - } - - self.observer.on_add_track(receiver, vec); + self.observer + .on_add_track(receiver, streams.into_iter().map(|v| v.ptr).collect()); } fn on_track(&self, transceiver: SharedPtr) { diff --git a/webrtc-sys/src/peer_connection_factory.cpp b/webrtc-sys/src/peer_connection_factory.cpp index 9e863bc..14653f6 100644 --- a/webrtc-sys/src/peer_connection_factory.cpp +++ b/webrtc-sys/src/peer_connection_factory.cpp @@ -16,6 +16,7 @@ #include "livekit/peer_connection_factory.h" +#include #include #include "api/audio_codecs/builtin_audio_decoder_factory.h" @@ -26,19 +27,49 @@ #include "api/video_codecs/builtin_video_decoder_factory.h" #include "api/video_codecs/builtin_video_encoder_factory.h" #include "livekit/audio_device.h" +#include "livekit/peer_connection.h" #include "livekit/rtc_error.h" #include "livekit/rtp_parameters.h" #include "livekit/video_decoder_factory.h" #include "livekit/video_encoder_factory.h" +#include "livekit/webrtc.h" #include "media/engine/webrtc_media_engine.h" #include "rtc_base/location.h" #include "rtc_base/thread.h" +#include "webrtc-sys/src/peer_connection.rs.h" +#include "webrtc-sys/src/peer_connection_factory.rs.h" namespace livekit { +webrtc::PeerConnectionInterface::RTCConfiguration to_native_rtc_configuration( + RtcConfiguration config) { + webrtc::PeerConnectionInterface::RTCConfiguration rtc_config{}; + + for (auto item : config.ice_servers) { + webrtc::PeerConnectionInterface::IceServer ice_server; + ice_server.username = item.username.c_str(); + ice_server.password = item.password.c_str(); + + for (auto url : item.urls) + ice_server.urls.emplace_back(url.c_str()); + + rtc_config.servers.push_back(ice_server); + } + + rtc_config.continual_gathering_policy = + static_cast( + config.continual_gathering_policy); + + rtc_config.type = + static_cast( + config.ice_transport_type); + + return rtc_config; +} + PeerConnectionFactory::PeerConnectionFactory( - std::shared_ptr rtc_runtime) - : rtc_runtime_(std::move(rtc_runtime)) { + std::shared_ptr rtc_runtime) + : rtc_runtime_(rtc_runtime) { RTC_LOG(LS_INFO) << "PeerConnectionFactory::PeerConnectionFactory()"; webrtc::PeerConnectionFactoryDependencies dependencies; @@ -87,31 +118,35 @@ PeerConnectionFactory::~PeerConnectionFactory() { } std::shared_ptr PeerConnectionFactory::create_peer_connection( - std::unique_ptr config, - NativePeerConnectionObserver* observer) const { - webrtc::PeerConnectionDependencies deps{observer}; - auto result = - peer_factory_->CreatePeerConnectionOrError(*config, std::move(deps)); + RtcConfiguration config, + std::unique_ptr observer) const { + observer->rtc_runtime_ = rtc_runtime_; // See peer_connection.h + webrtc::PeerConnectionDependencies deps{observer.get()}; + auto result = peer_factory_->CreatePeerConnectionOrError( + to_native_rtc_configuration(config), std::move(deps)); if (!result.ok()) { throw std::runtime_error(serialize_error(to_error(result.error()))); } - return std::make_shared(rtc_runtime_, result.value()); + return std::make_shared(rtc_runtime_, std::move(observer), + result.value()); } std::shared_ptr PeerConnectionFactory::create_video_track( rust::String label, - std::shared_ptr source) const { - return std::make_shared( - peer_factory_->CreateVideoTrack(label.c_str(), source->get().get())); + std::shared_ptr source) const { + return std::static_pointer_cast( + rtc_runtime_->get_or_create_media_stream_track( + peer_factory_->CreateVideoTrack(label.c_str(), source->get().get()))); } std::shared_ptr PeerConnectionFactory::create_audio_track( rust::String label, std::shared_ptr source) const { - return std::make_shared( - peer_factory_->CreateAudioTrack(label.c_str(), source->get().get())); + return std::static_pointer_cast( + rtc_runtime_->get_or_create_media_stream_track( + peer_factory_->CreateAudioTrack(label.c_str(), source->get().get()))); } RtpCapabilities PeerConnectionFactory::get_rtp_sender_capabilities( @@ -126,35 +161,8 @@ RtpCapabilities PeerConnectionFactory::get_rtp_receiver_capabilities( static_cast(type))); } -std::shared_ptr create_peer_connection_factory( - std::shared_ptr rtc_runtime) { - return std::make_shared(std::move(rtc_runtime)); +std::shared_ptr create_peer_connection_factory() { + return std::make_shared(RtcRuntime::create()); } -std::unique_ptr create_rtc_configuration( - RTCConfiguration conf) { - auto rtc = - std::make_unique(); - - for (auto item : conf.ice_servers) { - webrtc::PeerConnectionInterface::IceServer ice_server; - ice_server.username = item.username.c_str(); - ice_server.password = item.password.c_str(); - - for (auto url : item.urls) { - ice_server.urls.emplace_back(url.c_str()); - } - - rtc->servers.push_back(ice_server); - } - - rtc->continual_gathering_policy = - static_cast( - conf.continual_gathering_policy); - - rtc->type = static_cast( - conf.ice_transport_type); - - return rtc; -} } // namespace livekit diff --git a/webrtc-sys/src/peer_connection_factory.rs b/webrtc-sys/src/peer_connection_factory.rs index 79398f9..3f04f9d 100644 --- a/webrtc-sys/src/peer_connection_factory.rs +++ b/webrtc-sys/src/peer_connection_factory.rs @@ -2,21 +2,18 @@ use crate::impl_thread_safety; #[cxx::bridge(namespace = "livekit")] pub mod ffi { - #[derive(Debug, Clone)] - pub struct ICEServer { + pub struct IceServer { pub urls: Vec, pub username: String, pub password: String, } - #[derive(Debug)] #[repr(i32)] pub enum ContinualGatheringPolicy { GatherOnce, GatherContinually, } - #[derive(Debug)] #[repr(i32)] pub enum IceTransportsType { None, @@ -25,9 +22,8 @@ pub mod ffi { All, } - #[derive(Debug, Clone)] - pub struct RTCConfiguration { - pub ice_servers: Vec, + pub struct RtcConfiguration { + pub ice_servers: Vec, pub continual_gathering_policy: ContinualGatheringPolicy, pub ice_transport_type: IceTransportsType, } @@ -37,41 +33,34 @@ pub mod ffi { include!("livekit/webrtc.h"); include!("livekit/rtp_parameters.h"); - type AudioTrackSource = crate::media_stream::ffi::AudioTrackSource; - type AdaptedVideoTrackSource = crate::media_stream::ffi::AdaptedVideoTrackSource; - type AudioTrack = crate::media_stream::ffi::AudioTrack; - type VideoTrack = crate::media_stream::ffi::VideoTrack; + type AudioTrackSource = crate::audio_track::ffi::AudioTrackSource; + type VideoTrackSource = crate::video_track::ffi::VideoTrackSource; + type AudioTrack = crate::audio_track::ffi::AudioTrack; + type VideoTrack = crate::video_track::ffi::VideoTrack; type RtpCapabilities = crate::rtp_parameters::ffi::RtpCapabilities; type MediaType = crate::webrtc::ffi::MediaType; + type NativePeerConnectionObserver = + crate::peer_connection::ffi::NativePeerConnectionObserver; } unsafe extern "C++" { include!("livekit/peer_connection_factory.h"); type PeerConnection = crate::peer_connection::ffi::PeerConnection; - type NativePeerConnectionObserver = - crate::peer_connection::ffi::NativePeerConnectionObserver; type PeerConnectionFactory; - type NativeRTCConfiguration; - type RTCRuntime = crate::webrtc::ffi::RTCRuntime; - fn create_peer_connection_factory( - runtime: SharedPtr, - ) -> SharedPtr; - fn create_rtc_configuration(conf: RTCConfiguration) -> UniquePtr; + fn create_peer_connection_factory() -> SharedPtr; - /// # Safety - /// The observer must live as long as the PeerConnection does - unsafe fn create_peer_connection( + fn create_peer_connection( self: &PeerConnectionFactory, - config: UniquePtr, - observer: *mut NativePeerConnectionObserver, + config: RtcConfiguration, + observer: UniquePtr, ) -> Result>; fn create_video_track( self: &PeerConnectionFactory, label: String, - source: SharedPtr, + source: SharedPtr, ) -> SharedPtr; fn create_audio_track( diff --git a/webrtc-sys/src/rtc_error.cpp b/webrtc-sys/src/rtc_error.cpp index 32d745c..363bb02 100644 --- a/webrtc-sys/src/rtc_error.cpp +++ b/webrtc-sys/src/rtc_error.cpp @@ -22,17 +22,17 @@ namespace livekit { -RTCError to_error(const webrtc::RTCError& error) { - RTCError lk_error; - lk_error.error_detail = static_cast(error.error_detail()); - lk_error.error_type = static_cast(error.type()); +RtcError to_error(const webrtc::RTCError& error) { + RtcError lk_error; + lk_error.error_detail = static_cast(error.error_detail()); + lk_error.error_type = static_cast(error.type()); lk_error.has_sctp_cause_code = error.sctp_cause_code().has_value(); lk_error.sctp_cause_code = error.sctp_cause_code().value_or(0); lk_error.message = error.message(); return lk_error; } -std::string serialize_error(const RTCError& error) { +std::string serialize_error(const RtcError& error) { std::stringstream ss; ss << std::hex << std::setfill('0'); ss << std::setw(8) << (uint32_t)error.error_type; @@ -45,9 +45,9 @@ std::string serialize_error(const RTCError& error) { #ifdef LIVEKIT_TEST rust::String serialize_deserialize() { - RTCError lk_error; - lk_error.error_type = RTCErrorType::InternalError; - lk_error.error_detail = RTCErrorDetailType::DataChannelFailure; + RtcError lk_error; + lk_error.error_type = RtcErrorType::InternalError; + lk_error.error_detail = RtcErrorDetailType::DataChannelFailure; lk_error.has_sctp_cause_code = true; lk_error.sctp_cause_code = 24; lk_error.message = "this is not a test, I repeat, this is not a test"; @@ -55,9 +55,9 @@ rust::String serialize_deserialize() { } void throw_error() { - RTCError lk_error; - lk_error.error_type = RTCErrorType::InvalidModification; - lk_error.error_detail = RTCErrorDetailType::None; + RtcError lk_error; + lk_error.error_type = RtcErrorType::InvalidModification; + lk_error.error_detail = RtcErrorDetailType::None; lk_error.has_sctp_cause_code = false; lk_error.sctp_cause_code = 0; lk_error.message = "exception is thrown!"; diff --git a/webrtc-sys/src/rtc_error.rs b/webrtc-sys/src/rtc_error.rs index 77ebe9b..64557d4 100644 --- a/webrtc-sys/src/rtc_error.rs +++ b/webrtc-sys/src/rtc_error.rs @@ -1,15 +1,13 @@ use std::error::Error; use std::fmt::{Display, Formatter}; -use crate::rtc_error::ffi::RTCErrorType; - -// cxx doesn't support custom Exception type, so we serialize RTCError inside the cxx::Exception "what" string +// cxx doesn't support custom Exception type, so we serialize RtcError inside the cxx::Exception "what" string #[cxx::bridge(namespace = "livekit")] pub mod ffi { #[derive(Debug)] #[repr(i32)] - pub enum RTCErrorType { + pub enum RtcErrorType { None, UnsupportedOperation, UnsupportedParameter, @@ -26,7 +24,7 @@ pub mod ffi { #[derive(Debug)] #[repr(i32)] - pub enum RTCErrorDetailType { + pub enum RtcErrorDetailType { None, DataChannelFailure, DtlsFailure, @@ -38,17 +36,17 @@ pub mod ffi { } #[derive(Debug)] - pub struct RTCError { - pub error_type: RTCErrorType, + pub struct RtcError { + pub error_type: RtcErrorType, pub message: String, - pub error_detail: RTCErrorDetailType, - pub has_sctp_cause_code: bool, + pub error_detail: RtcErrorDetailType, // cxx doesn't support the Option trait + pub has_sctp_cause_code: bool, pub sctp_cause_code: u16, } } -impl ffi::RTCError { +impl ffi::RtcError { /// # Safety /// The value must be correctly encoded pub unsafe fn from(value: &str) -> Self { @@ -69,13 +67,13 @@ impl ffi::RTCError { } pub fn ok(&self) -> bool { - self.error_type == RTCErrorType::None + self.error_type == ffi::RtcErrorType::None } } -impl Error for ffi::RTCError {} +impl Error for ffi::RtcError {} -impl Display for ffi::RTCError { +impl Display for ffi::RtcError { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { write!( f, @@ -87,7 +85,7 @@ impl Display for ffi::RTCError { #[cfg(test)] mod tests { - use crate::rtc_error::ffi::{RTCError, RTCErrorDetailType, RTCErrorType}; + use crate::rtc_error::ffi::{RtcError, RtcErrorDetailType, RtcErrorType}; #[cxx::bridge(namespace = "livekit")] pub mod ffi { @@ -102,10 +100,10 @@ mod tests { #[test] fn serialize_deserialize() { let str = ffi::serialize_deserialize(); - let error = unsafe { RTCError::from(&str) }; + let error = unsafe { RtcError::from(&str) }; - assert_eq!(error.error_type, RTCErrorType::InternalError); - assert_eq!(error.error_detail, RTCErrorDetailType::DataChannelFailure); + assert_eq!(error.error_type, RtcErrorType::InternalError); + assert_eq!(error.error_detail, RtcErrorDetailType::DataChannelFailure); assert_eq!(error.has_sctp_cause_code, true); assert_eq!(error.sctp_cause_code, 24); assert_eq!( @@ -117,10 +115,10 @@ mod tests { #[test] fn throw_error() { let exc: cxx::Exception = ffi::throw_error().err().unwrap(); - let error = unsafe { RTCError::from(exc.what()) }; + let error = unsafe { RtcError::from(exc.what()) }; - assert_eq!(error.error_type, RTCErrorType::InvalidModification); - assert_eq!(error.error_detail, RTCErrorDetailType::None); + assert_eq!(error.error_type, RtcErrorType::InvalidModification); + assert_eq!(error.error_detail, RtcErrorDetailType::None); assert_eq!(error.has_sctp_cause_code, false); assert_eq!(error.sctp_cause_code, 0); assert_eq!(error.message, "exception is thrown!"); diff --git a/webrtc-sys/src/rtp_parameters.rs b/webrtc-sys/src/rtp_parameters.rs index 4e0b9ca..31cac36 100644 --- a/webrtc-sys/src/rtp_parameters.rs +++ b/webrtc-sys/src/rtp_parameters.rs @@ -183,7 +183,7 @@ pub mod ffi { } extern "C++" { - include!("webrtc-sys/src/webrtc.rs.h"); + include!("livekit/webrtc.h"); type Priority = crate::webrtc::ffi::Priority; type MediaType = crate::webrtc::ffi::MediaType; diff --git a/webrtc-sys/src/rtp_receiver.cpp b/webrtc-sys/src/rtp_receiver.cpp index 6fa0b19..14754e8 100644 --- a/webrtc-sys/src/rtp_receiver.cpp +++ b/webrtc-sys/src/rtp_receiver.cpp @@ -16,16 +16,19 @@ #include "livekit/rtp_receiver.h" +#include + #include "absl/types/optional.h" namespace livekit { RtpReceiver::RtpReceiver( + std::shared_ptr rtc_runtime, rtc::scoped_refptr receiver) - : receiver_(std::move(receiver)) {} + : rtc_runtime_(rtc_runtime), receiver_(std::move(receiver)) {} std::shared_ptr RtpReceiver::track() const { - return MediaStreamTrack::from(receiver_->track()); + return rtc_runtime_->get_or_create_media_stream_track(receiver_->track()); } rust::Vec RtpReceiver::stream_ids() const { @@ -38,7 +41,8 @@ rust::Vec RtpReceiver::stream_ids() const { rust::Vec RtpReceiver::streams() const { rust::Vec rust; for (auto stream : receiver_->streams()) - rust.push_back(MediaStreamPtr{std::make_shared(stream)}); + rust.push_back( + MediaStreamPtr{std::make_shared(rtc_runtime_, stream)}); return rust; } diff --git a/webrtc-sys/src/rtp_sender.cpp b/webrtc-sys/src/rtp_sender.cpp index e43022e..3d61876 100644 --- a/webrtc-sys/src/rtp_sender.cpp +++ b/webrtc-sys/src/rtp_sender.cpp @@ -18,15 +18,16 @@ namespace livekit { -RtpSender::RtpSender(rtc::scoped_refptr sender) - : sender_(std::move(sender)) {} +RtpSender::RtpSender(std::shared_ptr rtc_runtime, + rtc::scoped_refptr sender) + : rtc_runtime_(rtc_runtime), sender_(std::move(sender)) {} bool RtpSender::set_track(std::shared_ptr track) const { - return sender_->SetTrack(track->get().get()); + return sender_->SetTrack(track->rtc_track().get()); } std::shared_ptr RtpSender::track() const { - return MediaStreamTrack::from(sender_->track()); + return rtc_runtime_->get_or_create_media_stream_track(sender_->track()); } uint32_t RtpSender::ssrc() const { diff --git a/webrtc-sys/src/rtp_transceiver.cpp b/webrtc-sys/src/rtp_transceiver.cpp index 92996bd..e9f03ec 100644 --- a/webrtc-sys/src/rtp_transceiver.cpp +++ b/webrtc-sys/src/rtp_transceiver.cpp @@ -34,8 +34,9 @@ webrtc::RtpTransceiverInit to_native_rtp_transceiver_init( } RtpTransceiver::RtpTransceiver( + std::shared_ptr rtc_runtime, rtc::scoped_refptr transceiver) - : transceiver_(std::move(transceiver)) {} + : rtc_runtime_(rtc_runtime), transceiver_(std::move(transceiver)) {} MediaType RtpTransceiver::media_type() const { return static_cast(transceiver_->media_type()); @@ -48,11 +49,11 @@ rust::String RtpTransceiver::mid() const { } std::shared_ptr RtpTransceiver::sender() const { - return std::make_shared(transceiver_->sender()); + return std::make_shared(rtc_runtime_, transceiver_->sender()); } std::shared_ptr RtpTransceiver::receiver() const { - return std::make_shared(transceiver_->receiver()); + return std::make_shared(rtc_runtime_, transceiver_->receiver()); } bool RtpTransceiver::stopped() const { diff --git a/webrtc-sys/src/rtp_transceiver.rs b/webrtc-sys/src/rtp_transceiver.rs index 73fe2d5..908dcaf 100644 --- a/webrtc-sys/src/rtp_transceiver.rs +++ b/webrtc-sys/src/rtp_transceiver.rs @@ -24,7 +24,7 @@ pub mod ffi { crate::rtp_parameters::ffi::RtpHeaderExtensionCapability; type RtpSender = crate::rtp_sender::ffi::RtpSender; type RtpReceiver = crate::rtp_receiver::ffi::RtpReceiver; - type RTCError = crate::rtc_error::ffi::RTCError; + type RtcError = crate::rtc_error::ffi::RtcError; } unsafe extern "C++" { diff --git a/webrtc-sys/src/video_track.cpp b/webrtc-sys/src/video_track.cpp new file mode 100644 index 0000000..ced13d5 --- /dev/null +++ b/webrtc-sys/src/video_track.cpp @@ -0,0 +1,188 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the “License”); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an “AS IS” BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "livekit/video_track.h" + +#include +#include +#include + +#include "api/media_stream_interface.h" +#include "api/video/video_frame.h" +#include "api/video/video_rotation.h" +#include "audio/remix_resample.h" +#include "common_audio/include/audio_util.h" +#include "livekit/media_stream.h" +#include "livekit/video_track.h" +#include "rtc_base/logging.h" +#include "rtc_base/ref_counted_object.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/time_utils.h" + +namespace livekit { + +VideoTrack::VideoTrack(std::shared_ptr rtc_runtime, + rtc::scoped_refptr track) + : MediaStreamTrack(rtc_runtime, std::move(track)) {} + +VideoTrack::~VideoTrack() { + webrtc::MutexLock lock(&mutex_); + for (auto& sink : sinks_) { + track()->RemoveSink(sink.get()); + } +} + +void VideoTrack::add_sink(const std::shared_ptr& sink) const { + webrtc::MutexLock lock(&mutex_); + track()->AddOrUpdateSink(sink.get(), + rtc::VideoSinkWants()); // TODO(theomonnom): Expose + // VideoSinkWants to Rust? + sinks_.push_back(sink); +} + +void VideoTrack::remove_sink( + const std::shared_ptr& sink) const { + webrtc::MutexLock lock(&mutex_); + track()->RemoveSink(sink.get()); + sinks_.erase(std::remove(sinks_.begin(), sinks_.end(), sink), sinks_.end()); +} + +void VideoTrack::set_should_receive(bool should_receive) const { + track()->set_should_receive(should_receive); +} + +bool VideoTrack::should_receive() const { + return track()->should_receive(); +} + +ContentHint VideoTrack::content_hint() const { + return static_cast(track()->content_hint()); +} + +void VideoTrack::set_content_hint(ContentHint hint) const { + track()->set_content_hint( + static_cast(hint)); +} + +NativeVideoSink::NativeVideoSink(rust::Box observer) + : observer_(std::move(observer)) {} + +void NativeVideoSink::OnFrame(const webrtc::VideoFrame& frame) { + observer_->on_frame(std::make_unique(frame)); +} + +void NativeVideoSink::OnDiscardedFrame() { + observer_->on_discarded_frame(); +} + +void NativeVideoSink::OnConstraintsChanged( + const webrtc::VideoTrackSourceConstraints& constraints) { + VideoTrackSourceConstraints cst; + cst.has_min_fps = constraints.min_fps.has_value(); + cst.min_fps = constraints.min_fps.value_or(0); + cst.has_max_fps = constraints.max_fps.has_value(); + cst.max_fps = constraints.max_fps.value_or(0); + observer_->on_constraints_changed(cst); +} + +std::shared_ptr new_native_video_sink( + rust::Box observer) { + return std::make_shared(std::move(observer)); +} + +VideoTrackSource::InternalSource::InternalSource() + : rtc::AdaptedVideoTrackSource(4) {} + +VideoTrackSource::InternalSource::~InternalSource() {} + +bool VideoTrackSource::InternalSource::is_screencast() const { + return false; +} + +absl::optional VideoTrackSource::InternalSource::needs_denoising() const { + return false; +} + +webrtc::MediaSourceInterface::SourceState +VideoTrackSource::InternalSource::state() const { + return SourceState::kLive; +} + +bool VideoTrackSource::InternalSource::remote() const { + return false; +} + +bool VideoTrackSource::InternalSource::on_captured_frame( + const webrtc::VideoFrame& frame) { + webrtc::MutexLock lock(&mutex_); + + int64_t aligned_timestamp_us = timestamp_aligner_.TranslateTimestamp( + frame.timestamp_us(), rtc::TimeMicros()); + + rtc::scoped_refptr buffer = + frame.video_frame_buffer(); + + int adapted_width, adapted_height, crop_width, crop_height, crop_x, crop_y; + if (!AdaptFrame(buffer->width(), buffer->height(), aligned_timestamp_us, + &adapted_width, &adapted_height, &crop_width, &crop_height, + &crop_x, &crop_y)) { + return false; + } + + if (adapted_width != frame.width() || adapted_height != frame.height()) { + buffer = buffer->CropAndScale(crop_x, crop_y, crop_width, crop_height, + adapted_width, adapted_height); + } + + webrtc::VideoRotation rotation = frame.rotation(); + if (apply_rotation() && rotation != webrtc::kVideoRotation_0) { + // If the buffer is I420, rtc::AdaptedVideoTrackSource will handle the + // rotation for us. + buffer = buffer->ToI420(); + } + + OnFrame(webrtc::VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_rotation(rotation) + .set_timestamp_us(aligned_timestamp_us) + .build()); + + return true; +} + +VideoTrackSource::VideoTrackSource() { + source_ = rtc::make_ref_counted(); +} + +bool VideoTrackSource::on_captured_frame( + const std::unique_ptr& frame) const { + auto rtc_frame = frame->get(); + rtc_frame.set_timestamp_us( + rtc::TimeMicros()); // TODO(theomonnom): Expore capture ts to Rust + + return source_->on_captured_frame(rtc_frame); +} + +rtc::scoped_refptr VideoTrackSource::get() + const { + return source_; +} + +std::shared_ptr new_video_track_source() { + return std::make_shared(); +} + +} // namespace livekit diff --git a/webrtc-sys/src/video_track.rs b/webrtc-sys/src/video_track.rs new file mode 100644 index 0000000..bcb840c --- /dev/null +++ b/webrtc-sys/src/video_track.rs @@ -0,0 +1,97 @@ +use crate::impl_thread_safety; +use crate::video_frame::ffi::VideoFrame; +use cxx::UniquePtr; +use std::sync::Arc; + +#[cxx::bridge(namespace = "livekit")] +pub mod ffi { + #[repr(i32)] + pub enum ContentHint { + None, + Fluid, + Detailed, + Text, + } + + #[derive(Debug)] + pub struct VideoTrackSourceConstraints { + pub has_min_fps: bool, + pub min_fps: f64, + pub has_max_fps: bool, + pub max_fps: f64, + } + + extern "C++" { + include!("livekit/video_frame.h"); + include!("livekit/media_stream_track.h"); + + type VideoFrame = crate::video_frame::ffi::VideoFrame; + type MediaStreamTrack = crate::media_stream_track::ffi::MediaStreamTrack; + } + + unsafe extern "C++" { + include!("livekit/video_track.h"); + + type VideoTrack; + type NativeVideoSink; + type VideoTrackSource; + + fn add_sink(self: &VideoTrack, sink: &SharedPtr); + fn remove_sink(self: &VideoTrack, sink: &SharedPtr); + fn set_should_receive(self: &VideoTrack, should_receive: bool); + fn should_receive(self: &VideoTrack) -> bool; + fn content_hint(self: &VideoTrack) -> ContentHint; + fn set_content_hint(self: &VideoTrack, hint: ContentHint); + fn new_native_video_sink(observer: Box) -> SharedPtr; + + fn on_captured_frame(self: &VideoTrackSource, frame: &UniquePtr) -> bool; + fn new_video_track_source() -> SharedPtr; + + fn video_to_media(track: SharedPtr) -> SharedPtr; + unsafe fn media_to_video(track: SharedPtr) -> SharedPtr; + fn _shared_video_track() -> SharedPtr; + } + + extern "Rust" { + type VideoSinkWrapper; + + fn on_frame(self: &VideoSinkWrapper, frame: UniquePtr); + fn on_discarded_frame(self: &VideoSinkWrapper); + fn on_constraints_changed( + self: &VideoSinkWrapper, + constraints: VideoTrackSourceConstraints, + ); + } +} + +impl_thread_safety!(ffi::VideoTrack, Send + Sync); +impl_thread_safety!(ffi::NativeVideoSink, Send + Sync); +impl_thread_safety!(ffi::VideoTrackSource, Send + Sync); + +pub trait VideoSink: Send { + fn on_frame(&self, frame: UniquePtr); + fn on_discarded_frame(&self); + fn on_constraints_changed(&self, constraints: ffi::VideoTrackSourceConstraints); +} + +pub struct VideoSinkWrapper { + observer: Arc, +} + +impl VideoSinkWrapper { + pub fn new(observer: Arc) -> Self { + Self { observer } + } + + fn on_frame(&self, frame: UniquePtr) { + self.observer.on_frame(frame); + } + + fn on_discarded_frame(&self) { + self.observer.on_discarded_frame(); + } + + fn on_constraints_changed(&self, constraints: ffi::VideoTrackSourceConstraints) { + self.observer.on_constraints_changed(constraints); + } +} diff --git a/webrtc-sys/src/webrtc.cpp b/webrtc-sys/src/webrtc.cpp index e7c9ccc..eacd3bb 100644 --- a/webrtc-sys/src/webrtc.cpp +++ b/webrtc-sys/src/webrtc.cpp @@ -16,13 +16,20 @@ #include "livekit/webrtc.h" +#include + +#include "livekit/audio_track.h" +#include "livekit/media_stream_track.h" +#include "livekit/rtp_receiver.h" +#include "livekit/rtp_sender.h" +#include "livekit/video_track.h" #include "rtc_base/helpers.h" #include "rtc_base/logging.h" +#include "rtc_base/synchronization/mutex.h" namespace livekit { -RTCRuntime::RTCRuntime() { - // rtc::LogMessage::LogToDebug(rtc::LS_INFO); - RTC_LOG(LS_INFO) << "RTCRuntime()"; +RtcRuntime::RtcRuntime() { + RTC_LOG(LS_INFO) << "RtcRuntime()"; RTC_CHECK(rtc::InitializeSSL()) << "Failed to InitializeSSL()"; network_thread_ = rtc::Thread::CreateWithSocketServer(); @@ -36,34 +43,99 @@ RTCRuntime::RTCRuntime() { signaling_thread_->Start(); } -RTCRuntime::~RTCRuntime() { - RTC_LOG(LS_INFO) << "~RTCRuntime()"; +RtcRuntime::~RtcRuntime() { + RTC_LOG(LS_INFO) << "~RtcRuntime()"; rtc::ThreadManager::Instance()->SetCurrentThread(nullptr); RTC_CHECK(rtc::CleanupSSL()) << "Failed to CleanupSSL()"; - worker_thread_->Stop(); - signaling_thread_->Stop(); - network_thread_->Stop(); + worker_thread_->Quit(); + signaling_thread_->Quit(); + network_thread_->Quit(); } -rtc::Thread* RTCRuntime::network_thread() const { +rtc::Thread* RtcRuntime::network_thread() const { return network_thread_.get(); } -rtc::Thread* RTCRuntime::worker_thread() const { +rtc::Thread* RtcRuntime::worker_thread() const { return worker_thread_.get(); } -rtc::Thread* RTCRuntime::signaling_thread() const { +rtc::Thread* RtcRuntime::signaling_thread() const { return signaling_thread_.get(); } +std::shared_ptr RtcRuntime::get_or_create_media_stream_track( + rtc::scoped_refptr rtc_track) { + webrtc::MutexLock lock(&mutex_); + for (std::weak_ptr weak_existing_track : + media_stream_tracks_) { + if (std::shared_ptr existing_track = + weak_existing_track.lock()) { + if (existing_track->rtc_track() == rtc_track) { + return existing_track; + } + } + } + + if (rtc_track->kind() == webrtc::MediaStreamTrackInterface::kVideoKind) { + std::shared_ptr video_track = + std::shared_ptr(new VideoTrack( + shared_from_this(), + rtc::scoped_refptr( + static_cast(rtc_track.get())))); + + media_stream_tracks_.push_back( + std::static_pointer_cast(video_track)); + return video_track; + } else { + std::shared_ptr audio_track = + std::shared_ptr(new AudioTrack( + shared_from_this(), + rtc::scoped_refptr( + static_cast(rtc_track.get())))); + + media_stream_tracks_.push_back( + std::static_pointer_cast(audio_track)); + return audio_track; + } +} + +std::shared_ptr RtcRuntime::get_or_create_audio_track( + rtc::scoped_refptr track) { + return std::static_pointer_cast( + get_or_create_media_stream_track(track)); +} + +std::shared_ptr RtcRuntime::get_or_create_video_track( + rtc::scoped_refptr track) { + return std::static_pointer_cast( + get_or_create_media_stream_track(track)); +} + +LogSink::LogSink( + rust::Fn fnc) + : fnc_(fnc) { + rtc::LogMessage::AddLogToStream(this, rtc::LoggingSeverity::LS_VERBOSE); +} + +LogSink::~LogSink() { + rtc::LogMessage::RemoveLogToStream(this); +} + +void LogSink::OnLogMessage(const std::string& message, + rtc::LoggingSeverity severity) { + fnc_(rust::String(message), static_cast(severity)); +} + +std::unique_ptr new_log_sink( + rust::Fn fnc) { + return std::make_unique(fnc); +} + rust::String create_random_uuid() { return rtc::CreateRandomUuid(); } -std::shared_ptr create_rtc_runtime() { - return std::make_shared(); -} } // namespace livekit diff --git a/webrtc-sys/src/webrtc.rs b/webrtc-sys/src/webrtc.rs index 0223121..9f63543 100644 --- a/webrtc-sys/src/webrtc.rs +++ b/webrtc-sys/src/webrtc.rs @@ -2,7 +2,6 @@ use crate::impl_thread_safety; #[cxx::bridge(namespace = "livekit")] pub mod ffi { - #[derive(Debug)] #[repr(i32)] pub enum MediaType { @@ -31,14 +30,24 @@ pub mod ffi { Stopped, } + #[derive(Debug)] + #[repr(i32)] + pub enum LoggingSeverity { + Verbose, + Info, + Warning, + Error, + None, + } + unsafe extern "C++" { include!("livekit/webrtc.h"); - type RTCRuntime; + + type LogSink; fn create_random_uuid() -> String; - - fn create_rtc_runtime() -> SharedPtr; + fn new_log_sink(fnc: fn(String, LoggingSeverity)) -> UniquePtr; } } -impl_thread_safety!(ffi::RTCRuntime, Send + Sync); +impl_thread_safety!(ffi::LogSink, Send + Sync);