From 25f4c9a07568074146933d5dbf2a249bf8eeba99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sun, 18 Jun 2023 22:45:05 +0200 Subject: [PATCH] feat: add ffi datachannel & mute events (#88) --- Cargo.lock | 26 +- examples/Cargo.lock | 121 +------ examples/basic_room/src/main.rs | 2 +- examples/save_to_disk/src/main.rs | 4 +- examples/wgpu_room/Cargo.toml | 2 +- examples/wgpu_room/src/app.rs | 2 +- examples/wgpu_room/src/logo_track.rs | 8 +- examples/wgpu_room/src/sine_track.rs | 10 +- livekit-api/src/services/egress.rs | 3 +- livekit-api/src/services/ingress.rs | 4 +- livekit-ffi/Cargo.toml | 4 +- livekit-ffi/protocol/audio_frame.proto | 18 +- livekit-ffi/protocol/ffi.proto | 64 ++-- livekit-ffi/protocol/handle.proto | 5 +- livekit-ffi/protocol/participant.proto | 1 + livekit-ffi/protocol/room.proto | 32 +- livekit-ffi/protocol/track.proto | 19 +- livekit-ffi/protocol/video_frame.proto | 21 +- livekit-ffi/src/conversion/audio_frame.rs | 12 +- livekit-ffi/src/conversion/room.rs | 77 +--- livekit-ffi/src/conversion/track.rs | 21 +- livekit-ffi/src/conversion/video_frame.rs | 13 +- livekit-ffi/src/server/audio_frame.rs | 66 ++-- livekit-ffi/src/server/mod.rs | 100 ++++-- livekit-ffi/src/server/room.rs | 187 ++++++++-- livekit-ffi/src/server/tests.rs | 24 +- livekit-ffi/src/server/utils.rs | 32 -- livekit-ffi/src/server/video_frame.rs | 63 ++-- livekit-protocol/protocol | 2 +- livekit-webrtc/Cargo.toml | 2 +- livekit-webrtc/src/audio_source.rs | 47 ++- livekit-webrtc/src/native/audio_source.rs | 39 ++- livekit-webrtc/src/native/video_source.rs | 43 ++- livekit-webrtc/src/native/video_stream.rs | 2 +- livekit-webrtc/src/prelude.rs | 2 + livekit-webrtc/src/video_frame.rs | 2 +- livekit-webrtc/src/video_source.rs | 44 ++- livekit/Cargo.toml | 7 +- livekit/src/prelude.rs | 4 +- livekit/src/proto.rs | 20 +- livekit/src/room/id.rs | 22 +- livekit/src/room/mod.rs | 170 +++++---- livekit/src/room/options.rs | 33 +- .../src/room/participant/local_participant.rs | 66 ++-- livekit/src/room/participant/mod.rs | 103 +++--- .../room/participant/remote_participant.rs | 61 ++-- livekit/src/room/publication/local.rs | 56 ++- livekit/src/room/publication/mod.rs | 328 +++++++++++------- livekit/src/room/publication/remote.rs | 138 +++++++- livekit/src/room/track/local_audio_track.rs | 107 +++--- livekit/src/room/track/local_track.rs | 33 ++ livekit/src/room/track/local_video_track.rs | 106 +++--- livekit/src/room/track/mod.rs | 216 ++++-------- livekit/src/room/track/remote_audio_track.rs | 16 +- livekit/src/room/track/remote_track.rs | 33 ++ livekit/src/room/track/remote_video_track.rs | 17 +- livekit/src/rtc_engine/lk_runtime.rs | 5 +- livekit/src/rtc_engine/mod.rs | 306 ++++++++-------- livekit/src/rtc_engine/peer_transport.rs | 14 +- livekit/src/rtc_engine/rtc_events.rs | 2 +- livekit/src/rtc_engine/rtc_session.rs | 133 +++---- livekit/src/signal_client/mod.rs | 177 ++++++---- livekit/src/signal_client/signal_stream.rs | 44 +-- webrtc-sys/include/livekit/audio_track.h | 16 +- webrtc-sys/include/livekit/video_track.h | 17 +- webrtc-sys/src/audio_track.cpp | 53 ++- webrtc-sys/src/audio_track.rs | 11 +- webrtc-sys/src/video_track.cpp | 32 +- webrtc-sys/src/video_track.rs | 10 +- 69 files changed, 1934 insertions(+), 1546 deletions(-) delete mode 100644 livekit-ffi/src/server/utils.rs create mode 100644 livekit/src/room/track/local_track.rs create mode 100644 livekit/src/room/track/remote_track.rs diff --git a/Cargo.lock b/Cargo.lock index 52727b0..491a5f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -785,15 +785,14 @@ dependencies = [ "lazy_static", "livekit-protocol", "livekit-webrtc", + "log", "parking_lot", "prost", "serde", "serde_json", "thiserror", "tokio", - "tokio-stream", "tokio-tungstenite", - "tracing", "url", ] @@ -1676,17 +1675,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-stream" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-tungstenite" version = "0.19.0" @@ -1733,21 +1721,9 @@ checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if", "pin-project-lite", - "tracing-attributes", "tracing-core", ] -[[package]] -name = "tracing-attributes" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.18", -] - [[package]] name = "tracing-core" version = "0.1.31" diff --git a/examples/Cargo.lock b/examples/Cargo.lock index 03d805c..f66ed2f 100644 --- a/examples/Cargo.lock +++ b/examples/Cargo.lock @@ -841,7 +841,7 @@ dependencies = [ "futures-sink", "nanorand", "pin-project", - "spin 0.9.8", + "spin", ] [[package]] @@ -1481,15 +1481,14 @@ dependencies = [ "lazy_static", "livekit-protocol", "livekit-webrtc", + "log", "parking_lot", "prost", "serde", "serde_json", "thiserror", "tokio", - "tokio-stream", "tokio-tungstenite", - "tracing", "url", ] @@ -2352,21 +2351,6 @@ dependencies = [ "winreg", ] -[[package]] -name = "ring" -version = "0.16.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" -dependencies = [ - "cc", - "libc", - "once_cell", - "spin 0.5.2", - "untrusted", - "web-sys", - "winapi", -] - [[package]] name = "rustc-demangle" version = "0.1.23" @@ -2393,49 +2377,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "rustls" -version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c911ba11bc8433e811ce56fde130ccf32f5127cab0e0194e9c68c5a5b671791e" -dependencies = [ - "log", - "ring", - "rustls-webpki", - "sct", -] - -[[package]] -name = "rustls-native-certs" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" -dependencies = [ - "openssl-probe", - "rustls-pemfile", - "schannel", - "security-framework", -] - -[[package]] -name = "rustls-pemfile" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" -dependencies = [ - "base64", -] - -[[package]] -name = "rustls-webpki" -version = "0.100.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "ryu" version = "1.0.13" @@ -2489,16 +2430,6 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" -[[package]] -name = "sct" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "sctk-adwaita" version = "0.5.4" @@ -2687,12 +2618,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - [[package]] name = "spin" version = "0.9.8" @@ -2928,27 +2853,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-rustls" -version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0d409377ff5b1e3ca6437aa86c1eb7d40c134bfec254e44c830defa92669db5" -dependencies = [ - "rustls", - "tokio", -] - -[[package]] -name = "tokio-stream" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-tungstenite" version = "0.19.0" @@ -2957,10 +2861,7 @@ checksum = "ec509ac96e9a0c43427c74f003127d953a265737636129424288d27cb5c4b12c" dependencies = [ "futures-util", "log", - "rustls", - "rustls-native-certs", "tokio", - "tokio-rustls", "tungstenite", ] @@ -3084,12 +2985,10 @@ dependencies = [ "httparse", "log", "rand", - "rustls", "sha1", "thiserror", "url", "utf-8", - "webpki", ] [[package]] @@ -3140,12 +3039,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" -[[package]] -name = "untrusted" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" - [[package]] name = "url" version = "2.3.1" @@ -3379,16 +3272,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "webpki" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "webrtc-sys" version = "0.1.1" diff --git a/examples/basic_room/src/main.rs b/examples/basic_room/src/main.rs index 32f2f40..786ee20 100644 --- a/examples/basic_room/src/main.rs +++ b/examples/basic_room/src/main.rs @@ -11,7 +11,7 @@ async fn main() { let url = env::var("LIVEKIT_URL").expect("LIVEKIT_URL is not set"); let token = env::var("LIVEKIT_TOKEN").expect("LIVEKIT_TOKEN is not set"); - let (room, mut rx) = Room::connect(&url, &token).await.unwrap(); + let (room, mut rx) = Room::connect(&url, &token, RoomOptions::default()).await.unwrap(); log::info!("Connected to room: {} - {}", room.name(), room.sid()); while let Some(msg) = rx.recv().await { diff --git a/examples/save_to_disk/src/main.rs b/examples/save_to_disk/src/main.rs index c7b6e30..250a4ae 100644 --- a/examples/save_to_disk/src/main.rs +++ b/examples/save_to_disk/src/main.rs @@ -89,7 +89,9 @@ async fn main() { let url = env::var("LIVEKIT_URL").expect("LIVEKIT_URL is not set"); let token = env::var("LIVEKIT_TOKEN").expect("LIVEKIT_TOKEN is not set"); - let (room, mut rx) = Room::connect(&url, &token).await.unwrap(); + let (room, mut rx) = Room::connect(&url, &token, RoomOptions::default()) + .await + .unwrap(); println!("Connected to room: {} - {}", room.name(), room.sid()); while let Some(msg) = rx.recv().await { diff --git a/examples/wgpu_room/Cargo.toml b/examples/wgpu_room/Cargo.toml index ea9cea9..596d9d4 100644 --- a/examples/wgpu_room/Cargo.toml +++ b/examples/wgpu_room/Cargo.toml @@ -11,7 +11,7 @@ livekit = { path = "../../livekit", version = "0.1.1" } futures = "0.3" wgpu = "0.16" winit = "0.28" -parking_lot = "0.12.1" +parking_lot = { version = "0.12.1", features=["send_guard"] } image = "0.24" egui = { git = "https://github.com/emilk/egui" } egui-wgpu = { git = "https://github.com/emilk/egui", features = ["winit"] } diff --git a/examples/wgpu_room/src/app.rs b/examples/wgpu_room/src/app.rs index 026cf37..1cd36b8 100644 --- a/examples/wgpu_room/src/app.rs +++ b/examples/wgpu_room/src/app.rs @@ -106,7 +106,7 @@ pub fn run(rt: tokio::runtime::Runtime) { AsyncCmd::RoomConnect { url, token } => { state.connecting.store(true, Ordering::SeqCst); - let res = Room::connect(&url, &token).await; + let res = Room::connect(&url, &token, RoomOptions::default()).await; if let Ok((room, room_events)) = res { let room = Arc::new(room); let (close_tx, close_rx) = oneshot::channel(); diff --git a/examples/wgpu_room/src/logo_track.rs b/examples/wgpu_room/src/logo_track.rs index 9be3d59..f53b198 100644 --- a/examples/wgpu_room/src/logo_track.rs +++ b/examples/wgpu_room/src/logo_track.rs @@ -1,7 +1,8 @@ use image::ImageFormat; use image::RgbaImage; -use livekit::options::{TrackPublishOptions, VideoCaptureOptions}; +use livekit::options::TrackPublishOptions; use livekit::prelude::*; +use livekit::webrtc::video_source::RtcVideoSource; use livekit::webrtc::{ native::yuv_helper, video_frame::native::I420BufferExt, @@ -61,8 +62,7 @@ impl LogoTrack { let (close_tx, close_rx) = oneshot::channel(); let track = LocalVideoTrack::create_video_track( "livekit_logo", - VideoCaptureOptions::default(), - self.rtc_source.clone(), + RtcVideoSource::Native(self.rtc_source.clone()), ); let task = tokio::spawn(Self::track_task(close_rx, self.rtc_source.clone())); @@ -118,7 +118,7 @@ impl LogoTrack { video_frame: Arc::new(Mutex::new(VideoFrame { rotation: VideoRotation::VideoRotation0, buffer: I420Buffer::new(FB_WIDTH as u32, FB_HEIGHT as u32), - timestamp: 0, + timestamp_us: 0, })), pos: (0, 0), direction: (1, 1), diff --git a/examples/wgpu_room/src/sine_track.rs b/examples/wgpu_room/src/sine_track.rs index 86288ca..e0fd5bf 100644 --- a/examples/wgpu_room/src/sine_track.rs +++ b/examples/wgpu_room/src/sine_track.rs @@ -1,5 +1,6 @@ -use livekit::options::{AudioCaptureOptions, TrackPublishOptions}; +use livekit::options::TrackPublishOptions; use livekit::webrtc::audio_frame::AudioFrame; +use livekit::webrtc::audio_source::RtcAudioSource; use livekit::{prelude::*, webrtc::audio_source::native::NativeAudioSource}; use std::sync::Arc; use std::time::Duration; @@ -54,12 +55,7 @@ impl SineTrack { let (close_tx, close_rx) = oneshot::channel(); let track = LocalAudioTrack::create_audio_track( "sine_wave", - AudioCaptureOptions { - auto_gain_control: false, - echo_cancellation: false, - noise_suppression: false, - }, - self.rtc_source.clone(), + RtcAudioSource::Native(self.rtc_source.clone()), ); let task = tokio::spawn(Self::track_task(close_rx, self.rtc_source.clone())); diff --git a/livekit-api/src/services/egress.rs b/livekit-api/src/services/egress.rs index 137918f..eef837c 100644 --- a/livekit-api/src/services/egress.rs +++ b/livekit-api/src/services/egress.rs @@ -128,7 +128,8 @@ impl EgressClient { file_outputs, stream_outputs, segment_outputs, - output: None, // Deprecated + output: None, // Deprecated + await_start_signal: false, // TODO Expose }, self.base.auth_header(VideoGrants { room_record: true, diff --git a/livekit-api/src/services/ingress.rs b/livekit-api/src/services/ingress.rs index fd94300..4463757 100644 --- a/livekit-api/src/services/ingress.rs +++ b/livekit-api/src/services/ingress.rs @@ -57,6 +57,7 @@ impl IngressClient { participant_name: options.participant_name, audio: Some(options.audio), video: Some(options.video), + bypass_transcoding: false, // TODO Expose }, self.base.auth_header(VideoGrants { ingress_admin: true, @@ -84,6 +85,7 @@ impl IngressClient { participant_name: options.participant_name, audio: Some(options.audio), video: Some(options.video), + bypass_transcoding: None, // TODO Expose }, self.base.auth_header(VideoGrants { ingress_admin: true, @@ -104,7 +106,7 @@ impl IngressClient { SVC, "ListIngress", proto::ListIngressRequest { - room_name: match filter{ + room_name: match filter { IngressListFilter::All => Default::default(), IngressListFilter::Room(room) => room, }, diff --git a/livekit-ffi/Cargo.toml b/livekit-ffi/Cargo.toml index 380adcb..21f76d9 100644 --- a/livekit-ffi/Cargo.toml +++ b/livekit-ffi/Cargo.toml @@ -18,12 +18,12 @@ __rustls-tls = ["livekit/__rustls-tls"] livekit = { path = "../livekit", version = "0.1.1" } livekit-protocol = { path = "../livekit-protocol", version = "0.1.0" } tokio = { version = "1.0", features = ["full"] } -parking_lot = { version = "0.12.1", features = ["send_guard"] } +futures-util = { version = "0.3", default-features = false, features = ["sink"] } +parking_lot = { version = "0.12.1", features=["send_guard"] } prost = "0.11.0" prost-types = "0.11.1" lazy_static = "1.4.0" thiserror = "1.0.38" -futures-util = "0.3.23" log = "0.4.17" dashmap = "5.4.0" env_logger = "0.10.0" diff --git a/livekit-ffi/protocol/audio_frame.proto b/livekit-ffi/protocol/audio_frame.proto index f84b44f..6c09c71 100644 --- a/livekit-ffi/protocol/audio_frame.proto +++ b/livekit-ffi/protocol/audio_frame.proto @@ -18,15 +18,16 @@ message AllocAudioBufferResponse { AudioFrameBufferInfo buffer = 1; } // Create a new AudioStream // AudioStream is used to receive audio frames from a track message NewAudioStreamRequest { - FfiHandleId room_handle = 1; - string participant_sid = 2; - string track_sid = 3; - AudioStreamType type = 4; + FfiHandleId track_handle = 1; + AudioStreamType type = 2; } message NewAudioStreamResponse { AudioStreamInfo stream = 1; } // Create a new AudioSource -message NewAudioSourceRequest { AudioSourceType type = 1; } +message NewAudioSourceRequest { + AudioSourceType type = 1; + optional AudioSourceOptions options = 2; +} message NewAudioSourceResponse { AudioSourceInfo source = 1; } // Push a frame to an AudioSource @@ -78,7 +79,6 @@ enum AudioStreamType { message AudioStreamInfo { FfiHandleId handle = 1; AudioStreamType type = 2; - string track_sid = 3; } message AudioStreamEvent { @@ -94,6 +94,12 @@ message AudioFrameReceived { /// AudioSource /// /// +message AudioSourceOptions { + bool echo_cancellation = 1; + bool noise_suppression = 2; + bool auto_gain_control = 3; +} + enum AudioSourceType { AUDIO_SOURCE_NATIVE = 0; } diff --git a/livekit-ffi/protocol/ffi.proto b/livekit-ffi/protocol/ffi.proto index 9779c94..027eed9 100644 --- a/livekit-ffi/protocol/ffi.proto +++ b/livekit-ffi/protocol/ffi.proto @@ -22,26 +22,27 @@ message FfiRequest { DisconnectRequest disconnect = 4; PublishTrackRequest publish_track = 5; UnpublishTrackRequest unpublish_track = 6; + PublishDataRequest publish_data = 7; // Track - CreateVideoTrackRequest create_video_track = 7; - CreateAudioTrackRequest create_audio_track = 8; + CreateVideoTrackRequest create_video_track = 8; + CreateAudioTrackRequest create_audio_track = 9; // Video - AllocVideoBufferRequest alloc_video_buffer = 9; - NewVideoStreamRequest new_video_stream = 10; - NewVideoSourceRequest new_video_source = 11; - CaptureVideoFrameRequest capture_video_frame = 12; - ToI420Request to_i420 = 13; - ToArgbRequest to_argb = 14; + AllocVideoBufferRequest alloc_video_buffer = 10; + NewVideoStreamRequest new_video_stream = 11; + NewVideoSourceRequest new_video_source = 12; + CaptureVideoFrameRequest capture_video_frame = 13; + ToI420Request to_i420 = 14; + ToArgbRequest to_argb = 15; // Audio - AllocAudioBufferRequest alloc_audio_buffer = 15; - NewAudioStreamRequest new_audio_stream = 16; - NewAudioSourceRequest new_audio_source = 17; - CaptureAudioFrameRequest capture_audio_frame = 18; - NewAudioResamplerRequest new_audio_resampler = 19; - RemixAndResampleRequest remix_and_resample = 20; + AllocAudioBufferRequest alloc_audio_buffer = 16; + NewAudioStreamRequest new_audio_stream = 17; + NewAudioSourceRequest new_audio_source = 18; + CaptureAudioFrameRequest capture_audio_frame = 19; + NewAudioResamplerRequest new_audio_resampler = 20; + RemixAndResampleRequest remix_and_resample = 21; } } @@ -56,26 +57,27 @@ message FfiResponse { DisconnectResponse disconnect = 4; PublishTrackResponse publish_track = 5; UnpublishTrackResponse unpublish_track = 6; + PublishDataResponse publish_data = 7; // Track - CreateVideoTrackResponse create_video_track = 7; - CreateAudioTrackResponse create_audio_track = 8; + CreateVideoTrackResponse create_video_track = 8; + CreateAudioTrackResponse create_audio_track = 9; // Video - AllocVideoBufferResponse alloc_video_buffer = 9; - NewVideoStreamResponse new_video_stream = 10; - NewVideoSourceResponse new_video_source = 11; - CaptureVideoFrameResponse capture_video_frame = 12; - ToI420Response to_i420 = 13; - ToArgbResponse to_argb = 14; + AllocVideoBufferResponse alloc_video_buffer = 10; + NewVideoStreamResponse new_video_stream = 11; + NewVideoSourceResponse new_video_source = 12; + CaptureVideoFrameResponse capture_video_frame = 13; + ToI420Response to_i420 = 14; + ToArgbResponse to_argb = 15; // Audio - AllocAudioBufferResponse alloc_audio_buffer = 15; - NewAudioStreamResponse new_audio_stream = 16; - NewAudioSourceResponse new_audio_source = 17; - CaptureAudioFrameResponse capture_audio_frame = 18; - NewAudioResamplerResponse new_audio_resampler = 19; - RemixAndResampleResponse remix_and_resample = 20; + AllocAudioBufferResponse alloc_audio_buffer = 16; + NewAudioStreamResponse new_audio_stream = 17; + NewAudioSourceResponse new_audio_source = 18; + CaptureAudioFrameResponse capture_audio_frame = 19; + NewAudioResamplerResponse new_audio_resampler = 20; + RemixAndResampleResponse remix_and_resample = 21; } } @@ -87,8 +89,10 @@ message FfiEvent { VideoStreamEvent video_stream_event = 4; AudioStreamEvent audio_stream_event = 5; ConnectCallback connect = 6; - DisposeCallback dispose = 7; - PublishTrackCallback publish_track = 8; + DisconnectCallback disconnect = 7; + DisposeCallback dispose = 8; + PublishTrackCallback publish_track = 9; + PublishDataCallback publish_data = 10; } } diff --git a/livekit-ffi/protocol/handle.proto b/livekit-ffi/protocol/handle.proto index e0bb30a..16cb94e 100644 --- a/livekit-ffi/protocol/handle.proto +++ b/livekit-ffi/protocol/handle.proto @@ -6,7 +6,10 @@ option csharp_namespace = "LiveKit.Proto"; /// # Safety /// The foreign language is responsable for disposing handles /// Forgetting to dispose the handle may lead to memory leaks -/// Messages in this file can contain an FFIHandle +/// +/// A handle means that the foreign language may still use the corresponding object +/// Dropping a handle doesn't necessarily mean that the object is destroyed if it is still used +/// on the FfiServer (Rust) message FfiHandleId { uint64 id = 1; } diff --git a/livekit-ffi/protocol/participant.proto b/livekit-ffi/protocol/participant.proto index 9a0b8f2..08391ee 100644 --- a/livekit-ffi/protocol/participant.proto +++ b/livekit-ffi/protocol/participant.proto @@ -6,6 +6,7 @@ option csharp_namespace = "LiveKit.Proto"; //import "handle.proto"; import "track.proto"; +// Seems like we don't need a FfiHandle for participants (atm at least) message ParticipantInfo { string sid = 1; string name = 2; diff --git a/livekit-ffi/protocol/room.proto b/livekit-ffi/protocol/room.proto index 00722a3..0bb1155 100644 --- a/livekit-ffi/protocol/room.proto +++ b/livekit-ffi/protocol/room.proto @@ -26,7 +26,7 @@ message ConnectCallback { // Disconnect from the a room message DisconnectRequest { FfiHandleId room_handle = 1; } message DisconnectResponse { FfiAsyncId async_id = 1; } -message DisconnectCallback { } +message DisconnectCallback { FfiAsyncId async_id = 1; } // Publish a track to the room message PublishTrackRequest { @@ -53,9 +53,27 @@ message UnpublishTrackResponse { FfiAsyncId async_id = 1; } message UnpublishTrackCallback { - optional string error = 1; + FfiAsyncId async_id = 1; + optional string error = 2; } +// Publish data to other participants +message PublishDataRequest { + FfiHandleId room_handle = 1; + uint64 data_ptr = 2; + uint64 data_size = 3; + DataPacketKind kind = 4; + repeated string destination_sids = 5; // destination +} +message PublishDataResponse { + FfiAsyncId async_id = 1; +} +message PublishDataCallback { + FfiAsyncId async_id = 1; + optional string error = 2; +} + + /// /// Options /// @@ -77,13 +95,13 @@ message TrackPublishOptions { bool dtx = 4; bool red = 5; bool simulcast = 6; - string name = 7; - TrackSource source = 8; + TrackSource source = 7; } message RoomOptions { bool auto_subscribe = 1; bool adaptive_stream = 2; + bool dynacast = 3; } /// @@ -104,7 +122,7 @@ enum ConnectionState { } enum DataPacketKind { - KIND_UNRELIABLE = 0; + KIND_LOSSY = 0; KIND_RELIABLE = 1; } @@ -141,7 +159,7 @@ message RoomInfo { message DataReceived { FfiHandleId handle = 1; - optional string participant_sid = 2; + optional string participant_sid = 2; // Can be empty if the data is sent a server SDK uint64 data_ptr = 3; uint64 data_size = 4; DataPacketKind kind = 5; @@ -196,4 +214,4 @@ message ConnectionStateChanged { ConnectionState state = 1; } message Connected {} message Disconnected {} message Reconnecting {} -message Reconnected {} +message Reconnected {} \ No newline at end of file diff --git a/livekit-ffi/protocol/track.proto b/livekit-ffi/protocol/track.proto index 3f150e2..969dcae 100644 --- a/livekit-ffi/protocol/track.proto +++ b/livekit-ffi/protocol/track.proto @@ -7,21 +7,10 @@ import "handle.proto"; import "video_frame.proto"; import "audio_frame.proto"; -message VideoCaptureOptions { - VideoResolution resolution = 1; -} - -message AudioCaptureOptions { - bool echo_cancellation = 1; - bool noise_suppression = 2; - bool auto_gain_control = 3; -} - // Create a new VideoTrack from a VideoSource message CreateVideoTrackRequest { string name = 1; - VideoCaptureOptions options = 2; - FfiHandleId source_handle = 3; + FfiHandleId source_handle = 2; } message CreateVideoTrackResponse { TrackInfo track = 1; @@ -30,8 +19,7 @@ message CreateVideoTrackResponse { // Create a new AudioTrack from a AudioSource message CreateAudioTrackRequest { string name = 1; - AudioCaptureOptions options = 2; - FfiHandleId source_handle = 3; + FfiHandleId source_handle = 2; } message CreateAudioTrackResponse { TrackInfo track = 1; @@ -79,8 +67,7 @@ message TrackPublicationInfo { } message TrackInfo { - // Tracks created/owned by the client will have a handle - FfiHandleId opt_handle = 1; + FfiHandleId handle = 1; string sid = 2; string name = 3; TrackKind kind = 4; diff --git a/livekit-ffi/protocol/video_frame.proto b/livekit-ffi/protocol/video_frame.proto index 8af8bb2..a9fa1f8 100644 --- a/livekit-ffi/protocol/video_frame.proto +++ b/livekit-ffi/protocol/video_frame.proto @@ -16,16 +16,19 @@ message AllocVideoBufferResponse { VideoFrameBufferInfo buffer = 1; } // Create a new VideoStream // VideoStream is used to receive video frames from a track message NewVideoStreamRequest { - FfiHandleId room_handle = 1; - string participant_sid = 2; - string track_sid = 3; - VideoStreamType type = 4; + FfiHandleId track_handle = 1; + VideoStreamType type = 2; } message NewVideoStreamResponse { VideoStreamInfo stream = 1; } // Create a new VideoSource // VideoSource is used to send video frame to a track -message NewVideoSourceRequest { VideoSourceType type = 1; } +message NewVideoSourceRequest { + VideoSourceType type = 1; + // Used to determine which encodings to use + simulcast layers + // Most of the time it corresponds to the source resolution + optional VideoSourceResolution resolution = 2; +} message NewVideoSourceResponse { VideoSourceInfo source = 1; } // Push a frame to a VideoSource @@ -110,7 +113,7 @@ message ARGBBufferInfo { } message VideoFrameInfo { - int64 timestamp = 1; + int64 timestamp_us = 1; // In microseconds VideoRotation rotation = 2; } @@ -168,7 +171,6 @@ enum VideoStreamType { message VideoStreamInfo { FfiHandleId handle = 1; VideoStreamType type = 2; - string track_sid = 3; } message VideoStreamEvent { @@ -185,6 +187,11 @@ message VideoFrameReceived { /// VideoSource /// /// +message VideoSourceResolution { + uint32 width = 1; + uint32 height = 2; +} + enum VideoSourceType { VIDEO_SOURCE_NATIVE = 0; } diff --git a/livekit-ffi/src/conversion/audio_frame.rs b/livekit-ffi/src/conversion/audio_frame.rs index 314e4d4..f571fe3 100644 --- a/livekit-ffi/src/conversion/audio_frame.rs +++ b/livekit-ffi/src/conversion/audio_frame.rs @@ -1,7 +1,18 @@ use crate::server::audio_frame::{FfiAudioSource, FfiAudioSream}; use crate::{proto, FfiHandleId}; +use livekit::webrtc::audio_source::AudioSourceOptions; use livekit::webrtc::prelude::*; +impl From for AudioSourceOptions { + fn from(opts: proto::AudioSourceOptions) -> Self { + Self { + echo_cancellation: opts.echo_cancellation, + auto_gain_control: opts.auto_gain_control, + noise_suppression: opts.noise_suppression, + } + } +} + impl proto::AudioFrameBufferInfo { pub fn from(handle_id: FfiHandleId, buffer: &AudioFrame) -> Self { Self { @@ -20,7 +31,6 @@ impl From<&FfiAudioSream> for proto::AudioStreamInfo { handle: Some(proto::FfiHandleId { id: stream.handle_id() as u64, }), - track_sid: stream.track_sid().clone().into(), r#type: stream.stream_type() as i32, } } diff --git a/livekit-ffi/src/conversion/room.rs b/livekit-ffi/src/conversion/room.rs index 17975c0..4dd6105 100644 --- a/livekit-ffi/src/conversion/room.rs +++ b/livekit-ffi/src/conversion/room.rs @@ -1,67 +1,23 @@ -use crate::{proto, FfiHandleId, INVALID_HANDLE}; +use crate::{proto, FfiHandleId}; use livekit::options::{AudioEncoding, TrackPublishOptions, VideoEncoding}; use livekit::prelude::*; -impl proto::RoomEvent { - pub fn from(room_handle: FfiHandleId, event: RoomEvent) -> Option { - let message = match event { - RoomEvent::ParticipantConnected(participant) => Some( - proto::room_event::Message::ParticipantConnected(proto::ParticipantConnected { - info: Some((&participant).into()), - }), - ), - RoomEvent::ParticipantDisconnected(participant) => { - Some(proto::room_event::Message::ParticipantDisconnected( - proto::ParticipantDisconnected { - info: Some((&participant).into()), - }, - )) - } - RoomEvent::TrackPublished { - publication, - participant, - } => Some(proto::room_event::Message::TrackPublished( - proto::TrackPublished { - participant_sid: participant.sid().to_string(), - publication: Some((&publication).into()), - }, - )), - RoomEvent::TrackUnpublished { - publication, - participant, - } => Some(proto::room_event::Message::TrackUnpublished( - proto::TrackUnpublished { - participant_sid: participant.sid().to_string(), - publication_sid: publication.sid().into(), - }, - )), - RoomEvent::TrackSubscribed { - track, - publication: _, - participant, - } => Some(proto::room_event::Message::TrackSubscribed( - proto::TrackSubscribed { - participant_sid: participant.sid().to_string(), - track: Some(proto::TrackInfo::from_remote_track(INVALID_HANDLE, &track)), - }, - )), - RoomEvent::TrackUnsubscribed { - track, - publication: _, - participant, - } => Some(proto::room_event::Message::TrackUnsubscribed( - proto::TrackUnsubscribed { - participant_sid: participant.sid().to_string(), - track_sid: track.sid().to_string(), - }, - )), - _ => None, - }; +impl From for RoomOptions { + fn from(value: proto::RoomOptions) -> Self { + Self { + adaptive_stream: value.adaptive_stream, + auto_subscribe: value.auto_subscribe, + dynacast: value.dynacast, + } + } +} - message.map(|message| proto::RoomEvent { - room_handle: Some(room_handle.into()), - message: Some(message), - }) +impl From for DataPacketKind { + fn from(value: proto::DataPacketKind) -> Self { + match value { + proto::DataPacketKind::KindReliable => Self::Reliable, + proto::DataPacketKind::KindLossy => Self::Lossy, + } } } @@ -93,7 +49,6 @@ impl From for TrackPublishOptions { dtx: opts.dtx, red: opts.red, simulcast: opts.simulcast, - name: opts.name, source: proto::TrackSource::from_i32(opts.source).unwrap().into(), } } diff --git a/livekit-ffi/src/conversion/track.rs b/livekit-ffi/src/conversion/track.rs index 1c81af3..274d715 100644 --- a/livekit-ffi/src/conversion/track.rs +++ b/livekit-ffi/src/conversion/track.rs @@ -1,25 +1,6 @@ use crate::{proto, FfiHandleId}; -use livekit::options::{AudioCaptureOptions, VideoCaptureOptions}; use livekit::prelude::*; -impl From for VideoCaptureOptions { - fn from(opts: proto::VideoCaptureOptions) -> Self { - Self { - resolution: opts.resolution.unwrap_or_default().into(), - } - } -} - -impl From for AudioCaptureOptions { - fn from(opts: proto::AudioCaptureOptions) -> Self { - Self { - echo_cancellation: opts.echo_cancellation, - auto_gain_control: opts.auto_gain_control, - noise_suppression: opts.noise_suppression, - } - } -} - impl From for proto::TrackSource { fn from(source: TrackSource) -> proto::TrackSource { match source { @@ -63,7 +44,7 @@ macro_rules! impl_track_into { #[allow(dead_code)] pub fn $fnc(handle_id: FfiHandleId, track: $t) -> Self { Self { - opt_handle: Some(handle_id.into()), + handle: Some(handle_id.into()), name: track.name(), stream_state: proto::StreamState::from(track.stream_state()).into(), sid: track.sid().to_string(), diff --git a/livekit-ffi/src/conversion/video_frame.rs b/livekit-ffi/src/conversion/video_frame.rs index 21ceb6c..c64eeaf 100644 --- a/livekit-ffi/src/conversion/video_frame.rs +++ b/livekit-ffi/src/conversion/video_frame.rs @@ -4,6 +4,16 @@ use crate::FfiHandleId; use livekit::options::{VideoCodec, VideoResolution}; use livekit::webrtc::prelude::*; use livekit::webrtc::video_frame; +use livekit::webrtc::video_source::VideoResolution as VideoSourceResolution; + +impl From for VideoSourceResolution { + fn from(res: proto::VideoSourceResolution) -> Self { + Self { + width: res.width, + height: res.height, + } + } +} macro_rules! impl_yuv_into { (@fields, $buffer:ident, $data_y:ident, $data_u:ident, $data_v: ident) => { @@ -72,7 +82,7 @@ impl proto::VideoFrameInfo { T: AsRef, { Self { - timestamp: frame.timestamp, + timestamp_us: frame.timestamp_us, rotation: proto::VideoRotation::from(frame.rotation).into(), } } @@ -236,7 +246,6 @@ impl From<&FfiVideoStream> for proto::VideoStreamInfo { handle: Some(proto::FfiHandleId { id: stream.handle_id() as u64, }), - track_sid: stream.track_sid().clone().into(), r#type: stream.stream_type() as i32, } } diff --git a/livekit-ffi/src/server/audio_frame.rs b/livekit-ffi/src/server/audio_frame.rs index 93ea001..eb8a32a 100644 --- a/livekit-ffi/src/server/audio_frame.rs +++ b/livekit-ffi/src/server/audio_frame.rs @@ -2,11 +2,9 @@ use crate::{proto, server, FfiError, FfiHandleId, FfiResult}; use futures_util::StreamExt; use livekit::prelude::*; use livekit::webrtc::audio_frame::AudioFrame; -use livekit::webrtc::audio_source::native::NativeAudioSource; use livekit::webrtc::audio_stream::native::NativeAudioStream; use livekit::webrtc::prelude::*; use log::warn; -use server::utils; use tokio::sync::oneshot; // ===== FFIAudioStream ===== @@ -14,7 +12,6 @@ use tokio::sync::oneshot; pub struct FfiAudioSream { handle_id: FfiHandleId, stream_type: proto::AudioStreamType, - track_sid: TrackSid, #[allow(dead_code)] close_tx: oneshot::Sender<()>, // Close the stream on drop @@ -35,42 +32,45 @@ impl FfiAudioSream { ) -> FfiResult { let (close_tx, close_rx) = oneshot::channel(); let stream_type = proto::AudioStreamType::from_i32(new_stream.r#type).unwrap(); - let track_sid: TrackSid = new_stream.track_sid.into(); - let room_handle = new_stream - .room_handle - .ok_or(FfiError::InvalidRequest("room_handle is empty"))? + let handle_id = new_stream + .track_handle + .ok_or(FfiError::InvalidRequest("track_handle is empty"))? .id as FfiHandleId; - let track = utils::find_remote_track( - server, - &track_sid, - &new_stream.participant_sid.into(), - room_handle, - )? - .rtc_track(); + let track = server + .ffi_handles() + .get(&handle_id) + .ok_or(FfiError::InvalidRequest("track not found"))?; - let MediaStreamTrack::Audio(track) = track else { + let track = track + .downcast_ref::() + .ok_or(FfiError::InvalidRequest("handle is not a Track"))?; + + let rtc_track = track.rtc_track(); + + let MediaStreamTrack::Audio(rtc_track) = rtc_track else { return Err(FfiError::InvalidRequest("not an audio track")); }; let audio_stream = match stream_type { + #[cfg(not(target_arch = "wasm32"))] proto::AudioStreamType::AudioStreamNative => { let audio_stream = Self { handle_id: server.next_id(), stream_type, close_tx, - track_sid, }; + + let native_stream = NativeAudioStream::new(rtc_track); server.async_runtime.spawn(Self::native_audio_stream_task( server, audio_stream.handle_id, - NativeAudioStream::new(track), + native_stream, close_rx, )); Ok::(audio_stream) } - // TODO(theomonnom): Support other stream types _ => return Err(FfiError::InvalidRequest("unsupported audio stream type")), }?; @@ -91,10 +91,6 @@ impl FfiAudioSream { self.stream_type } - pub fn track_sid(&self) -> &TrackSid { - &self.track_sid - } - async fn native_audio_stream_task( server: &'static server::FfiServer, stream_handle_id: FfiHandleId, @@ -139,12 +135,7 @@ impl FfiAudioSream { pub struct FfiAudioSource { handle_id: FfiHandleId, source_type: proto::AudioSourceType, - source: AudioSource, -} - -#[derive(Clone)] -pub enum AudioSource { - Native(NativeAudioSource), + source: RtcAudioSource, } impl FfiAudioSource { @@ -153,12 +144,17 @@ impl FfiAudioSource { new_source: proto::NewAudioSourceRequest, ) -> FfiResult { let source_type = proto::AudioSourceType::from_i32(new_source.r#type).unwrap(); + #[allow(unreachable_patterns)] let source_inner = match source_type { + #[cfg(not(target_arch = "wasm32"))] proto::AudioSourceType::AudioSourceNative => { - let audio_source = NativeAudioSource::default(); - Ok::(AudioSource::Native(audio_source)) - } //_ => return Err(FfiError::InvalidRequest("unsupported audio source type")), - }?; + use livekit::webrtc::audio_source::native::NativeAudioSource; + let audio_source = + NativeAudioSource::new(new_source.options.map(Into::into).unwrap_or_default()); + RtcAudioSource::Native(audio_source) + } + _ => return Err(FfiError::InvalidRequest("unsupported audio source type")), + }; let audio_source = Self { handle_id: server.next_id(), @@ -180,7 +176,8 @@ impl FfiAudioSource { capture: proto::CaptureAudioFrameRequest, ) -> FfiResult<()> { match self.source { - AudioSource::Native(ref source) => { + #[cfg(not(target_arch = "wasm32"))] + RtcAudioSource::Native(ref source) => { let buffer_handle = capture .buffer_handle .ok_or(FfiError::InvalidRequest("buffer_handle is empty"))? @@ -197,6 +194,7 @@ impl FfiAudioSource { source.capture_frame(frame); } + _ => {} } Ok(()) @@ -210,7 +208,7 @@ impl FfiAudioSource { self.source_type } - pub fn inner_source(&self) -> &AudioSource { + pub fn inner_source(&self) -> &RtcAudioSource { &self.source } } diff --git a/livekit-ffi/src/server/mod.rs b/livekit-ffi/src/server/mod.rs index c932741..f86aac8 100644 --- a/livekit-ffi/src/server/mod.rs +++ b/livekit-ffi/src/server/mod.rs @@ -15,7 +15,6 @@ use std::sync::Arc; pub mod audio_frame; pub mod room; -pub mod utils; pub mod video_frame; #[cfg(test)] @@ -31,7 +30,7 @@ pub struct FfiConfig { pub struct FfiServer { rooms: Mutex>, - /// Store all FFI handles inside an HashMap, if this isn't efficient enough + /// Store all Ffi handles inside an HashMap, if this isn't efficient enough /// We can still use Box::into_raw & Box::from_raw in the future (but keep it safe for now) ffi_handles: DashMap, next_id: AtomicUsize, @@ -116,11 +115,9 @@ impl FfiServer { } // # SAFETY: The foreign language is responsible for ensuring that the callback function is valid - unsafe { - *self.config.lock() = Some(FfiConfig { - callback_fn: std::mem::transmute(init.event_callback_ptr), - }); - } + *self.config.lock() = Some(FfiConfig { + callback_fn: unsafe { std::mem::transmute(init.event_callback_ptr) }, + }); Ok(proto::InitializeResponse::default()) } @@ -191,9 +188,37 @@ impl FfiServer { fn on_disconnect( &'static self, - _disconnect: proto::DisconnectRequest, + disconnect: proto::DisconnectRequest, ) -> FfiResult { - Ok(proto::DisconnectResponse::default()) + let async_id = self.next_id() as FfiAsyncId; + let room_handle = disconnect + .room_handle + .as_ref() + .ok_or(FfiError::InvalidRequest("room_handle is empty"))? + .id as FfiHandleId; + + let ffi_room = self + .ffi_handles + .remove(&room_handle) + .ok_or(FfiError::InvalidRequest("room not found"))? + .1; + + let ffi_room = ffi_room + .downcast::() + .map_err(|_| FfiError::InvalidRequest("room is not a FfiRoom"))?; + + self.async_runtime.spawn(async move { + ffi_room.close().await; + let _ = self.send_event(proto::ffi_event::Message::Disconnect( + proto::DisconnectCallback { + async_id: Some(async_id.into()), + }, + )); + }); + + Ok(proto::DisconnectResponse { + async_id: Some(async_id.into()), + }) } fn on_publish_track( @@ -230,14 +255,17 @@ impl FfiServer { .ok_or(FfiError::InvalidRequest("track not found"))?; let track = track - .downcast_ref::() - .ok_or(FfiError::InvalidRequest("track is not a LocalTrack"))?; + .downcast_ref::() + .ok_or(FfiError::InvalidRequest("track is not a Track"))?; + + let local_track = LocalTrack::try_from(track.clone()) + .map_err(|_| FfiError::InvalidRequest("track is not a LocalTrack"))?; let publication = ffi_room .room() .local_participant() .publish_track( - track.clone(), + local_track, publish.options.map(Into::into).unwrap_or_default(), ) .await?; @@ -269,6 +297,29 @@ impl FfiServer { Ok(proto::UnpublishTrackResponse::default()) } + fn on_publish_data( + &'static self, + publish: proto::PublishDataRequest, + ) -> FfiResult { + let room_handle = publish + .room_handle + .as_ref() + .ok_or(FfiError::InvalidRequest("room_handle is empty"))? + .id as FfiHandleId; + + let ffi_room = self + .ffi_handles + .get(&room_handle) + .ok_or(FfiError::InvalidRequest("room not found"))?; + + let ffi_room = ffi_room + .downcast_ref::() + .ok_or(FfiError::InvalidRequest("room is not a FfiRoom"))?; + + // Push the data to an async queue (avoid blocking and keep the order) + ffi_room.publish_data(self, publish) + } + // Track fn on_create_video_track( &'static self, @@ -276,7 +327,6 @@ impl FfiServer { ) -> FfiResult { let handle_id = create .source_handle - .as_ref() .ok_or(FfiError::InvalidRequest("source_handle is empty"))? .id as FfiHandleId; @@ -290,19 +340,13 @@ impl FfiServer { .ok_or(FfiError::InvalidRequest("handle is not a video source"))?; let source = source.inner_source().clone(); - let video_track = match source { - video_frame::VideoSource::Native(native_source) => LocalVideoTrack::create_video_track( - &create.name, - create.options.unwrap_or_default().into(), - native_source, - ), - }; + let video_track = LocalVideoTrack::create_video_track(&create.name, source); let handle_id = self.next_id() as FfiHandleId; let track_info = proto::TrackInfo::from_local_video_track(handle_id, &video_track); self.ffi_handles - .insert(handle_id, Box::new(LocalTrack::Video(video_track))); + .insert(handle_id, Box::new(Track::LocalVideo(video_track))); Ok(proto::CreateVideoTrackResponse { track: Some(track_info), @@ -315,7 +359,6 @@ impl FfiServer { ) -> FfiResult { let handle_id = create .source_handle - .as_ref() .ok_or(FfiError::InvalidRequest("source_handle is empty"))? .id as FfiHandleId; @@ -329,19 +372,13 @@ impl FfiServer { .ok_or(FfiError::InvalidRequest("handle is not an audio source"))?; let source = source.inner_source().clone(); - let audio_track = match source { - audio_frame::AudioSource::Native(native_source) => LocalAudioTrack::create_audio_track( - &create.name, - create.options.unwrap_or_default().into(), - native_source, - ), - }; + let audio_track = LocalAudioTrack::create_audio_track(&create.name, source); let handle_id = self.next_id() as FfiHandleId; let track_info = proto::TrackInfo::from_local_audio_track(handle_id, &audio_track); self.ffi_handles - .insert(handle_id, Box::new(LocalTrack::Audio(audio_track))); + .insert(handle_id, Box::new(Track::LocalAudio(audio_track))); Ok(proto::CreateAudioTrackResponse { track: Some(track_info), @@ -714,6 +751,9 @@ impl FfiServer { proto::ffi_request::Message::UnpublishTrack(unpublish) => { proto::ffi_response::Message::UnpublishTrack(self.on_unpublish_track(unpublish)?) } + proto::ffi_request::Message::PublishData(publish) => { + proto::ffi_response::Message::PublishData(self.on_publish_data(publish)?) + } proto::ffi_request::Message::CreateVideoTrack(create) => { proto::ffi_response::Message::CreateVideoTrack(self.on_create_video_track(create)?) } diff --git a/livekit-ffi/src/server/room.rs b/livekit-ffi/src/server/room.rs index f4970f6..68374a7 100644 --- a/livekit-ffi/src/server/room.rs +++ b/livekit-ffi/src/server/room.rs @@ -1,14 +1,24 @@ use crate::server::FfiServer; -use crate::{proto, FfiHandleId, FfiResult}; +use crate::{proto, FfiAsyncId, FfiError, FfiHandleId, FfiResult}; use livekit::prelude::*; +use std::slice; use std::sync::Arc; -use tokio::sync::{mpsc, oneshot}; +use tokio::sync::{broadcast, mpsc}; use tokio::task::JoinHandle; +struct DataPacket { + data: Vec, + kind: DataPacketKind, + destination_sids: Vec, + async_id: FfiAsyncId, +} + pub struct FfiRoom { room: Arc, - handle: JoinHandle<()>, - close_tx: oneshot::Sender<()>, + event_handle: JoinHandle<()>, + data_handle: JoinHandle<()>, + close_tx: broadcast::Sender<()>, + data_tx: mpsc::UnboundedSender, } impl FfiRoom { @@ -16,33 +26,77 @@ impl FfiRoom { server: &'static FfiServer, connect: proto::ConnectRequest, ) -> FfiResult { - let (room, events) = Room::connect(&connect.url, &connect.token).await?; + let (room, events) = Room::connect( + &connect.url, + &connect.token, + connect.options.map(Into::into).unwrap_or_default(), + ) + .await?; let room = Arc::new(room); - let (close_tx, close_rx) = oneshot::channel(); - let next_id = server.next_id() as FfiHandleId; + let (close_tx, close_rx) = broadcast::channel(1); + let (data_tx, data_rx) = mpsc::unbounded_channel(); - let handle = + let next_id = server.next_id() as FfiHandleId; + let event_handle = server.async_runtime.spawn(room_task( + server, + room.clone(), + next_id, + events, + close_rx.resubscribe(), + )); + let data_handle = server .async_runtime - .spawn(room_task(server, room.clone(), next_id, events, close_rx)); - let room_info = proto::RoomInfo::from_room(next_id, &room); + .spawn(data_task(server, room.clone(), data_rx, close_rx)); let ffi_room = Self { room: room.clone(), - handle, + event_handle, + data_handle, close_tx, + data_tx, }; server.ffi_handles().insert(next_id, Box::new(ffi_room)); server.rooms().lock().insert(room.sid(), next_id); + let room_info = proto::RoomInfo::from_room(next_id, &room); Ok(room_info) } + pub fn publish_data( + &self, + server: &'static FfiServer, + publish: proto::PublishDataRequest, + ) -> FfiResult { + let data = unsafe { + slice::from_raw_parts(publish.data_ptr as *const u8, publish.data_size as usize) + }; + let kind = proto::DataPacketKind::from_i32(publish.kind).unwrap(); + let destination_sids: Vec = publish.destination_sids; + let async_id = server.next_id() as FfiAsyncId; + + let packet = DataPacket { + data: data.to_vec(), // Avoid copy? + kind: kind.into(), + destination_sids, + async_id, + }; + + self.data_tx + .send(packet) + .map_err(|_| FfiError::InvalidRequest("failed to send data packet"))?; + + Ok(proto::PublishDataResponse { + async_id: Some(async_id.into()), + }) + } + pub async fn close(self) { let _ = self.room.close().await; let _ = self.close_tx.send(()); - let _ = self.handle.await; + let _ = self.event_handle.await; + let _ = self.data_handle.await; } pub fn room(&self) -> &Arc { @@ -50,12 +104,41 @@ impl FfiRoom { } } +async fn data_task( + server: &'static FfiServer, + room: Arc, + mut data_rx: mpsc::UnboundedReceiver, + mut close_rx: broadcast::Receiver<()>, +) { + loop { + tokio::select! { + Some(event) = data_rx.recv() => { + let res = room.local_participant().publish_data( + event.data, + event.kind, + event.destination_sids, + ).await; + + let cb = proto::PublishDataCallback { + async_id: Some(event.async_id.into()), + error: res.err().map(|e| e.to_string()), + }; + + let _ = server.send_event(proto::ffi_event::Message::PublishData(cb)); + }, + _ = close_rx.recv() => { + break; + } + } + } +} + async fn room_task( server: &'static FfiServer, room: Arc, room_handle: FfiHandleId, mut events: mpsc::UnboundedReceiver, - mut close_rx: oneshot::Receiver<()>, + mut close_rx: broadcast::Receiver<()>, ) { server .async_runtime @@ -66,18 +149,78 @@ async fn room_task( loop { tokio::select! { Some(event) = events.recv() => { - if let Some(event) = proto::RoomEvent::from(room_handle, event.clone()) { - let _ = server.send_event(proto::ffi_event::Message::RoomEvent(event)); + let message = match event { + RoomEvent::ParticipantConnected(participant) => { + server.async_runtime.spawn(participant_task(Participant::Remote(participant.clone()))); + Some(proto::room_event::Message::ParticipantConnected( + proto::ParticipantConnected { + info: Some(proto::ParticipantInfo::from(&participant)), + } + )) + }, + RoomEvent::ParticipantDisconnected(participant) => { + Some(proto::room_event::Message::ParticipantDisconnected( + proto::ParticipantDisconnected { + info: Some(proto::ParticipantInfo::from(&participant)), + }, + )) + } + RoomEvent::TrackPublished { + publication, + participant, + } => Some(proto::room_event::Message::TrackPublished( + proto::TrackPublished { + participant_sid: participant.sid().to_string(), + publication: Some(proto::TrackPublicationInfo::from(&publication)) + }, + )), + RoomEvent::TrackUnpublished { + publication, + participant, + } => Some(proto::room_event::Message::TrackUnpublished( + proto::TrackUnpublished { + participant_sid: participant.sid().to_string(), + publication_sid: publication.sid().into(), + }, + )), + RoomEvent::TrackSubscribed { + track, + publication: _, + participant, + } => { + let handle_id = server.next_id() as FfiHandleId; + let track_info = proto::TrackInfo::from_remote_track(handle_id, &track); + server.ffi_handles().insert(handle_id, Box::new(Track::from(track))); + + Some(proto::room_event::Message::TrackSubscribed( + proto::TrackSubscribed { + participant_sid: participant.sid().to_string(), + track: Some(track_info), + }, + )) + }, + RoomEvent::TrackUnsubscribed { + track, + publication: _, + participant, + } => Some(proto::room_event::Message::TrackUnsubscribed( + proto::TrackUnsubscribed { + participant_sid: participant.sid().to_string(), + track_sid: track.sid().to_string(), + }, + )), + _ => None + }; + + if message.is_some() { + let _ = server.send_event(proto::ffi_event::Message::RoomEvent(proto::RoomEvent{ + room_handle: Some(room_handle.into()), + message + })); } - match event { - RoomEvent::ParticipantConnected(p) => { - server.async_runtime.spawn(participant_task(Participant::Remote(p))); - } - _ => {} - } }, - _ = &mut close_rx => { + _ = close_rx.recv() => { break; } }; diff --git a/livekit-ffi/src/server/tests.rs b/livekit-ffi/src/server/tests.rs index 9994743..a5dd9a9 100644 --- a/livekit-ffi/src/server/tests.rs +++ b/livekit-ffi/src/server/tests.rs @@ -212,18 +212,22 @@ fn publish_video_track() { client::FfiHandle(connect.room.unwrap().handle.unwrap().id as FfiHandleId); // Create a new VideoSource + const VIDEO_WIDTH: u32 = 640; + const VIDEO_HEIGHT: u32 = 480; + const VIDEO_FPS: f64 = 8.0; + let res = client.send_request(proto::FfiRequest { message: Some(proto::ffi_request::Message::NewVideoSource( proto::NewVideoSourceRequest { r#type: proto::VideoSourceType::VideoSourceNative as i32, + resolution: Some(proto::VideoSourceResolution { + width: VIDEO_WIDTH, + height: VIDEO_HEIGHT, + }), }, )), }); - const VIDEO_WIDTH: u32 = 640; - const VIDEO_HEIGHT: u32 = 480; - const VIDEO_FPS: f64 = 8.0; - let proto::ffi_response::Message::NewVideoSource(new_video_source) = res.message.unwrap() else { panic!("unexpected response"); @@ -241,13 +245,6 @@ fn publish_video_track() { source_handle: Some(proto::FfiHandleId { id: source_handle.0 as u64, }), - options: Some(proto::VideoCaptureOptions { - resolution: Some(proto::VideoResolution { - width: VIDEO_WIDTH, - height: VIDEO_HEIGHT, - frame_rate: VIDEO_FPS, - }), - }), }, )), }); @@ -258,11 +255,10 @@ fn publish_video_track() { }; let track_handle = client::FfiHandle( - create_video_track.track.unwrap().opt_handle.unwrap().id as FfiHandleId, + create_video_track.track.unwrap().handle.unwrap().id as FfiHandleId, ); let publish_options = proto::TrackPublishOptions { - name: "video_test".to_string(), video_codec: proto::VideoCodec::H264 as i32, source: proto::TrackSource::SourceCamera as i32, ..Default::default() @@ -320,7 +316,7 @@ fn publish_video_track() { id: buffer_handle.0 as u64, }), frame: Some(proto::VideoFrameInfo { - timestamp: 0, // TODO + timestamp_us: 0, rotation: proto::VideoRotation::VideoRotation0 as i32, }), }, diff --git a/livekit-ffi/src/server/utils.rs b/livekit-ffi/src/server/utils.rs deleted file mode 100644 index 12de89b..0000000 --- a/livekit-ffi/src/server/utils.rs +++ /dev/null @@ -1,32 +0,0 @@ -use crate::{server, FfiError, FfiHandleId, FfiResult}; -use livekit::prelude::*; - -pub fn find_remote_track( - server: &'static server::FfiServer, - track_sid: &TrackSid, - participant_sid: &ParticipantSid, - room_handle: FfiHandleId, -) -> FfiResult { - let ffi_room = server - .ffi_handles() - .get(&room_handle) - .ok_or(FfiError::InvalidRequest("room not found"))?; - - let ffi_room = ffi_room - .downcast_ref::() - .ok_or(FfiError::InvalidRequest("room is not ffi room"))?; - - let room = ffi_room.room(); - let participants = room.participants(); - let participant = participants - .get(participant_sid) - .ok_or(FfiError::InvalidRequest("participant not found"))?; - - let track = participant - .get_track_publication(track_sid) - .ok_or(FfiError::InvalidRequest("publication not found"))? - .track() - .ok_or(FfiError::InvalidRequest("track not found/subscribed"))?; - - Ok(track) -} diff --git a/livekit-ffi/src/server/video_frame.rs b/livekit-ffi/src/server/video_frame.rs index 0aedf7c..2861f97 100644 --- a/livekit-ffi/src/server/video_frame.rs +++ b/livekit-ffi/src/server/video_frame.rs @@ -3,10 +3,8 @@ use futures_util::StreamExt; use livekit::prelude::*; use livekit::webrtc::prelude::*; use livekit::webrtc::video_frame::{BoxVideoFrameBuffer, VideoFrame}; -use livekit::webrtc::video_source::native::NativeVideoSource; use livekit::webrtc::video_stream::native::NativeVideoStream; use log::warn; -use server::utils; use tokio::sync::oneshot; // ===== FFIVideoStream ===== @@ -14,7 +12,6 @@ use tokio::sync::oneshot; pub struct FfiVideoStream { handle_id: FfiHandleId, stream_type: proto::VideoStreamType, - track_sid: TrackSid, #[allow(dead_code)] close_tx: oneshot::Sender<()>, // Close the stream on drop @@ -35,42 +32,43 @@ impl FfiVideoStream { ) -> FfiResult { let (close_tx, close_rx) = oneshot::channel(); let stream_type = proto::VideoStreamType::from_i32(new_stream.r#type).unwrap(); - let track_sid: TrackSid = new_stream.track_sid.into(); - let room_handle = new_stream - .room_handle - .ok_or(FfiError::InvalidRequest("room_handle is empty"))? + let handle_id = new_stream + .track_handle + .ok_or(FfiError::InvalidRequest("track_handle is empty"))? .id as FfiHandleId; - let track = utils::find_remote_track( - server, - &track_sid, - &new_stream.participant_sid.into(), - room_handle, - )? - .rtc_track(); + let track = server + .ffi_handles() + .get(&handle_id) + .ok_or(FfiError::InvalidRequest("track not found"))?; - let MediaStreamTrack::Video(track) = track else { + let track = track + .downcast_ref::() + .ok_or(FfiError::InvalidRequest("handle is not a Track"))?; + + let rtc_track = track.rtc_track(); + + let MediaStreamTrack::Video(rtc_track) = rtc_track else { return Err(FfiError::InvalidRequest("not a video track")); }; let stream = match stream_type { + #[cfg(not(target_arch = "wasm32"))] proto::VideoStreamType::VideoStreamNative => { let video_stream = Self { handle_id: server.next_id(), close_tx, stream_type, - track_sid, }; server.async_runtime.spawn(Self::native_video_stream_task( server, video_stream.handle_id, - NativeVideoStream::new(track), + NativeVideoStream::new(rtc_track), close_rx, )); Ok::(video_stream) } - // TODO(theomonnom): Support other stream types _ => return Err(FfiError::InvalidRequest("unsupported video stream type")), }?; @@ -91,10 +89,6 @@ impl FfiVideoStream { self.stream_type } - pub fn track_sid(&self) -> &TrackSid { - &self.track_sid - } - async fn native_video_stream_task( server: &'static server::FfiServer, stream_handle_id: FfiHandleId, @@ -143,12 +137,7 @@ impl FfiVideoStream { pub struct FfiVideoSource { handle_id: FfiHandleId, source_type: proto::VideoSourceType, - source: VideoSource, -} - -#[derive(Clone)] -pub enum VideoSource { - Native(NativeVideoSource), + source: RtcVideoSource, } impl FfiVideoSource { @@ -157,11 +146,17 @@ impl FfiVideoSource { new_source: proto::NewVideoSourceRequest, ) -> FfiResult { let source_type = proto::VideoSourceType::from_i32(new_source.r#type).unwrap(); + #[allow(unreachable_patterns)] let source_inner = match source_type { + #[cfg(not(target_arch = "wasm32"))] proto::VideoSourceType::VideoSourceNative => { - let video_source = NativeVideoSource::default(); - VideoSource::Native(video_source) + use livekit::webrtc::video_source::native::NativeVideoSource; + let video_source = NativeVideoSource::new( + new_source.resolution.map(Into::into).unwrap_or_default(), + ); + RtcVideoSource::Native(video_source) } + _ => return Err(FfiError::InvalidRequest("unsupported video source type")), }; let video_source = Self { @@ -184,7 +179,8 @@ impl FfiVideoSource { capture: proto::CaptureVideoFrameRequest, ) -> FfiResult<()> { match self.source { - VideoSource::Native(ref source) => { + #[cfg(not(target_arch = "wasm32"))] + RtcVideoSource::Native(ref source) => { let frame_info = capture .frame .ok_or(FfiError::InvalidRequest("frame is empty"))?; @@ -206,12 +202,13 @@ impl FfiVideoSource { let rotation = proto::VideoRotation::from_i32(frame_info.rotation).unwrap(); let frame = VideoFrame { rotation: rotation.into(), - timestamp: frame_info.timestamp, + timestamp_us: frame_info.timestamp_us, buffer, }; source.capture_frame(&frame); } + _ => {} } Ok(()) } @@ -224,7 +221,7 @@ impl FfiVideoSource { self.source_type } - pub fn inner_source(&self) -> &VideoSource { + pub fn inner_source(&self) -> &RtcVideoSource { &self.source } } diff --git a/livekit-protocol/protocol b/livekit-protocol/protocol index 1e793a8..8935996 160000 --- a/livekit-protocol/protocol +++ b/livekit-protocol/protocol @@ -1 +1 @@ -Subproject commit 1e793a88707b8301286bb8306fff27e0e37d16f2 +Subproject commit 89359963fc2468a97a1c1b18612ab1582da13f60 diff --git a/livekit-webrtc/Cargo.toml b/livekit-webrtc/Cargo.toml index acdfe63..45c8706 100644 --- a/livekit-webrtc/Cargo.toml +++ b/livekit-webrtc/Cargo.toml @@ -15,7 +15,7 @@ thiserror = "1.0" [target.'cfg(not(target_arch = "wasm32"))'.dependencies] webrtc-sys = { path = "../webrtc-sys", version = "0.1.1" } lazy_static = "1.4" -parking_lot = { version = "0.12.1", features = ["send_guard"] } +parking_lot = { version = "0.12.1" } futures = { version = "0.3" } tokio = { version = "1", features = ["full"] } cxx = "1.0" diff --git a/livekit-webrtc/src/audio_source.rs b/livekit-webrtc/src/audio_source.rs index b13adcd..061088b 100644 --- a/livekit-webrtc/src/audio_source.rs +++ b/livekit-webrtc/src/audio_source.rs @@ -1,12 +1,35 @@ use crate::imp::audio_source as imp_as; +use livekit_protocol::enum_dispatch; + +#[derive(Default, Debug)] +pub struct AudioSourceOptions { + pub echo_cancellation: bool, + pub noise_suppression: bool, + pub auto_gain_control: bool, +} + +#[non_exhaustive] +#[derive(Debug, Clone)] +pub enum RtcAudioSource { + #[cfg(not(target_arch = "wasm32"))] + Native(native::NativeAudioSource), +} + +impl RtcAudioSource { + enum_dispatch!( + [Native]; + fn set_audio_options(self: &Self, options: AudioSourceOptions) -> (); + fn audio_options(self: &Self) -> AudioSourceOptions; + ); +} #[cfg(not(target_arch = "wasm32"))] pub mod native { - use super::imp_as; + use super::*; use crate::audio_frame::AudioFrame; use std::fmt::{Debug, Formatter}; - #[derive(Default, Clone)] + #[derive(Clone)] pub struct NativeAudioSource { pub(crate) handle: imp_as::NativeAudioSource, } @@ -17,9 +40,29 @@ pub mod native { } } + impl Default for NativeAudioSource { + fn default() -> Self { + Self::new(AudioSourceOptions::default()) + } + } + impl NativeAudioSource { + pub fn new(options: AudioSourceOptions) -> NativeAudioSource { + Self { + handle: imp_as::NativeAudioSource::new(options), + } + } + pub fn capture_frame(&self, frame: &AudioFrame) { self.handle.capture_frame(frame) } + + pub fn set_audio_options(&self, options: AudioSourceOptions) { + self.handle.set_audio_options(options) + } + + pub fn audio_options(&self) -> AudioSourceOptions { + self.handle.audio_options() + } } } diff --git a/livekit-webrtc/src/native/audio_source.rs b/livekit-webrtc/src/native/audio_source.rs index 3b00c64..e9d45a6 100644 --- a/livekit-webrtc/src/native/audio_source.rs +++ b/livekit-webrtc/src/native/audio_source.rs @@ -1,9 +1,29 @@ -use crate::audio_frame::AudioFrame; +use crate::{audio_frame::AudioFrame, audio_source::AudioSourceOptions}; use cxx::SharedPtr; use parking_lot::Mutex; use std::sync::Arc; use webrtc_sys::audio_track as sys_at; +impl From for AudioSourceOptions { + fn from(options: sys_at::ffi::AudioSourceOptions) -> Self { + Self { + echo_cancellation: options.echo_cancellation, + noise_suppression: options.noise_suppression, + auto_gain_control: options.auto_gain_control, + } + } +} + +impl From for sys_at::ffi::AudioSourceOptions { + fn from(options: AudioSourceOptions) -> Self { + Self { + echo_cancellation: options.echo_cancellation, + noise_suppression: options.noise_suppression, + auto_gain_control: options.auto_gain_control, + } + } +} + #[derive(Clone)] pub struct NativeAudioSource { sys_handle: SharedPtr, @@ -18,20 +38,27 @@ struct AudioSourceInner { num_channels: u32, } -impl Default for NativeAudioSource { - fn default() -> Self { +impl NativeAudioSource { + pub fn new(options: AudioSourceOptions) -> NativeAudioSource { Self { - sys_handle: sys_at::ffi::new_audio_track_source(), + sys_handle: sys_at::ffi::new_audio_track_source(options.into()), inner: Default::default(), } } -} -impl NativeAudioSource { pub fn sys_handle(&self) -> SharedPtr { self.sys_handle.clone() } + pub fn set_audio_options(&self, options: AudioSourceOptions) { + self.sys_handle + .set_audio_options(&sys_at::ffi::AudioSourceOptions::from(options)) + } + + pub fn audio_options(&self) -> AudioSourceOptions { + self.sys_handle.audio_options().into() + } + pub fn capture_frame(&self, frame: &AudioFrame) { let mut inner = self.inner.lock(); let samples_10ms = (frame.sample_rate / 100 * frame.num_channels) as usize; diff --git a/livekit-webrtc/src/native/video_source.rs b/livekit-webrtc/src/native/video_source.rs index 102b1f0..c4206c9 100644 --- a/livekit-webrtc/src/native/video_source.rs +++ b/livekit-webrtc/src/native/video_source.rs @@ -1,22 +1,42 @@ use crate::video_frame::{VideoFrame, VideoFrameBuffer}; +use crate::video_source::VideoResolution; use cxx::SharedPtr; +use std::time::{SystemTime, UNIX_EPOCH}; use webrtc_sys::video_frame as vf_sys; use webrtc_sys::video_track as vt_sys; +impl From for VideoResolution { + fn from(res: vt_sys::ffi::VideoResolution) -> Self { + Self { + width: res.width, + height: res.height, + } + } +} + +impl From for vt_sys::ffi::VideoResolution { + fn from(res: VideoResolution) -> Self { + Self { + width: res.width, + height: res.height, + } + } +} + #[derive(Clone)] pub struct NativeVideoSource { sys_handle: SharedPtr, } -impl Default for NativeVideoSource { - fn default() -> Self { +impl NativeVideoSource { + pub fn new(resolution: VideoResolution) -> NativeVideoSource { Self { - sys_handle: vt_sys::ffi::new_video_track_source(), + sys_handle: vt_sys::ffi::new_video_track_source(&vt_sys::ffi::VideoResolution::from( + resolution, + )), } } -} -impl NativeVideoSource { pub fn sys_handle(&self) -> SharedPtr { self.sys_handle.clone() } @@ -27,7 +47,20 @@ impl NativeVideoSource { builder .pin_mut() .set_video_frame_buffer(frame.buffer.as_ref().sys_handle()); + + if frame.timestamp_us == 0 { + // If the timestamp is set to 0, default to now + let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); + builder.pin_mut().set_timestamp_us(now.as_micros() as i64); + } else { + builder.pin_mut().set_timestamp_us(frame.timestamp_us); + } + self.sys_handle .on_captured_frame(&builder.pin_mut().build()); } + + pub fn video_resolution(&self) -> VideoResolution { + self.sys_handle.video_resolution().into() + } } diff --git a/livekit-webrtc/src/native/video_stream.rs b/livekit-webrtc/src/native/video_stream.rs index 52321b4..ef6dad2 100644 --- a/livekit-webrtc/src/native/video_stream.rs +++ b/livekit-webrtc/src/native/video_stream.rs @@ -67,7 +67,7 @@ impl sys_vt::VideoSink for VideoTrackObserver { fn on_frame(&self, frame: UniquePtr) { let _ = self.frame_tx.send(VideoFrame { rotation: frame.rotation().into(), - timestamp: frame.timestamp_us(), + timestamp_us: frame.timestamp_us(), buffer: new_video_frame_buffer(unsafe { frame.video_frame_buffer() }), }); } diff --git a/livekit-webrtc/src/prelude.rs b/livekit-webrtc/src/prelude.rs index 5aedb53..1e6b47b 100644 --- a/livekit-webrtc/src/prelude.rs +++ b/livekit-webrtc/src/prelude.rs @@ -1,4 +1,5 @@ pub use crate::audio_frame::AudioFrame; +pub use crate::audio_source::{AudioSourceOptions, RtcAudioSource}; pub use crate::audio_track::RtcAudioTrack; pub use crate::data_channel::{ DataBuffer, DataChannel, DataChannelError, DataChannelInit, DataState, @@ -22,5 +23,6 @@ pub use crate::video_frame::{ BoxVideoFrame, I010Buffer, I420ABuffer, I420Buffer, I422Buffer, I444Buffer, NV12Buffer, VideoFormatType, VideoFrame, VideoFrameBuffer, VideoFrameBufferType, VideoRotation, }; +pub use crate::video_source::{RtcVideoSource, VideoResolution}; pub use crate::video_track::RtcVideoTrack; pub use crate::{MediaType, RtcError, RtcErrorType}; diff --git a/livekit-webrtc/src/video_frame.rs b/livekit-webrtc/src/video_frame.rs index 389b7a9..0599a8e 100644 --- a/livekit-webrtc/src/video_frame.rs +++ b/livekit-webrtc/src/video_frame.rs @@ -43,7 +43,7 @@ where T: AsRef, { pub rotation: VideoRotation, - pub timestamp: i64, // When the frame was captured + pub timestamp_us: i64, // When the frame was captured in microseconds pub buffer: T, } diff --git a/livekit-webrtc/src/video_source.rs b/livekit-webrtc/src/video_source.rs index 9ba3f90..9b0053f 100644 --- a/livekit-webrtc/src/video_source.rs +++ b/livekit-webrtc/src/video_source.rs @@ -1,12 +1,36 @@ +use livekit_protocol::enum_dispatch; + use crate::imp::video_source as vs_imp; +#[derive(Default, Debug, Clone)] +pub struct VideoResolution { + pub width: u32, + pub height: u32, +} + +#[non_exhaustive] +#[derive(Debug, Clone)] +pub enum RtcVideoSource { + // TODO(theomonnom): Web video sources (eq. to tracks on browsers?) + #[cfg(not(target_arch = "wasm32"))] + Native(native::NativeVideoSource), +} + +// TODO(theomonnom): Support enum dispatch with conditional compilation? +impl RtcVideoSource { + enum_dispatch!( + [Native]; + pub fn video_resolution(self: &Self) -> VideoResolution; + ); +} + #[cfg(not(target_arch = "wasm32"))] pub mod native { - use super::vs_imp; + use super::*; use crate::video_frame::{VideoFrame, VideoFrameBuffer}; use std::fmt::{Debug, Formatter}; - #[derive(Default, Clone)] + #[derive(Clone)] pub struct NativeVideoSource { pub(crate) handle: vs_imp::NativeVideoSource, } @@ -17,10 +41,26 @@ pub mod native { } } + impl Default for NativeVideoSource { + fn default() -> Self { + Self::new(VideoResolution::default()) + } + } + impl NativeVideoSource { + pub fn new(resolution: VideoResolution) -> Self { + Self { + handle: vs_imp::NativeVideoSource::new(resolution), + } + } + pub fn capture_frame>(&self, frame: &VideoFrame) { self.handle.capture_frame(frame) } + + pub fn video_resolution(&self) -> VideoResolution { + self.handle.video_resolution() + } } } diff --git a/livekit/Cargo.toml b/livekit/Cargo.toml index a64dfd5..53dfab8 100644 --- a/livekit/Cargo.toml +++ b/livekit/Cargo.toml @@ -26,10 +26,9 @@ serde = { version = "1", features = ["derive"] } serde_json = "1.0" tokio-tungstenite = { version = "0.19" } tokio = { version = "1", features = ["full"] } -tokio-stream = "0.1" -parking_lot = { version = "0.12.1", features = ["send_guard"] } +parking_lot = { version = "0.12.1" } url = "2.3" -futures-util = "0.3" +futures-util = { version = "0.3", default-features = false, features = ["sink"] } thiserror = "1.0" lazy_static = "1.4" -tracing = "0.1" +log = "0.4" diff --git a/livekit/src/prelude.rs b/livekit/src/prelude.rs index 5c9a311..ff989eb 100644 --- a/livekit/src/prelude.rs +++ b/livekit/src/prelude.rs @@ -1,6 +1,8 @@ pub use crate::participant::{LocalParticipant, Participant, ParticipantEvent, RemoteParticipant}; -pub use crate::{ConnectionState, Room, RoomError, RoomEvent, RoomResult}; +pub use crate::{ + ConnectionState, DataPacketKind, Room, RoomError, RoomEvent, RoomOptions, RoomResult, +}; pub use crate::publication::{LocalTrackPublication, RemoteTrackPublication, TrackPublication}; diff --git a/livekit/src/proto.rs b/livekit/src/proto.rs index 84203d5..03544dc 100644 --- a/livekit/src/proto.rs +++ b/livekit/src/proto.rs @@ -1,4 +1,4 @@ -use crate::track; +use crate::{track, DataPacketKind}; use livekit_protocol::*; // Conversions @@ -46,3 +46,21 @@ impl From for TrackSource { } } } + +impl From for data_packet::Kind { + fn from(kind: DataPacketKind) -> Self { + match kind { + DataPacketKind::Lossy => Self::Lossy, + DataPacketKind::Reliable => Self::Reliable, + } + } +} + +impl From for DataPacketKind { + fn from(kind: data_packet::Kind) -> Self { + match kind { + data_packet::Kind::Lossy => Self::Lossy, + data_packet::Kind::Reliable => Self::Reliable, + } + } +} diff --git a/livekit/src/room/id.rs b/livekit/src/room/id.rs index cb5c3e5..97da6c5 100644 --- a/livekit/src/room/id.rs +++ b/livekit/src/room/id.rs @@ -3,16 +3,6 @@ use std::fmt; macro_rules! id_str { ($($name:ident;)*) => { $( - impl $name { - pub fn new(str: String) -> Self { - Self(str) - } - - pub fn as_str(&self) -> &str { - &self.0 - } - } - impl From for $name { fn from(str: String) -> $name { $name(str) @@ -41,16 +31,20 @@ macro_rules! id_str { } #[derive(Clone, Default, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] -pub struct ParticipantSid(String); +#[repr(transparent)] +pub struct ParticipantSid(pub String); #[derive(Clone, Default, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] -pub struct ParticipantIdentity(String); +#[repr(transparent)] +pub struct ParticipantIdentity(pub String); #[derive(Clone, Default, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] -pub struct TrackSid(String); +#[repr(transparent)] +pub struct TrackSid(pub String); #[derive(Clone, Default, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] -pub struct RoomSid(String); +#[repr(transparent)] +pub struct RoomSid(pub String); id_str! { ParticipantSid; diff --git a/livekit/src/room/mod.rs b/livekit/src/room/mod.rs index 144d0d9..dc09f4a 100644 --- a/livekit/src/room/mod.rs +++ b/livekit/src/room/mod.rs @@ -9,12 +9,10 @@ use livekit_protocol::observer::Dispatcher; use parking_lot::{Mutex, RwLock, RwLockReadGuard}; use std::collections::HashMap; use std::fmt::Debug; -use std::sync::atomic::{AtomicU8, Ordering}; use std::sync::Arc; use thiserror::Error; use tokio::sync::{mpsc, oneshot}; use tokio::task::JoinHandle; -use tracing::{error, info, instrument, trace, Level}; pub use crate::rtc_engine::SimulateScenario; @@ -82,7 +80,7 @@ pub enum RoomEvent { }, DataReceived { payload: Arc>, - kind: proto::data_packet::Kind, + kind: DataPacketKind, participant: RemoteParticipant, }, ConnectionStateChanged(ConnectionState), @@ -100,18 +98,41 @@ pub enum ConnectionState { Unknown, } +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum DataPacketKind { + Lossy, + Reliable, +} + +#[derive(Debug, Clone)] +pub struct RoomOptions { + pub auto_subscribe: bool, + pub adaptive_stream: bool, + pub dynacast: bool, +} + +impl Default for RoomOptions { + fn default() -> Self { + Self { + auto_subscribe: true, + adaptive_stream: false, + dynacast: false, + } + } +} + struct RoomHandle { session_task: JoinHandle<()>, close_emitter: oneshot::Sender<()>, } pub struct Room { - inner: Arc, + inner: Arc, handle: Mutex>, } impl Debug for Room { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { f.debug_struct("Room") .field("sid", &self.sid()) .field("name", &self.name()) @@ -124,14 +145,21 @@ impl Room { pub async fn connect( url: &str, token: &str, + options: RoomOptions, ) -> RoomResult<(Self, mpsc::UnboundedReceiver)> { - let (rtc_engine, engine_events) = RtcEngine::new(); + let (rtc_engine, engine_events) = RtcEngine::connect( + url, + token, + SignalOptions { + auto_subscribe: options.auto_subscribe, + adaptive_stream: options.adaptive_stream, + ..Default::default() + }, + ) + .await?; let rtc_engine = Arc::new(rtc_engine); - rtc_engine - .connect(url, token, SignalOptions::default()) - .await?; - let join_response = rtc_engine.join_response().unwrap(); + let join_response = rtc_engine.join_response(); let pi = join_response.participant.unwrap().clone(); let local_participant = LocalParticipant::new( rtc_engine.clone(), @@ -142,11 +170,13 @@ impl Room { ); let room_info = join_response.room.unwrap(); - let inner = Arc::new(SessionInner { - state: AtomicU8::new(ConnectionState::Disconnected as u8), - sid: Mutex::new(room_info.sid.into()), - name: Mutex::new(room_info.name), - metadata: Mutex::new(room_info.metadata), + let inner = Arc::new(RoomSession { + sid: room_info.sid.into(), + name: room_info.name, + info: RwLock::new(RoomInfo { + state: ConnectionState::Disconnected, + metadata: room_info.metadata, + }), participants: Default::default(), participants_tasks: Default::default(), active_speakers: Default::default(), @@ -196,15 +226,15 @@ impl Room { } pub fn sid(&self) -> RoomSid { - self.inner.sid.lock().clone() + self.inner.sid.clone() } pub fn name(&self) -> String { - self.inner.name.lock().clone() + self.inner.name.clone() } pub fn metadata(&self) -> String { - self.inner.metadata.lock().clone() + self.inner.info.read().metadata.clone() } pub fn local_participant(&self) -> LocalParticipant { @@ -212,7 +242,7 @@ impl Room { } pub fn connection_state(&self) -> ConnectionState { - self.inner.state.load(Ordering::Acquire).try_into().unwrap() + self.inner.info.read().state } pub fn participants(&self) -> RwLockReadGuard> { @@ -224,20 +254,24 @@ impl Room { } } -struct SessionInner { - state: AtomicU8, // ConnectionState - sid: Mutex, - name: Mutex, - metadata: Mutex, - participants: RwLock>, - participants_tasks: RwLock, oneshot::Sender<()>)>>, - active_speakers: RwLock>, - rtc_engine: Arc, - local_participant: LocalParticipant, - dispatcher: Dispatcher, +struct RoomInfo { + metadata: String, + state: ConnectionState, } -impl Debug for SessionInner { +pub(crate) struct RoomSession { + rtc_engine: Arc, + sid: RoomSid, + name: String, + info: RwLock, + dispatcher: Dispatcher, + active_speakers: RwLock>, + local_participant: LocalParticipant, + participants: RwLock>, + participants_tasks: RwLock, oneshot::Sender<()>)>>, +} + +impl Debug for RoomSession { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("SessionInner") .field("sid", &self.sid) @@ -247,8 +281,7 @@ impl Debug for SessionInner { } } -impl SessionInner { - #[instrument(level = Level::DEBUG)] +impl RoomSession { async fn room_task( self: Arc, mut engine_events: EngineEvents, @@ -259,20 +292,19 @@ impl SessionInner { res = engine_events.recv() => { if let Some(event) = res { if let Err(err) = self.on_engine_event(event).await { - error!("failed to handle engine event: {:?}", err); + log::error!("failed to handle engine event: {:?}", err); } } }, _ = &mut close_receiver => { - trace!("closing room_task"); + log::trace!("closing room_task"); break; } } } } - /// Listen to the Participant events and forward them to the Room Dispatcher - #[instrument(level = Level::DEBUG)] + /// Forward participant events to the room dispatcher async fn participant_task( self: Arc, participant: Participant, @@ -284,19 +316,18 @@ impl SessionInner { res = participant_events.recv() => { if let Some(event) = res { if let Err(err) = self.on_participant_event(&participant, event).await { - error!("failed to handle participant event for {:?}: {:?}", participant.sid(), err); + log::error!("failed to handle participant event for {:?}: {:?}", participant.sid(), err); } } }, _ = &mut close_rx => { - trace!("closing participant_task for {:?}", participant.sid()); + log::trace!("closing participant_task for {:?}", participant.sid()); break; }, } } } - #[instrument(level = Level::DEBUG)] async fn on_participant_event( self: &Arc, participant: &Participant, @@ -337,7 +368,6 @@ impl SessionInner { Ok(()) } - #[instrument(level = Level::DEBUG)] async fn on_engine_event(self: &Arc, event: EngineEvent) -> RoomResult<()> { match event { EngineEvent::ParticipantUpdate { updates } => self.handle_participant_update(updates), @@ -414,21 +444,20 @@ impl SessionInner { Ok(()) } - #[instrument(level = Level::DEBUG)] async fn close(&self) { self.rtc_engine.close().await; } /// Change the connection state and emit an event /// Does nothing if the state is already the same - #[instrument(level = Level::DEBUG)] + /// Returns true if the state changed fn update_connection_state(&self, state: ConnectionState) -> bool { - let old_state = self.state.load(Ordering::Acquire); - if old_state == state as u8 { + let mut info = self.info.write(); + if info.state == state { return false; } - self.state.store(state as u8, Ordering::Release); + info.state = state; self.dispatcher .dispatch(&RoomEvent::ConnectionStateChanged(state)); return true; @@ -437,7 +466,6 @@ impl SessionInner { /// Update the participants inside a Room. /// It'll create, update or remove a participant /// It also update the participant tracks. - #[instrument(level = Level::DEBUG)] fn handle_participant_update(self: &Arc, updates: Vec) { for pi in updates { if pi.sid == self.local_participant.sid() @@ -452,7 +480,7 @@ impl SessionInner { if let Some(remote_participant) = remote_participant { if pi.state == proto::participant_info::State::Disconnected as i32 { // Participant disconnected - info!("Participant disconnected: {}", pi.sid); + log::info!("Participant disconnected: {}", pi.sid); self.clone() .handle_participant_disconnect(remote_participant) } else { @@ -461,7 +489,7 @@ impl SessionInner { } } else { // Create a new participant - info!("Participant connected: {}", pi.sid); + log::info!("Participant connected: {}", pi.sid); let remote_participant = { let pi = pi.clone(); self.create_participant(pi.sid.into(), pi.identity.into(), pi.name, pi.metadata) @@ -478,7 +506,6 @@ impl SessionInner { /// Active speakers changed /// Update the participants & sort the active_speakers by audio_level - #[instrument(level = Level::DEBUG)] fn handle_speakers_changed(&self, speakers_info: Vec) { let mut speakers = Vec::new(); @@ -513,7 +540,6 @@ impl SessionInner { /// Handle a connection quality update /// Emit ConnectionQualityChanged event for the concerned participants - #[instrument(level = Level::DEBUG)] fn handle_connection_quality_update(&self, updates: Vec) { for update in updates { let participant = { @@ -542,7 +568,6 @@ impl SessionInner { } } - #[instrument(level = Level::DEBUG)] fn handle_restarting(self: &Arc) { // Remove existing participants/subscriptions on full reconnect for (_, participant) in self.participants.read().iter() { @@ -555,10 +580,9 @@ impl SessionInner { } } - #[instrument(level = Level::DEBUG)] fn handle_restarted(self: &Arc) { // Full reconnect succeeded! - let join_response = self.rtc_engine.join_response().unwrap(); + let join_response = self.rtc_engine.join_response(); self.update_connection_state(ConnectionState::Connected); self.dispatcher.dispatch(&RoomEvent::Reconnected); @@ -569,22 +593,19 @@ impl SessionInner { self.handle_participant_update(join_response.other_participants); + // TODO(theomonnom): Synchronize states + // TODO(theomonnom): Room info changed? // TODO(theomonnom): unpublish & republish tracks } - #[instrument(level = Level::DEBUG)] fn handle_disconnected(&self) { - if self.state.load(Ordering::Acquire) == ConnectionState::Disconnected as u8 { - return; + if self.update_connection_state(ConnectionState::Disconnected) { + self.dispatcher.dispatch(&RoomEvent::Disconnected); } - - self.update_connection_state(ConnectionState::Disconnected); - self.dispatcher.dispatch(&RoomEvent::Disconnected); } /// Create a new participant /// Also add it to the participants list - #[instrument(level = Level::DEBUG)] fn create_participant( self: &Arc, sid: ParticipantSid, @@ -592,7 +613,13 @@ impl SessionInner { name: String, metadata: String, ) -> RemoteParticipant { - let participant = RemoteParticipant::new(sid.clone(), identity, name, metadata); + let participant = RemoteParticipant::new( + self.rtc_engine.clone(), + sid.clone(), + identity, + name, + metadata, + ); // Create the participant task let (close_tx, close_rx) = oneshot::channel(); @@ -611,7 +638,6 @@ impl SessionInner { /// A participant has disconnected /// Cleanup the participant and emit an event - #[instrument(level = Level::DEBUG)] fn handle_participant_disconnect(self: Arc, remote_participant: RemoteParticipant) { tokio::spawn(async move { for (sid, _) in &*remote_participant.tracks() { @@ -619,11 +645,12 @@ impl SessionInner { } // Close the participant task - if let Some((task, close_tx)) = self + let ptask = self .participants_tasks .write() - .remove(&remote_participant.sid()) - { + .remove(&remote_participant.sid()); + + if let Some((task, close_tx)) = ptask { let _ = close_tx.send(()); let _ = task.await; } @@ -649,14 +676,3 @@ fn unpack_stream_id(stream_id: &str) -> Option<(&str, &str)> { None } } - -impl From for ConnectionState { - fn from(value: u8) -> Self { - match value { - 0 => ConnectionState::Disconnected, - 1 => ConnectionState::Connected, - 2 => ConnectionState::Reconnecting, - _ => ConnectionState::Unknown, - } - } -} diff --git a/livekit/src/room/options.rs b/livekit/src/room/options.rs index 35275cc..a3b6c58 100644 --- a/livekit/src/room/options.rs +++ b/livekit/src/room/options.rs @@ -58,36 +58,6 @@ impl AudioPreset { } } -#[derive(Debug, Clone)] -pub struct AudioCaptureOptions { - pub echo_cancellation: bool, - pub noise_suppression: bool, - pub auto_gain_control: bool, -} - -impl Default for AudioCaptureOptions { - fn default() -> Self { - Self { - echo_cancellation: true, - noise_suppression: true, - auto_gain_control: true, - } - } -} - -#[derive(Clone, Debug)] -pub struct VideoCaptureOptions { - pub resolution: VideoResolution, -} - -impl Default for VideoCaptureOptions { - fn default() -> Self { - Self { - resolution: video::H720.resolution(), - } - } -} - #[derive(Clone, Debug)] pub struct TrackPublishOptions { // If the encodings aren't set, LiveKit will compute the most appropriate ones @@ -97,7 +67,7 @@ pub struct TrackPublishOptions { pub dtx: bool, pub red: bool, pub simulcast: bool, - pub name: String, + // pub name: String, pub source: TrackSource, } @@ -110,7 +80,6 @@ impl Default for TrackPublishOptions { dtx: true, red: true, simulcast: true, - name: "unnamed track".to_owned(), source: TrackSource::Unknown, } } diff --git a/livekit/src/room/participant/local_participant.rs b/livekit/src/room/participant/local_participant.rs index 70f5fdd..1c3963a 100644 --- a/livekit/src/room/participant/local_participant.rs +++ b/livekit/src/room/participant/local_participant.rs @@ -1,10 +1,12 @@ -use super::{ConnectionQuality, ParticipantInner}; +use super::ConnectionQuality; +use super::ParticipantInternal; use crate::options; use crate::options::compute_video_encodings; use crate::options::video_layers_from_encodings; use crate::options::TrackPublishOptions; use crate::prelude::*; use crate::rtc_engine::RtcEngine; +use crate::DataPacketKind; use livekit_protocol as proto; use livekit_webrtc::rtp_parameters::RtpEncodingParameters; use parking_lot::RwLockReadGuard; @@ -12,12 +14,10 @@ use std::collections::HashMap; use std::fmt::Debug; use std::sync::Arc; use tokio::sync::mpsc; -use tracing::debug; #[derive(Clone)] pub struct LocalParticipant { - inner: Arc, - rtc_engine: Arc, + inner: Arc, } impl Debug for LocalParticipant { @@ -39,8 +39,9 @@ impl LocalParticipant { metadata: String, ) -> Self { Self { - inner: Arc::new(ParticipantInner::new(sid, identity, name, metadata)), - rtc_engine, + inner: Arc::new(ParticipantInternal::new( + rtc_engine, sid, identity, name, metadata, + )), } } @@ -51,7 +52,7 @@ impl LocalParticipant { ) -> RoomResult { let mut req = proto::AddTrackRequest { cid: track.rtc_track().id(), - name: options.name.clone(), + name: track.name().clone(), r#type: proto::TrackType::from(track.kind()) as i32, muted: track.is_muted(), source: proto::TrackSource::from(options.source) as i32, @@ -65,9 +66,9 @@ impl LocalParticipant { LocalTrack::Video(video_track) => { // Get the video dimension // TODO(theomonnom): Use MediaStreamTrack::getSettings() on web - let capture_options = video_track.capture_options(); - req.width = capture_options.resolution.width; - req.height = capture_options.resolution.height; + let resolution = video_track.rtc_source().video_resolution(); + req.width = resolution.width; + req.height = resolution.height; encodings = compute_video_encodings(req.width, req.height, &options); req.layers = video_layers_from_encodings(req.width, req.height, &encodings); @@ -85,29 +86,34 @@ impl LocalParticipant { }); } } + let track_info = self.inner.rtc_engine.add_track(req).await?; + let publication = LocalTrackPublication::new( + track_info.clone(), + Arc::downgrade(&self.inner), + track.clone(), + ); + track.update_info(track_info); // Update sid + source - let track_info = self.rtc_engine.add_track(req).await?; - let publication = - LocalTrackPublication::new(track_info.clone(), track.clone(), options.clone()); - track.update_info(track_info); // Update SID + Source - debug!("publishing track with cid {:?}", track.rtc_track().id()); + log::debug!("publishing track with cid {:?}", track.rtc_track().id()); let transceiver = self + .inner .rtc_engine .create_sender(track.clone(), options, encodings) .await?; track.update_transceiver(Some(transceiver)); - track.start(); + //track.start(); + track.enable(); tokio::spawn({ - let rtc_engine = self.rtc_engine.clone(); + let rtc_engine = self.inner.rtc_engine.clone(); async move { let _ = rtc_engine.negotiate_publisher().await; } }); self.inner - .add_track_publication(TrackPublication::Local(publication.clone())); + .add_publication(TrackPublication::Local(publication.clone())); self.inner .dispatcher @@ -125,9 +131,10 @@ impl LocalParticipant { ) -> RoomResult { let mut tracks = self.inner.tracks.write(); if let Some(TrackPublication::Local(publication)) = tracks.remove(&track) { - let track = publication.track().unwrap(); + let track = publication.track(); let sender = track.transceiver().unwrap().sender(); - self.rtc_engine.remove_track(sender).await?; + + self.inner.rtc_engine.remove_track(sender).await?; track.update_transceiver(None); self.inner @@ -135,10 +142,10 @@ impl LocalParticipant { .dispatch(&ParticipantEvent::LocalTrackUnpublished { publication: publication.clone(), }); - publication.update_track(None); + // publication.update_track(None); tokio::spawn({ - let rtc_engine = self.rtc_engine.clone(); + let rtc_engine = self.inner.rtc_engine.clone(); async move { let _ = rtc_engine.negotiate_publisher().await; } @@ -152,20 +159,21 @@ impl LocalParticipant { pub async fn publish_data( &self, - data: &[u8], - kind: proto::data_packet::Kind, - ) -> Result<(), RoomError> { + data: Vec, + kind: DataPacketKind, + destination_sids: Vec, + ) -> RoomResult<()> { let data = proto::DataPacket { kind: kind as i32, value: Some(proto::data_packet::Value::User(proto::UserPacket { - participant_sid: self.sid().to_string(), - payload: data.to_vec(), - destination_sids: vec![], + payload: data, + destination_sids: destination_sids.to_owned(), ..Default::default() })), }; - self.rtc_engine + self.inner + .rtc_engine .publish_data(&data, kind) .await .map_err(Into::into) diff --git a/livekit/src/room/participant/mod.rs b/livekit/src/room/participant/mod.rs index be6f71a..4f72636 100644 --- a/livekit/src/room/participant/mod.rs +++ b/livekit/src/room/participant/mod.rs @@ -1,14 +1,15 @@ use crate::prelude::*; +use crate::rtc_engine::RtcEngine; use crate::track::TrackError; use livekit_protocol as proto; use livekit_protocol::enum_dispatch; use livekit_protocol::observer::Dispatcher; -use parking_lot::{Mutex, RwLock, RwLockReadGuard}; +use parking_lot::{RwLock, RwLockReadGuard}; use std::collections::HashMap; use std::fmt::Debug; -use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU8, Ordering}; use std::sync::Arc; -use tokio::sync::mpsc; +use std::thread::JoinHandle; +use tokio::sync::{mpsc, oneshot}; mod local_participant; mod remote_participant; @@ -38,7 +39,7 @@ pub enum ParticipantEvent { }, DataReceived { payload: Arc>, - kind: proto::data_packet::Kind, + kind: DataPacketKind, }, SpeakingChanged { speaking: bool, @@ -109,7 +110,6 @@ impl Participant { pub fn tracks(self: &Self) -> RwLockReadGuard>; pub fn register_observer(self: &Self) -> mpsc::UnboundedReceiver; - // Internal functions pub(crate) fn set_speaking(self: &Self, speaking: bool) -> (); pub(crate) fn set_audio_level(self: &Self, level: f32) -> (); pub(crate) fn set_connection_quality(self: &Self, quality: ConnectionQuality) -> (); @@ -118,56 +118,76 @@ impl Participant { } #[derive(Debug)] -pub(crate) struct ParticipantInner { - pub sid: Mutex, - pub identity: Mutex, - pub name: Mutex, - pub metadata: Mutex, - pub speaking: AtomicBool, - pub tracks: RwLock>, - pub audio_level: AtomicU32, - pub connection_quality: AtomicU8, - pub dispatcher: Dispatcher, +pub(crate) struct ParticipantInfo { + pub sid: ParticipantSid, + pub identity: ParticipantIdentity, + pub name: String, + pub metadata: String, + pub speaking: bool, + pub audio_level: f32, + pub connection_quality: ConnectionQuality, } -impl ParticipantInner { +#[derive(Debug)] +pub(crate) struct ParticipantInternal { + pub(super) rtc_engine: Arc, + pub(super) dispatcher: Dispatcher, + info: RwLock, + tracks: RwLock>, + tracks_tasks: RwLock, oneshot::Sender<()>)>>, +} + +impl ParticipantInternal { pub fn new( + rtc_engine: Arc, sid: ParticipantSid, identity: ParticipantIdentity, name: String, metadata: String, ) -> Self { Self { - sid: Mutex::new(sid), - identity: Mutex::new(identity), - name: Mutex::new(name), - metadata: Mutex::new(metadata), - tracks: Default::default(), - speaking: Default::default(), - audio_level: Default::default(), - connection_quality: AtomicU8::new(ConnectionQuality::Unknown as u8), + rtc_engine, + info: RwLock::new(ParticipantInfo { + sid, + identity, + name, + metadata, + speaking: false, + audio_level: 0.0, + connection_quality: ConnectionQuality::Unknown, + }), dispatcher: Default::default(), + tracks: Default::default(), + tracks_tasks: Default::default(), } } + pub fn update_info(&self, new_info: proto::ParticipantInfo) { + let mut info = self.info.write(); + info.sid = new_info.sid.into(); + info.name = new_info.name; + info.identity = new_info.identity.into(); + info.metadata = new_info.metadata; // TODO(theomonnom): callback MetadataChanged + } + pub fn sid(&self) -> ParticipantSid { - self.sid.lock().clone() + self.info.read().sid.clone() } pub fn identity(&self) -> ParticipantIdentity { - self.identity.lock().clone() + self.info.read().identity.clone() } pub fn name(&self) -> String { - self.name.lock().clone() + self.info.read().name.clone() } pub fn metadata(&self) -> String { - self.metadata.lock().clone() + self.info.read().metadata.clone() } pub fn is_speaking(&self) -> bool { - self.speaking.load(Ordering::SeqCst) + self.info.read().speaking } pub fn tracks(&self) -> RwLockReadGuard> { @@ -175,39 +195,34 @@ impl ParticipantInner { } pub fn audio_level(&self) -> f32 { - f32::from_bits(self.audio_level.load(Ordering::SeqCst)) + self.info.read().audio_level } pub fn connection_quality(&self) -> ConnectionQuality { - self.connection_quality.load(Ordering::SeqCst).into() + self.info.read().connection_quality } pub fn register_observer(&self) -> mpsc::UnboundedReceiver { self.dispatcher.register() } - pub fn update_info(&self, info: proto::ParticipantInfo) { - *self.sid.lock() = info.sid.into(); - *self.identity.lock() = info.identity.into(); - *self.name.lock() = info.name; - *self.metadata.lock() = info.metadata; // TODO(theomonnom): callback MetadataChanged - } - pub fn set_speaking(&self, speaking: bool) { - self.speaking.store(speaking, Ordering::SeqCst); + self.info.write().speaking = speaking; } pub fn set_audio_level(&self, audio_level: f32) { - self.audio_level - .store(audio_level.to_bits(), Ordering::SeqCst) + self.info.write().audio_level = audio_level; } pub fn set_connection_quality(&self, quality: ConnectionQuality) { - self.connection_quality - .store(quality as u8, Ordering::SeqCst); + self.info.write().connection_quality = quality; } - pub fn add_track_publication(&self, publication: TrackPublication) { + pub fn remove_publication(&self, sid: &TrackSid) { + self.tracks.write().remove(sid); + } + + pub fn add_publication(&self, publication: TrackPublication) { self.tracks.write().insert(publication.sid(), publication); } } diff --git a/livekit/src/room/participant/remote_participant.rs b/livekit/src/room/participant/remote_participant.rs index cec9555..6acc0f3 100644 --- a/livekit/src/room/participant/remote_participant.rs +++ b/livekit/src/room/participant/remote_participant.rs @@ -1,6 +1,8 @@ -use super::{ConnectionQuality, ParticipantInner}; -use crate::prelude::*; +use super::TrackKind; +use super::{ConnectionQuality, ParticipantInternal}; +use crate::rtc_engine::RtcEngine; use crate::track::TrackError; +use crate::{prelude::*, DataPacketKind}; use livekit_protocol as proto; use livekit_webrtc::prelude::*; use parking_lot::RwLockReadGuard; @@ -10,13 +12,12 @@ use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc; use tokio::time::timeout; -use tracing::{debug, error, instrument, Level}; const ADD_TRACK_TIMEOUT: Duration = Duration::from_secs(5); #[derive(Clone)] pub struct RemoteParticipant { - inner: Arc, + inner: Arc, } impl Debug for RemoteParticipant { @@ -31,29 +32,22 @@ impl Debug for RemoteParticipant { impl RemoteParticipant { pub(crate) fn new( + rtc_engine: Arc, sid: ParticipantSid, identity: ParticipantIdentity, name: String, metadata: String, ) -> Self { Self { - inner: Arc::new(ParticipantInner::new(sid, identity, name, metadata)), + inner: Arc::new(ParticipantInternal::new( + rtc_engine, sid, identity, name, metadata, + )), } } - #[inline] - pub fn get_track_publication(&self, sid: &TrackSid) -> Option { - self.inner.tracks.read().get(sid).map(|track| { - if let TrackPublication::Remote(remote) = track { - return remote.clone(); - } - unreachable!() - }) - } - - /// Called by the RoomSession when receiving data from the RrcSession + /// Called by the RoomSession when receiving data from the RtcSession /// It is just used to emit the Data event on the participant dispatcher. - pub(crate) fn on_data_received(&self, data: Arc>, kind: proto::data_packet::Kind) { + pub(crate) fn on_data_received(&self, data: Arc>, kind: DataPacketKind) { self.inner .dispatcher .dispatch(&ParticipantEvent::DataReceived { @@ -62,7 +56,6 @@ impl RemoteParticipant { }); } - #[instrument(level = Level::DEBUG)] pub(crate) async fn add_subscribed_media_track( &self, sid: TrackSid, @@ -78,7 +71,7 @@ impl RemoteParticipant { return publication; } - tokio::task::yield_now().await; // Remove yield + tokio::time::sleep(Duration::from_millis(50)).await; } } }; @@ -111,10 +104,10 @@ impl RemoteParticipant { } }; - debug!("starting track: {:?}", sid); + log::debug!("starting track: {:?}", sid); remote_publication.update_track(Some(track.clone().into())); - track.set_muted(remote_publication.is_muted()); + //track.set_muted(remote_publication.is_muted()); track.update_info(proto::TrackInfo { sid: remote_publication.sid().to_string(), name: remote_publication.name().to_string(), @@ -124,8 +117,9 @@ impl RemoteParticipant { }); self.inner - .add_track_publication(TrackPublication::Remote(remote_publication.clone())); - track.start(); + .add_publication(TrackPublication::Remote(remote_publication.clone())); + // track.start(); + track.enable(); self.inner .dispatcher @@ -134,7 +128,7 @@ impl RemoteParticipant { publication: remote_publication, }); } else { - error!("could not find published track with sid: {:?}", sid); + log::error!("could not find published track with sid: {:?}", sid); self.inner .dispatcher @@ -149,7 +143,7 @@ impl RemoteParticipant { if let Some(publication) = self.get_track_publication(sid) { // Unsubscribe to the track if needed if let Some(track) = publication.track() { - track.stop(); + track.disable(); self.inner .dispatcher @@ -159,6 +153,8 @@ impl RemoteParticipant { }); } + self.inner.remove_publication(sid); + self.inner .dispatcher .dispatch(&ParticipantEvent::TrackUnpublished { @@ -177,9 +173,10 @@ impl RemoteParticipant { if let Some(publication) = self.get_track_publication(&track.sid.clone().into()) { publication.update_info(track.clone()); } else { - let publication = RemoteTrackPublication::new(track.clone(), None); + let publication = + RemoteTrackPublication::new(track.clone(), Arc::downgrade(&self.inner), None); self.inner - .add_track_publication(TrackPublication::Remote(publication.clone())); + .add_publication(TrackPublication::Remote(publication.clone())); // This is a new track, dispatch publish event self.inner @@ -200,6 +197,16 @@ impl RemoteParticipant { } } + #[inline] + pub fn get_track_publication(&self, sid: &TrackSid) -> Option { + self.inner.tracks.read().get(sid).map(|track| { + if let TrackPublication::Remote(remote) = track { + return remote.clone(); + } + unreachable!() + }) + } + #[inline] pub fn sid(&self) -> ParticipantSid { self.inner.sid() diff --git a/livekit/src/room/publication/local.rs b/livekit/src/room/publication/local.rs index 1ad8f56..2e3002a 100644 --- a/livekit/src/room/publication/local.rs +++ b/livekit/src/room/publication/local.rs @@ -1,17 +1,13 @@ use super::TrackPublicationInner; use crate::id::TrackSid; -use crate::options::TrackPublishOptions; -use crate::track::{LocalTrack, Track, TrackDimension, TrackKind, TrackSource}; +use crate::participant::ParticipantInternal; +use crate::track::{LocalTrack, TrackDimension, TrackKind, TrackSource}; use livekit_protocol as proto; -use parking_lot::Mutex; -use std::sync::Arc; +use std::sync::{Arc, Weak}; #[derive(Debug)] struct LocalTrackPublicationInner { publication_inner: TrackPublicationInner, - - #[allow(unused)] // TODO(theomonnom) - options: Mutex, } #[derive(Clone, Debug)] @@ -22,17 +18,47 @@ pub struct LocalTrackPublication { impl LocalTrackPublication { pub(crate) fn new( info: proto::TrackInfo, + participant: Weak, track: LocalTrack, - options: TrackPublishOptions, ) -> Self { Self { inner: Arc::new(LocalTrackPublicationInner { - publication_inner: TrackPublicationInner::new(info, Some(track.into())), - options: Mutex::new(options), + publication_inner: TrackPublicationInner::new( + info, + participant, + Some(track.into()), + ), }), } } + pub async fn mute(&self) {} + + pub async fn unmute(&self) {} + + pub async fn pause_upstream(&self) {} + + pub async fn resume_upstream(&self) {} + + /*pub fn set_muted(&self, muted: bool) { + if self.is_muted() == muted { + return; + } + + self.track().rtc_track().set_enabled(!muted); + + let participant = self.inner.publication_inner.participant().upgrade(); + if participant.is_none() { + log::warn!("publication's participant is invalid, set_muted failed"); + return; + } + let participant = participant.unwrap(); + + // Engine update muted + + // Participant MUTED/UNMUTED event + }*/ + #[inline] pub fn sid(&self) -> TrackSid { self.inner.publication_inner.sid() @@ -64,11 +90,13 @@ impl LocalTrackPublication { } #[inline] - pub fn track(&self) -> Option { + pub fn track(&self) -> LocalTrack { self.inner .publication_inner .track() - .map(|track| track.try_into().unwrap()) + .unwrap() + .try_into() + .unwrap() } #[inline] @@ -86,10 +114,10 @@ impl LocalTrackPublication { false } - #[inline] + /*#[inline] pub(crate) fn update_track(&self, track: Option) { self.inner.publication_inner.update_track(track); - } + }*/ #[allow(dead_code)] #[inline] diff --git a/livekit/src/room/publication/mod.rs b/livekit/src/room/publication/mod.rs index d586625..61aa16b 100644 --- a/livekit/src/room/publication/mod.rs +++ b/livekit/src/room/publication/mod.rs @@ -1,140 +1,49 @@ -use super::track::{TrackDimension, TrackEvent}; +use super::track::TrackDimension; +use crate::participant::ParticipantInternal; use crate::prelude::*; use crate::track::Track; -use futures_util::stream::StreamExt; use livekit_protocol as proto; use livekit_protocol::enum_dispatch; -use livekit_protocol::observer::Dispatcher; -use parking_lot::Mutex; -use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; +use parking_lot::{Mutex, RwLock}; +use proto::observer::Dispatcher; use std::sync::Arc; +use std::sync::Weak; use tokio::sync::Notify; -use tokio_stream::wrappers::UnboundedReceiverStream; mod local; -pub use local::*; - mod remote; + +pub use local::*; pub use remote::*; -#[derive(Debug)] -pub(crate) struct TrackPublicationInner { - track: Mutex>, - name: Mutex, - sid: Mutex, - kind: AtomicU8, // Casted to TrackKind - source: AtomicU8, // Casted to TrackSource - simulcasted: AtomicBool, - dimension: Mutex, - mime_type: Mutex, - muted: AtomicBool, - dispatcher: Dispatcher, - close_notifier: Arc, +#[derive(Debug, Clone)] +pub enum PublicationEvent { + Muted, + Unmuted, + Subscribed, + Unsubscribed, + SubscriptionStatusChanged { + old_state: SubscriptionStatus, + new_state: SubscriptionStatus, + }, + SubscriptionPermissionChanged { + old_state: PermissionStatus, + new_state: PermissionStatus, + }, + SubscriptionFailed, } -impl TrackPublicationInner { - pub fn new(info: proto::TrackInfo, track: Option) -> Self { - Self { - track: Mutex::new(track), - name: Mutex::new(info.name), - sid: Mutex::new(info.sid.into()), - kind: AtomicU8::new( - TrackKind::try_from(proto::TrackType::from_i32(info.r#type).unwrap()).unwrap() - as u8, - ), - source: AtomicU8::new(TrackSource::from( - proto::TrackSource::from_i32(info.source).unwrap(), - ) as u8), - simulcasted: AtomicBool::new(info.simulcast), - dimension: Mutex::new(TrackDimension(info.width, info.height)), - mime_type: Mutex::new(info.mime_type), - muted: AtomicBool::new(info.muted), - dispatcher: Default::default(), - close_notifier: Default::default(), - } - } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SubscriptionStatus { + Desired, + Subscribed, + Unsubscribed, +} - pub fn update_track(&self, track: Option) { - let mut old_track = self.track.lock(); - *old_track = track.clone(); - - self.close_notifier.notify_waiters(); - - if let Some(track) = track.as_ref() { - let track_stream = UnboundedReceiverStream::new(track.register_observer()); - tokio::spawn({ - let dispatcher = self.dispatcher.clone(); - let notifier = self.close_notifier.clone(); - - async move { - let notified = notifier.notified(); - futures_util::pin_mut!(notified); - futures_util::future::select( - track_stream.map(Ok).forward(dispatcher), - notified, - ) - .await; - } - }); - } - } - - pub fn update_info(&self, info: proto::TrackInfo) { - *self.name.lock() = info.name; - *self.sid.lock() = info.sid.into(); - *self.dimension.lock() = TrackDimension(info.width, info.height); - *self.mime_type.lock() = info.mime_type; - self.kind.store( - TrackKind::try_from(proto::TrackType::from_i32(info.r#type).unwrap()).unwrap() as u8, - Ordering::SeqCst, - ); - self.source.store( - TrackSource::from(proto::TrackSource::from_i32(info.source).unwrap()) as u8, - Ordering::SeqCst, - ); - self.simulcasted.store(info.simulcast, Ordering::SeqCst); - self.muted.store(info.muted, Ordering::SeqCst); - - if let Some(track) = self.track.lock().as_ref() { - track.set_muted(info.muted); - } - } - - pub fn sid(&self) -> TrackSid { - self.sid.lock().clone() - } - - pub fn name(&self) -> String { - self.name.lock().clone() - } - - pub fn kind(&self) -> TrackKind { - self.kind.load(Ordering::SeqCst).try_into().unwrap() - } - - pub fn source(&self) -> TrackSource { - self.source.load(Ordering::SeqCst).into() - } - - pub fn simulcasted(&self) -> bool { - self.simulcasted.load(Ordering::Relaxed) - } - - pub fn dimension(&self) -> TrackDimension { - self.dimension.lock().clone() - } - - pub fn mime_type(&self) -> String { - self.mime_type.lock().clone() - } - - pub fn track(&self) -> Option { - self.track.lock().clone() - } - - pub fn is_muted(&self) -> bool { - self.muted.load(Ordering::Relaxed) - } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PermissionStatus { + Allowed, + NotAllowed, } #[derive(Clone, Debug)] @@ -159,8 +68,179 @@ impl TrackPublication { pub fn track(&self) -> Option { match self { - TrackPublication::Local(p) => p.track().map(Into::into), + TrackPublication::Local(p) => Some(p.track().into()), TrackPublication::Remote(p) => p.track().map(Into::into), } } } + +#[derive(Debug)] +pub(crate) struct PublicationInfo { + track: Option, + name: String, + sid: TrackSid, + kind: TrackKind, + source: TrackSource, + simulcasted: bool, + dimension: TrackDimension, + mime_type: String, + muted: bool, +} + +#[derive(Debug)] +pub(crate) struct TrackPublicationInner { + info: RwLock, + dispatcher: Dispatcher, + participant: Weak, + //forward_handle: Mutex>>, + forward_close: Arc, +} + +impl TrackPublicationInner { + pub fn new( + info: proto::TrackInfo, + participant: Weak, + track: Option, + ) -> Self { + let info = PublicationInfo { + track, + name: info.name, + sid: info.sid.into(), + kind: proto::TrackType::from_i32(info.r#type) + .unwrap() + .try_into() + .unwrap(), + source: proto::TrackSource::from_i32(info.source) + .unwrap() + .try_into() + .unwrap(), + simulcasted: info.simulcast, + dimension: TrackDimension(info.width, info.height), + mime_type: info.mime_type, + muted: info.muted, + }; + + Self { + info: RwLock::new(info), + dispatcher: Default::default(), + participant, + //forward_handle: Default::default(), + forward_close: Default::default(), + } + } + + // Forward track events to the publication events + // e.g: this also allow us to access the signal_client and notify the server if + // a local track changed mute state + async fn track_forward_task( + close_notifier: Weak, + track: Track, + dispatcher: Dispatcher, + ) { + let mut track_events = track.register_observer(); + loop { + let notifier = close_notifier.upgrade(); + if notifier.is_none() { + break; + } + let notified = notifier.as_ref().unwrap().notified(); + + tokio::select! { + _ = notified => { + break; + } + Some(event) = track_events.recv() => { + match event { + TrackEvent::Muted => { + dispatcher.dispatch(&PublicationEvent::Muted); + } + TrackEvent::Unmuted => { + dispatcher.dispatch(&PublicationEvent::Unmuted); + } + } + } + } + } + } + + pub fn update_track(&self, track: Option) { + //let forward_task = self.forward_handle.lock().take(); + //if let Some(task) = forward_task { + // Make sure to close the old forwarder before changing the track + self.forward_close.notify_waiters(); + //let _ = task.await; + // } + + let mut info = self.info.write(); + info.track = track.clone(); + + if let Some(track) = track { + let _handle = tokio::spawn(Self::track_forward_task( + Arc::downgrade(&self.forward_close), + track, + self.dispatcher.clone(), + )); + //let mut forward_handle = self.forward_handle.lock(); + //*forward_handle = Some(handle); + } + } + + // Called when updating a participant info + pub fn update_info(&self, new_info: proto::TrackInfo) { + let mut info = self.info.write(); + info.name = new_info.name; + info.sid = new_info.sid.into(); + info.dimension = TrackDimension(new_info.width, new_info.height); + info.mime_type = new_info.mime_type; + info.kind = + TrackKind::try_from(proto::TrackType::from_i32(new_info.r#type).unwrap()).unwrap(); + info.source = TrackSource::from(proto::TrackSource::from_i32(new_info.source).unwrap()); + info.simulcasted = new_info.simulcast; + + // TODO MUTE ????????????????? + // info.muted = new_info.muted; + // if let Some(track) = info.track.as_ref() { + // track.set_muted(info.muted); + // } + } + + pub fn participant(&self) -> Weak { + self.participant.clone() + } + + pub fn sid(&self) -> TrackSid { + self.info.read().sid.clone() + } + + pub fn name(&self) -> String { + self.info.read().name.clone() + } + + pub fn kind(&self) -> TrackKind { + self.info.read().kind + } + + pub fn source(&self) -> TrackSource { + self.info.read().source + } + + pub fn simulcasted(&self) -> bool { + self.info.read().simulcasted + } + + pub fn dimension(&self) -> TrackDimension { + self.info.read().dimension.clone() + } + + pub fn mime_type(&self) -> String { + self.info.read().mime_type.clone() + } + + pub fn track(&self) -> Option { + self.info.read().track.clone() + } + + pub fn is_muted(&self) -> bool { + self.info.read().muted + } +} diff --git a/livekit/src/room/publication/remote.rs b/livekit/src/room/publication/remote.rs index 5f43fcb..7ca4149 100644 --- a/livekit/src/room/publication/remote.rs +++ b/livekit/src/room/publication/remote.rs @@ -1,64 +1,170 @@ -use super::TrackPublicationInner; +use super::{PermissionStatus, SubscriptionStatus, TrackPublicationInner}; use crate::id::TrackSid; +use crate::participant::ParticipantInternal; +use crate::publication::PublicationEvent; use crate::track::{RemoteTrack, Track, TrackDimension, TrackKind, TrackSource}; use livekit_protocol as proto; -use std::sync::Arc; +use parking_lot::RwLock; +use std::sync::{Arc, Weak}; + +#[derive(Debug)] +struct RemoteInfo { + subscribed: bool, + allowed: bool, + // TODO(theomonnom): other remote info +} + +#[derive(Debug)] +struct RemoteInner { + publication_inner: TrackPublicationInner, + info: RwLock, +} #[derive(Clone, Debug)] pub struct RemoteTrackPublication { - inner: Arc, + inner: Arc, } impl RemoteTrackPublication { - pub(crate) fn new(info: proto::TrackInfo, track: Option) -> Self { + pub(crate) fn new( + info: proto::TrackInfo, + participant: Weak, + track: Option, + ) -> Self { Self { - inner: Arc::new(TrackPublicationInner::new(info, track.map(Into::into))), + inner: Arc::new(RemoteInner { + publication_inner: TrackPublicationInner::new( + info, + participant, + track.map(Into::into), + ), + info: RwLock::new(RemoteInfo { + subscribed: false, + allowed: false, + }), + }), + } + } + + pub fn set_subscribed(&self, subscribed: bool) { + let old_subscription_state = self.subscription_status(); + let old_permission_state = self.permission_status(); + let mut info = self.inner.info.write(); + info.subscribed = subscribed; + + if subscribed { + info.allowed = true; + } + + let participant = self.inner.publication_inner.participant.upgrade(); + if participant.is_none() { + log::warn!("publication's participant is invalid, set_subscribed failed"); + return; + } + let participant = participant.unwrap(); + + let update_subscription = proto::UpdateSubscription { + track_sids: vec![self.sid().0], + subscribe: subscribed, + participant_tracks: vec![proto::ParticipantTracks { + participant_sid: participant.sid().0, + track_sids: vec![self.sid().0], + }], + }; + + // Engine update subscription + + if old_subscription_state != self.subscription_status() { + self.inner.publication_inner.dispatcher.dispatch( + &PublicationEvent::SubscriptionStatusChanged { + old_state: old_subscription_state, + new_state: self.subscription_status(), + }, + ) + } + + if old_permission_state != self.permission_status() { + self.inner.publication_inner.dispatcher.dispatch( + &PublicationEvent::SubscriptionPermissionChanged { + old_state: old_permission_state, + new_state: self.permission_status(), + }, + ) } } + #[inline] + pub fn subscription_status(&self) -> SubscriptionStatus { + if !self.inner.info.read().subscribed { + return SubscriptionStatus::Unsubscribed; + } + + if self.track().is_none() { + return SubscriptionStatus::Desired; + } + + SubscriptionStatus::Subscribed + } + + #[inline] + pub fn permission_status(&self) -> PermissionStatus { + if self.inner.info.read().allowed { + PermissionStatus::Allowed + } else { + PermissionStatus::NotAllowed + } + } + + pub fn is_subscribed(&self) -> bool { + self.inner.info.read().allowed && self.track().is_some() + } + #[inline] pub fn sid(&self) -> TrackSid { - self.inner.sid() + self.inner.publication_inner.sid() } #[inline] pub fn name(&self) -> String { - self.inner.name() + self.inner.publication_inner.name() } #[inline] pub fn kind(&self) -> TrackKind { - self.inner.kind() + self.inner.publication_inner.kind() } #[inline] pub fn source(&self) -> TrackSource { - self.inner.source() + self.inner.publication_inner.source() } #[inline] pub fn simulcasted(&self) -> bool { - self.inner.simulcasted() + self.inner.publication_inner.simulcasted() } #[inline] pub fn dimension(&self) -> TrackDimension { - self.inner.dimension() + self.inner.publication_inner.dimension() } #[inline] pub fn track(&self) -> Option { - self.inner.track().map(|track| track.try_into().unwrap()) + self.inner + .publication_inner + .track() + .map(|track| track.try_into().unwrap()) } #[inline] pub fn mime_type(&self) -> String { - self.inner.mime_type() + self.inner.publication_inner.mime_type() } #[inline] pub fn is_muted(&self) -> bool { - self.inner.is_muted() + self.inner.publication_inner.is_muted() } #[inline] @@ -68,11 +174,11 @@ impl RemoteTrackPublication { #[inline] pub(crate) fn update_track(&self, track: Option) { - self.inner.update_track(track); + self.inner.publication_inner.update_track(track); } #[inline] pub(crate) fn update_info(&self, info: proto::TrackInfo) { - self.inner.update_info(info); + self.inner.publication_inner.update_info(info); } } diff --git a/livekit/src/room/track/local_audio_track.rs b/livekit/src/room/track/local_audio_track.rs index 602eda1..309a46a 100644 --- a/livekit/src/room/track/local_audio_track.rs +++ b/livekit/src/room/track/local_audio_track.rs @@ -1,24 +1,17 @@ use super::TrackInner; -use crate::options::AudioCaptureOptions; use crate::prelude::*; use crate::rtc_engine::lk_runtime::LkRuntime; -use crate::webrtc::peer_connection_factory::native::PeerConnectionFactoryExt; +use core::panic; use livekit_protocol as proto; use livekit_webrtc::prelude::*; -use parking_lot::Mutex; use std::fmt::Debug; use std::sync::Arc; use tokio::sync::mpsc; -#[derive(Debug)] -pub struct LocalAudioTrackInner { - track_inner: TrackInner, - capture_options: Mutex, -} - #[derive(Clone)] pub struct LocalAudioTrack { - inner: Arc, + inner: Arc, + source: RtcAudioSource, } impl Debug for LocalAudioTrack { @@ -32,85 +25,84 @@ impl Debug for LocalAudioTrack { } impl LocalAudioTrack { - pub(crate) fn new( - name: String, - rtc_track: RtcAudioTrack, - capture_options: AudioCaptureOptions, - ) -> Self { + pub(crate) fn new(name: String, rtc_track: RtcAudioTrack, source: RtcAudioSource) -> Self { Self { - inner: Arc::new(LocalAudioTrackInner { - track_inner: TrackInner::new( - "unknown".to_string().into(), // sid - name, - TrackKind::Audio, - MediaStreamTrack::Audio(rtc_track), - ), - capture_options: Mutex::new(capture_options), - }), + inner: Arc::new(TrackInner::new( + "unknown".to_string().into(), // sid + name, + TrackKind::Audio, + MediaStreamTrack::Audio(rtc_track), + )), + source, } } - #[inline] - pub fn capture_options(&self) -> AudioCaptureOptions { - self.inner.capture_options.lock().clone() - } - #[inline] pub fn sid(&self) -> TrackSid { - self.inner.track_inner.sid() + self.inner.sid() } #[inline] pub fn name(&self) -> String { - self.inner.track_inner.name() + self.inner.name() } #[inline] pub fn kind(&self) -> TrackKind { - self.inner.track_inner.kind() + self.inner.kind() } #[inline] pub fn source(&self) -> TrackSource { - self.inner.track_inner.source() + self.inner.source() } #[inline] pub fn stream_state(&self) -> StreamState { - self.inner.track_inner.stream_state() + self.inner.stream_state() } #[inline] - pub fn start(&self) { - self.inner.track_inner.start() + pub fn enable(&self) { + self.inner.enable() } #[inline] - pub fn stop(&self) { - self.inner.track_inner.stop() + pub fn disable(&self) { + self.inner.disable() } #[inline] pub fn is_muted(&self) -> bool { - self.inner.track_inner.is_muted() + self.inner.is_muted() } #[inline] - pub fn set_muted(&self, muted: bool) { - self.inner.track_inner.set_muted(muted) + pub fn mute(&self) { + self.inner.set_muted(true); + } + + #[inline] + pub fn unmute(&self) { + self.inner.set_muted(false); } #[inline] pub fn rtc_track(&self) -> RtcAudioTrack { - if let MediaStreamTrack::Audio(audio) = self.inner.track_inner.rtc_track() { + if let MediaStreamTrack::Audio(audio) = self.inner.rtc_track() { return audio; } unreachable!() } + #[inline] + pub fn rtc_source(&self) -> RtcAudioSource { + self.source.clone() + } + #[inline] pub fn register_observer(&self) -> mpsc::UnboundedReceiver { - self.inner.track_inner.register_observer() + self.inner.register_observer() } #[inline] @@ -120,30 +112,33 @@ impl LocalAudioTrack { #[inline] pub(crate) fn transceiver(&self) -> Option { - self.inner.track_inner.transceiver() + self.inner.transceiver() } #[inline] pub(crate) fn update_transceiver(&self, transceiver: Option) { - self.inner.track_inner.update_transceiver(transceiver) + self.inner.update_transceiver(transceiver) } #[inline] pub(crate) fn update_info(&self, info: proto::TrackInfo) { - self.inner.track_inner.update_info(info) + self.inner.update_info(info) } } impl LocalAudioTrack { - pub fn create_audio_track( - name: &str, - options: AudioCaptureOptions, - source: livekit_webrtc::audio_source::native::NativeAudioSource, - ) -> LocalAudioTrack { - let rtc_track = LkRuntime::instance() - .pc_factory() - .create_audio_track(&livekit_webrtc::native::create_random_uuid(), source); - - Self::new(name.to_string(), rtc_track, options) + pub fn create_audio_track(name: &str, source: RtcAudioSource) -> LocalAudioTrack { + let rtc_track = match source.clone() { + #[cfg(not(target_arch = "wasm32"))] + RtcAudioSource::Native(native_source) => { + use livekit_webrtc::peer_connection_factory::native::PeerConnectionFactoryExt; + LkRuntime::instance().pc_factory().create_audio_track( + &livekit_webrtc::native::create_random_uuid(), + native_source, + ) + } + _ => panic!("unsupported audio source"), + }; + Self::new(name.to_string(), rtc_track, source) } } diff --git a/livekit/src/room/track/local_track.rs b/livekit/src/room/track/local_track.rs new file mode 100644 index 0000000..9aef391 --- /dev/null +++ b/livekit/src/room/track/local_track.rs @@ -0,0 +1,33 @@ +use super::TrackInner; +use super::{track_dispatch, LocalAudioTrack, LocalVideoTrack}; +use crate::prelude::*; +use crate::track::TrackEvent; +use livekit_protocol as proto; +use livekit_protocol::enum_dispatch; +use livekit_webrtc::prelude::*; +use std::sync::Arc; +use tokio::sync::mpsc; + +#[derive(Clone, Debug)] +pub enum LocalTrack { + Audio(LocalAudioTrack), + Video(LocalVideoTrack), +} + +impl LocalTrack { + track_dispatch!([Audio, Video]); + + enum_dispatch!( + [Audio, Video]; + pub fn mute(self: &Self) -> (); + pub fn unmute(self: &Self) -> (); + ); + + #[inline] + pub fn rtc_track(&self) -> MediaStreamTrack { + match self { + Self::Audio(track) => track.rtc_track().into(), + Self::Video(track) => track.rtc_track().into(), + } + } +} diff --git a/livekit/src/room/track/local_video_track.rs b/livekit/src/room/track/local_video_track.rs index 1c9f2cf..a2fded7 100644 --- a/livekit/src/room/track/local_video_track.rs +++ b/livekit/src/room/track/local_video_track.rs @@ -1,23 +1,16 @@ use super::TrackInner; +use crate::prelude::*; use crate::rtc_engine::lk_runtime::LkRuntime; -use crate::{options::VideoCaptureOptions, prelude::*}; use livekit_protocol as proto; -use livekit_webrtc::peer_connection_factory::native::PeerConnectionFactoryExt; use livekit_webrtc::prelude::*; -use parking_lot::Mutex; use std::fmt::Debug; use std::sync::Arc; use tokio::sync::mpsc; -#[derive(Debug)] -struct LocalVideoTrackInner { - track_inner: TrackInner, - capture_options: Mutex, -} - #[derive(Clone)] pub struct LocalVideoTrack { - inner: Arc, + inner: Arc, + source: RtcVideoSource, } impl Debug for LocalVideoTrack { @@ -31,85 +24,84 @@ impl Debug for LocalVideoTrack { } impl LocalVideoTrack { - pub fn new( - name: String, - rtc_track: RtcVideoTrack, - capture_options: VideoCaptureOptions, - ) -> Self { + pub fn new(name: String, rtc_track: RtcVideoTrack, source: RtcVideoSource) -> Self { Self { - inner: Arc::new(LocalVideoTrackInner { - track_inner: TrackInner::new( - "unknown".to_string().into(), // sid - name, - TrackKind::Video, - MediaStreamTrack::Video(rtc_track), - ), - capture_options: Mutex::new(capture_options), - }), + inner: Arc::new(TrackInner::new( + "unknown".to_string().into(), // sid + name, + TrackKind::Video, + MediaStreamTrack::Video(rtc_track), + )), + source, } } - #[inline] - pub fn capture_options(&self) -> VideoCaptureOptions { - self.inner.capture_options.lock().clone() - } - #[inline] pub fn sid(&self) -> TrackSid { - self.inner.track_inner.sid() + self.inner.sid() } #[inline] pub fn name(&self) -> String { - self.inner.track_inner.name() + self.inner.name() } #[inline] pub fn kind(&self) -> TrackKind { - self.inner.track_inner.kind() + self.inner.kind() } #[inline] pub fn source(&self) -> TrackSource { - self.inner.track_inner.source() + self.inner.source() } #[inline] pub fn stream_state(&self) -> StreamState { - self.inner.track_inner.stream_state() + self.inner.stream_state() } #[inline] - pub fn start(&self) { - self.inner.track_inner.start() + pub fn enable(&self) { + self.inner.enable() } #[inline] - pub fn stop(&self) { - self.inner.track_inner.stop() + pub fn disable(&self) { + self.inner.disable() } #[inline] pub fn is_muted(&self) -> bool { - self.inner.track_inner.is_muted() + self.inner.is_muted() } #[inline] - pub fn set_muted(&self, muted: bool) { - self.inner.track_inner.set_muted(muted) + pub fn mute(&self) { + self.inner.set_muted(true); + } + + #[inline] + pub fn unmute(&self) { + self.inner.set_muted(false); } #[inline] pub fn rtc_track(&self) -> RtcVideoTrack { - if let MediaStreamTrack::Video(video) = self.inner.track_inner.rtc_track() { + if let MediaStreamTrack::Video(video) = self.inner.rtc_track() { return video; } unreachable!() } + #[inline] + pub fn rtc_source(&self) -> RtcVideoSource { + self.source.clone() + } + #[inline] pub fn register_observer(&self) -> mpsc::UnboundedReceiver { - self.inner.track_inner.register_observer() + self.inner.register_observer() } #[inline] @@ -119,30 +111,34 @@ impl LocalVideoTrack { #[inline] pub(crate) fn transceiver(&self) -> Option { - self.inner.track_inner.transceiver() + self.inner.transceiver() } #[inline] pub(crate) fn update_transceiver(&self, transceiver: Option) { - self.inner.track_inner.update_transceiver(transceiver) + self.inner.update_transceiver(transceiver) } #[inline] pub(crate) fn update_info(&self, info: proto::TrackInfo) { - self.inner.track_inner.update_info(info) + self.inner.update_info(info) } } impl LocalVideoTrack { - pub fn create_video_track( - name: &str, - options: VideoCaptureOptions, - source: livekit_webrtc::video_source::native::NativeVideoSource, - ) -> LocalVideoTrack { - let rtc_track = LkRuntime::instance() - .pc_factory() - .create_video_track(&livekit_webrtc::native::create_random_uuid(), source); + pub fn create_video_track(name: &str, source: RtcVideoSource) -> LocalVideoTrack { + let rtc_track = match source.clone() { + #[cfg(not(target_arch = "wasm32"))] + RtcVideoSource::Native(native_source) => { + use livekit_webrtc::peer_connection_factory::native::PeerConnectionFactoryExt; + LkRuntime::instance().pc_factory().create_video_track( + &livekit_webrtc::native::create_random_uuid(), + native_source, + ) + } + _ => panic!("unsupported video source"), + }; - Self::new(name.to_string(), rtc_track, options) + Self::new(name.to_string(), rtc_track, source) } } diff --git a/livekit/src/room/track/mod.rs b/livekit/src/room/track/mod.rs index d84ad93..adb0529 100644 --- a/livekit/src/room/track/mod.rs +++ b/livekit/src/room/track/mod.rs @@ -3,19 +3,22 @@ use livekit_protocol as proto; use livekit_protocol::enum_dispatch; use livekit_protocol::observer::Dispatcher; use livekit_webrtc::prelude::*; -use parking_lot::Mutex; -use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; +use parking_lot::RwLock; use thiserror::Error; use tokio::sync::mpsc; mod local_audio_track; +mod local_track; mod local_video_track; mod remote_audio_track; +mod remote_track; mod remote_video_track; pub use local_audio_track::*; +pub use local_track::*; pub use local_video_track::*; pub use remote_audio_track::*; +pub use remote_track::*; pub use remote_video_track::*; #[derive(Error, Debug, Clone)] @@ -45,10 +48,10 @@ pub enum TrackSource { ScreenshareAudio, } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone)] pub enum TrackEvent { - Mute, - Unmute, + Muted, + Unmuted, } #[derive(Clone, Copy, Debug)] @@ -62,18 +65,6 @@ pub enum Track { RemoteVideo(RemoteVideoTrack), } -#[derive(Clone, Debug)] -pub enum LocalTrack { - Audio(LocalAudioTrack), - Video(LocalVideoTrack), -} - -#[derive(Clone, Debug)] -pub enum RemoteTrack { - Audio(RemoteAudioTrack), - Video(RemoteVideoTrack), -} - #[derive(Clone, Debug)] pub enum VideoTrack { Local(LocalVideoTrack), @@ -95,12 +86,11 @@ macro_rules! track_dispatch { pub fn kind(self: &Self) -> TrackKind; pub fn source(self: &Self) -> TrackSource; pub fn stream_state(self: &Self) -> StreamState; - pub fn start(self: &Self) -> (); - pub fn stop(self: &Self) -> (); + pub fn enable(self: &Self) -> (); + pub fn disable(self: &Self) -> (); pub fn is_muted(self: &Self) -> bool; - pub fn set_muted(self: &Self, muted: bool) -> (); - pub fn register_observer(self: &Self) -> mpsc::UnboundedReceiver; pub fn is_remote(self: &Self) -> bool; + pub fn register_observer(self: &Self) -> mpsc::UnboundedReceiver; pub(crate) fn transceiver(self: &Self) -> Option; pub(crate) fn update_transceiver(self: &Self, transceiver: Option) -> (); @@ -109,6 +99,8 @@ macro_rules! track_dispatch { }; } +pub(crate) use track_dispatch; + impl Track { track_dispatch!([LocalAudio, LocalVideo, RemoteAudio, RemoteVideo]); @@ -123,30 +115,6 @@ impl Track { } } -impl LocalTrack { - track_dispatch!([Audio, Video]); - - #[inline] - pub fn rtc_track(&self) -> MediaStreamTrack { - match self { - Self::Audio(track) => track.rtc_track().into(), - Self::Video(track) => track.rtc_track().into(), - } - } -} - -impl RemoteTrack { - track_dispatch!([Audio, Video]); - - #[inline] - pub fn rtc_track(&self) -> MediaStreamTrack { - match self { - Self::Audio(track) => track.rtc_track().into(), - Self::Video(track) => track.rtc_track().into(), - } - } -} - impl VideoTrack { track_dispatch!([Local, Remote]); @@ -171,90 +139,73 @@ impl AudioTrack { } } +#[derive(Debug)] +struct TrackInfo { + sid: TrackSid, + name: String, + kind: TrackKind, + source: TrackSource, + stream_state: StreamState, + muted: bool, + transceiver: Option, +} + #[derive(Debug)] pub(crate) struct TrackInner { - pub sid: Mutex, - pub name: Mutex, - pub kind: AtomicU8, // TrackKind - pub source: AtomicU8, // TrackSource - pub stream_state: AtomicU8, // StreamState - pub muted: AtomicBool, - pub rtc_track: MediaStreamTrack, - pub transceiver: Mutex>, - pub dispatcher: Dispatcher, + info: RwLock, + rtc_track: MediaStreamTrack, + dispatcher: Dispatcher, } impl TrackInner { pub fn new(sid: TrackSid, name: String, kind: TrackKind, rtc_track: MediaStreamTrack) -> Self { Self { - sid: Mutex::new(sid), - name: Mutex::new(name), - kind: AtomicU8::new(kind as u8), - source: AtomicU8::new(TrackSource::Unknown as u8), - stream_state: AtomicU8::new(StreamState::Active as u8), - muted: AtomicBool::new(false), + info: RwLock::new(TrackInfo { + sid, + name, + kind, + source: TrackSource::Unknown, + stream_state: StreamState::Active, + muted: false, + transceiver: None, + }), rtc_track, - transceiver: Default::default(), dispatcher: Default::default(), } } pub fn sid(&self) -> TrackSid { - self.sid.lock().clone() + self.info.read().sid.clone() } pub fn name(&self) -> String { - self.name.lock().clone() + self.info.read().name.clone() } pub fn kind(&self) -> TrackKind { - self.kind.load(Ordering::SeqCst).try_into().unwrap() + self.info.read().kind } pub fn source(&self) -> TrackSource { - self.source.load(Ordering::SeqCst).into() + self.info.read().source } pub fn stream_state(&self) -> StreamState { - self.stream_state.load(Ordering::SeqCst).try_into().unwrap() + self.info.read().stream_state } pub fn is_muted(&self) -> bool { - self.muted.load(Ordering::SeqCst) + self.info.read().muted } - pub fn start(&self) { + pub fn enable(&self) { self.rtc_track.set_enabled(true); } - pub fn stop(&self) { + pub fn disable(&self) { self.rtc_track.set_enabled(false); } - pub fn set_muted(&self, muted: bool) { - if self - .muted - .compare_exchange(!muted, muted, Ordering::SeqCst, Ordering::SeqCst) - .is_err() - { - return; - } - - if !muted { - self.start(); - } else { - self.stop(); - } - - let event = if muted { - TrackEvent::Mute - } else { - TrackEvent::Unmute - }; - - self.dispatcher.dispatch(&event); - } - pub fn rtc_track(&self) -> MediaStreamTrack { self.rtc_track.clone() } @@ -264,24 +215,39 @@ impl TrackInner { } pub fn transceiver(&self) -> Option { - self.transceiver.lock().clone() + self.info.read().transceiver.clone() } pub fn update_transceiver(&self, transceiver: Option) { - *self.transceiver.lock() = transceiver; + self.info.write().transceiver = transceiver; } - pub fn update_info(&self, info: proto::TrackInfo) { - *self.name.lock() = info.name; - *self.sid.lock() = info.sid.into(); - self.kind.store( - TrackKind::try_from(proto::TrackType::from_i32(info.r#type).unwrap()).unwrap() as u8, - Ordering::SeqCst, - ); - self.source.store( - TrackSource::from(proto::TrackSource::from_i32(info.source).unwrap()) as u8, - Ordering::SeqCst, - ); + pub fn set_muted(&self, muted: bool) { + log::debug!("set_muted: {} {}", self.sid(), muted); + if self.is_muted() == muted { + return; + } + + if muted { + self.disable(); + } else { + self.enable(); + } + + self.dispatcher.dispatch(if muted { + &TrackEvent::Muted + } else { + &TrackEvent::Unmuted + }); + } + + pub fn update_info(&self, new_info: proto::TrackInfo) { + let mut info = self.info.write(); + info.name = new_info.name; + info.sid = new_info.sid.into(); + info.kind = + TrackKind::try_from(proto::TrackType::from_i32(new_info.r#type).unwrap()).unwrap(); + info.source = TrackSource::from(proto::TrackSource::from_i32(new_info.source).unwrap()); // Muted and StreamState are not handled separately (events) } } @@ -370,44 +336,6 @@ impl TryFrom for AudioTrack { } } -// Conversions from integers (Useful since we're using atomic values to represent our enums) - -impl TryFrom for TrackKind { - type Error = &'static str; - - fn try_from(kind: u8) -> Result { - match kind { - 0 => Ok(Self::Audio), - 1 => Ok(Self::Video), - _ => Err("invalid track kind"), - } - } -} - -impl TryFrom for StreamState { - type Error = &'static str; - - fn try_from(state: u8) -> Result { - match state { - 0 => Ok(Self::Active), - 1 => Ok(Self::Paused), - _ => Err("invalid stream state"), - } - } -} - -impl From for TrackSource { - fn from(source: u8) -> Self { - match source { - 1 => Self::Camera, - 2 => Self::Microphone, - 3 => Self::Screenshare, - 4 => Self::ScreenshareAudio, - _ => Self::Unknown, - } - } -} - impl From for MediaType { fn from(kind: TrackKind) -> Self { match kind { diff --git a/livekit/src/room/track/remote_audio_track.rs b/livekit/src/room/track/remote_audio_track.rs index c5ca90f..1504311 100644 --- a/livekit/src/room/track/remote_audio_track.rs +++ b/livekit/src/room/track/remote_audio_track.rs @@ -1,3 +1,4 @@ +use super::remote_track; use super::TrackInner; use crate::prelude::*; use livekit_protocol as proto; @@ -59,13 +60,13 @@ impl RemoteAudioTrack { } #[inline] - pub fn start(&self) { - self.inner.start() + pub fn enable(&self) { + self.inner.enable() } #[inline] - pub fn stop(&self) { - self.inner.stop() + pub fn disable(&self) { + self.inner.disable() } #[inline] @@ -73,11 +74,6 @@ impl RemoteAudioTrack { self.inner.is_muted() } - #[inline] - pub fn set_muted(&self, muted: bool) { - self.inner.set_muted(muted) - } - #[inline] pub fn rtc_track(&self) -> RtcAudioTrack { if let MediaStreamTrack::Audio(audio) = self.inner.rtc_track() { @@ -110,6 +106,6 @@ impl RemoteAudioTrack { #[inline] pub(crate) fn update_info(&self, info: proto::TrackInfo) { - self.inner.update_info(info) + remote_track::update_info(&self.inner, info); } } diff --git a/livekit/src/room/track/remote_track.rs b/livekit/src/room/track/remote_track.rs new file mode 100644 index 0000000..f105881 --- /dev/null +++ b/livekit/src/room/track/remote_track.rs @@ -0,0 +1,33 @@ +use super::track_dispatch; +use super::TrackInner; +use super::{RemoteAudioTrack, RemoteVideoTrack}; +use crate::prelude::*; +use crate::track::TrackEvent; +use livekit_protocol as proto; +use livekit_protocol::enum_dispatch; +use livekit_webrtc::prelude::*; +use std::sync::Arc; +use tokio::sync::mpsc; + +#[derive(Clone, Debug)] +pub enum RemoteTrack { + Audio(RemoteAudioTrack), + Video(RemoteVideoTrack), +} + +impl RemoteTrack { + track_dispatch!([Audio, Video]); + + #[inline] + pub fn rtc_track(&self) -> MediaStreamTrack { + match self { + Self::Audio(track) => track.rtc_track().into(), + Self::Video(track) => track.rtc_track().into(), + } + } +} + +pub(crate) fn update_info(track: &Arc, new_info: proto::TrackInfo) { + track.update_info(new_info.clone()); + track.set_muted(new_info.muted); +} diff --git a/livekit/src/room/track/remote_video_track.rs b/livekit/src/room/track/remote_video_track.rs index 7f6ef96..825043b 100644 --- a/livekit/src/room/track/remote_video_track.rs +++ b/livekit/src/room/track/remote_video_track.rs @@ -1,4 +1,4 @@ -use super::TrackInner; +use super::{remote_track, TrackInner}; use crate::prelude::*; use livekit_protocol as proto; use livekit_webrtc::prelude::*; @@ -59,13 +59,13 @@ impl RemoteVideoTrack { } #[inline] - pub fn start(&self) { - self.inner.start() + pub fn enable(&self) { + self.inner.enable() } #[inline] - pub fn stop(&self) { - self.inner.stop() + pub fn disable(&self) { + self.inner.disable() } #[inline] @@ -73,11 +73,6 @@ impl RemoteVideoTrack { self.inner.is_muted() } - #[inline] - pub fn set_muted(&self, muted: bool) { - self.inner.set_muted(muted) - } - #[inline] pub fn rtc_track(&self) -> RtcVideoTrack { if let MediaStreamTrack::Video(video) = self.inner.rtc_track() { @@ -110,6 +105,6 @@ impl RemoteVideoTrack { #[inline] pub(crate) fn update_info(&self, info: proto::TrackInfo) { - self.inner.update_info(info); + remote_track::update_info(&self.inner, info); } } diff --git a/livekit/src/rtc_engine/lk_runtime.rs b/livekit/src/rtc_engine/lk_runtime.rs index 0868afa..9051d9d 100644 --- a/livekit/src/rtc_engine/lk_runtime.rs +++ b/livekit/src/rtc_engine/lk_runtime.rs @@ -3,7 +3,6 @@ use livekit_webrtc::prelude::*; use parking_lot::Mutex; use std::fmt::{Debug, Formatter}; use std::sync::{Arc, Weak}; -use tracing::trace; lazy_static! { static ref LK_RUNTIME: Mutex> = Mutex::new(Weak::new()); @@ -25,7 +24,7 @@ impl LkRuntime { if let Some(lk_runtime) = lk_runtime_ref.upgrade() { lk_runtime } else { - trace!("LkRuntime::new()"); + log::trace!("LkRuntime::new()"); let new_runtime = Arc::new(Self { pc_factory: PeerConnectionFactory::default(), }); @@ -41,6 +40,6 @@ impl LkRuntime { impl Drop for LkRuntime { fn drop(&mut self) { - trace!("LkRuntime::drop()"); + log::trace!("LkRuntime::drop()"); } } diff --git a/livekit/src/rtc_engine/mod.rs b/livekit/src/rtc_engine/mod.rs index 7b3dbf6..b8742d0 100644 --- a/livekit/src/rtc_engine/mod.rs +++ b/livekit/src/rtc_engine/mod.rs @@ -1,8 +1,9 @@ use crate::options::TrackPublishOptions; use crate::prelude::LocalTrack; use crate::rtc_engine::lk_runtime::LkRuntime; -use crate::rtc_engine::rtc_session::{RtcSession, SessionEvent, SessionEvents, SessionInfo}; +use crate::rtc_engine::rtc_session::{RtcSession, SessionEvent, SessionEvents}; use crate::signal_client::{SignalError, SignalOptions}; +use crate::DataPacketKind; use livekit_protocol as proto; use livekit_webrtc::prelude::*; use livekit_webrtc::session_description::SdpParseError; @@ -14,9 +15,9 @@ use std::time::Duration; use thiserror::Error; use tokio::sync::RwLock as AsyncRwLock; use tokio::sync::{mpsc, oneshot}; +use tokio::sync::{Mutex as AsyncMutex, Notify}; use tokio::task::JoinHandle; -use tokio::time::{interval, Interval}; -use tracing::{error, info, trace, warn}; +use tokio::time::{interval, Interval, MissedTickBehavior}; pub mod lk_runtime; mod peer_transport; @@ -72,7 +73,7 @@ pub enum EngineEvent { Data { participant_sid: String, payload: Vec, - kind: proto::data_packet::Kind, + kind: DataPacketKind, }, SpeakersChanged { speakers: Vec, @@ -90,7 +91,6 @@ pub enum EngineEvent { pub const RECONNECT_ATTEMPTS: u32 = 10; pub const RECONNECT_INTERVAL: Duration = Duration::from_secs(5); -/// /// Represents a running RTCSession with the ability to close the session /// and the engine_task #[derive(Debug)] @@ -101,23 +101,30 @@ struct EngineHandle { } struct EngineInner { + // Keep a strong reference to LkRuntime to avoid creating a new RtcRuntime or PeerConnection factory accross multiple Rtc sessions + #[allow(dead_code)] lk_runtime: Arc, - session_info: Mutex>, // Last/Current Sessioninfo - running_handle: AsyncRwLock>, - opened: AtomicBool, engine_emitter: EngineEmitter, + // Last/current session JoinResponse + // We keep a clone of the join response here because the room needs it + // (directly accessing the running_handle requires an async context to lock the Mutex and a getter needs a short lock) + // Maybe there is a better way to do it? + join_response: Mutex, + running_handle: AsyncRwLock>, + // Reconnecting fields + closed: AtomicBool, // True if closed or the reconnection failed (Note that this is false when reconnecting or resuming) reconnecting: AtomicBool, - full_reconnect: AtomicBool, - reconnect_interval: Mutex, + full_reconnect: AtomicBool, // If true, the next reconnect attempt will skip resume and directly try a full reconnect + reconnect_interval: AsyncMutex, + reconnect_notifier: Arc, // Called when the reconnection task finisehd, successful or not } impl Debug for EngineInner { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { f.debug_struct("EngineInner") - .field("session_info", &self.session_info) - .field("opened", &self.opened) + .field("closed", &self.closed) .field("reconnecting", &self.reconnecting) .field("full_reconnect", &self.full_reconnect) .finish() @@ -130,93 +137,68 @@ pub struct RtcEngine { } impl RtcEngine { - pub fn new() -> (Self, EngineEvents) { - let (engine_emitter, engine_events) = mpsc::channel(8); - let inner = Arc::new(EngineInner { - lk_runtime: LkRuntime::instance(), - session_info: Default::default(), - running_handle: Default::default(), - opened: Default::default(), - engine_emitter, - reconnecting: Default::default(), - full_reconnect: Default::default(), - reconnect_interval: Mutex::new(interval(RECONNECT_INTERVAL)), - }); - - (Self { inner }, engine_events) - } - - #[tracing::instrument] pub async fn connect( - &self, url: &str, token: &str, options: SignalOptions, - ) -> EngineResult<()> { - self.inner.connect(url, token, options).await + ) -> EngineResult<(Self, EngineEvents)> { + let (engine_emitter, engine_events) = mpsc::channel(8); + + let mut reconnect_interval = interval(RECONNECT_INTERVAL); + reconnect_interval.set_missed_tick_behavior(MissedTickBehavior::Delay); + + let inner = Arc::new(EngineInner { + lk_runtime: LkRuntime::instance(), + running_handle: Default::default(), + engine_emitter, + join_response: Default::default(), // Will directly be replaced by the connect method below + closed: Default::default(), + reconnecting: Default::default(), + full_reconnect: Default::default(), + reconnect_interval: AsyncMutex::new(reconnect_interval), + reconnect_notifier: Arc::new(Notify::new()), + }); + + inner.connect(url, token, options).await?; + Ok((Self { inner }, engine_events)) } - #[tracing::instrument] pub async fn close(&self) { self.inner.close().await } - #[tracing::instrument(skip(data))] pub async fn publish_data( &self, data: &proto::DataPacket, - kind: proto::data_packet::Kind, + kind: DataPacketKind, ) -> EngineResult<()> { + // Make sure we are connected before trying to send data self.inner.wait_reconnection().await?; - self.inner - .running_handle - .read() - .await - .as_ref() - .unwrap() - .session - .publish_data(data, kind) - .await + let handle = self.inner.running_handle.read().await; + let session = &handle.as_ref().unwrap().session; + session.publish_data(data, kind).await } pub async fn simulate_scenario(&self, scenario: SimulateScenario) -> EngineResult<()> { self.inner.wait_reconnection().await?; - self.inner - .running_handle - .read() - .await - .as_ref() - .unwrap() - .session - .simulate_scenario(scenario) - .await; + let handle = self.inner.running_handle.read().await; + let session = &handle.as_ref().unwrap().session; + session.simulate_scenario(scenario).await; Ok(()) } pub async fn add_track(&self, req: proto::AddTrackRequest) -> EngineResult { self.inner.wait_reconnection().await?; - self.inner - .running_handle - .read() - .await - .as_ref() - .unwrap() - .session - .add_track(req) - .await + let handle = self.inner.running_handle.read().await; + let session = &handle.as_ref().unwrap().session; + session.add_track(req).await } pub async fn remove_track(&self, sender: RtpSender) -> EngineResult<()> { self.inner.wait_reconnection().await?; - self.inner - .running_handle - .read() - .await - .as_ref() - .unwrap() - .session - .remove_track(sender) - .await + let handle = self.inner.running_handle.read().await; + let session = &handle.as_ref().unwrap().session; + session.remove_track(sender).await } pub async fn create_sender( @@ -226,37 +208,33 @@ impl RtcEngine { encodings: Vec, ) -> EngineResult { self.inner.wait_reconnection().await?; - self.inner - .running_handle - .read() - .await - .as_ref() - .unwrap() - .session - .create_sender(track, options, encodings) - .await + let handle = self.inner.running_handle.read().await; + let session = &handle.as_ref().unwrap().session; + session.create_sender(track, options, encodings).await } pub async fn negotiate_publisher(&self) -> EngineResult<()> { // TODO(theomonnom): guard for reconnection self.inner.wait_reconnection().await?; - self.inner - .running_handle - .read() - .await - .as_ref() - .unwrap() - .session - .negotiate_publisher() - .await + let handle = self.inner.running_handle.read().await; + let session = &handle.as_ref().unwrap().session; + session.negotiate_publisher().await } - pub fn join_response(&self) -> Option { - if let Some(info) = self.inner.session_info.lock().as_ref() { - Some(info.join_response.clone()) - } else { - None + pub async fn send_request(&self, msg: proto::signal_request::Message) -> EngineResult<()> { + if self.inner.reconnecting.load(Ordering::Acquire) { + // When doing a full reconnect, it is safe to ignore the messages, we don't wait for reconnection here + return Ok(()); // TODO(theomonnom): Maybe we should still return an error instead? } + + let handle = self.inner.running_handle.read().await; + let session = &handle.as_ref().unwrap().session; // Unwrap should be OK here (running_handle is always valid when not reconnecting) + session.signal_client().send(msg).await; + Ok(()) + } + + pub fn join_response(&self) -> proto::JoinResponse { + self.inner.join_response.lock().clone() } } @@ -271,12 +249,12 @@ impl EngineInner { res = session_events.recv() => { if let Some(event) = res { if let Err(err) = self.on_session_event(event).await { - error!("failed to handle session event: {:?}", err); + log::error!("failed to handle session event: {:?}", err); } } }, _ = &mut close_receiver => { - trace!("closing engine task"); + log::trace!("closing engine task"); break; } } @@ -292,12 +270,12 @@ impl EngineInner { retry_now, full_reconnect, } => { - info!("received session close: {}, {:?}", source, reason); + log::info!("received session close: {}, {:?}", source, reason); if can_reconnect { - self.clone().try_reconnect(retry_now, full_reconnect); + self.try_reconnect(retry_now, full_reconnect); } else { // Spawning a new task because the close function wait for the engine_task to - // finish. + // finish. (Where this function is called from) tokio::spawn({ let inner = self.clone(); async move { @@ -363,26 +341,23 @@ impl EngineInner { token: &str, options: SignalOptions, ) -> EngineResult<()> { - let (session_emitter, session_events) = mpsc::unbounded_channel(); - let session = RtcSession::connect( - url, - token, - options, - self.lk_runtime.clone(), - session_emitter, - ) - .await?; + let mut running_handle = self.running_handle.write().await; + + let (session, join_response, session_events) = + RtcSession::connect(url, token, options).await?; let (close_sender, close_receiver) = oneshot::channel(); let engine_task = tokio::spawn(self.clone().engine_task(session_events, close_receiver)); - *self.session_info.lock() = Some(session.info().clone()); - *self.running_handle.write().await = Some(EngineHandle { + + let engine_handle = EngineHandle { session, engine_task, close_sender, - }); + }; - self.opened.store(true, Ordering::SeqCst); + // Always update the join response after a new session is created (first session or full reconnect) + *self.join_response.lock() = join_response; + *running_handle = Some(engine_handle); Ok(()) } @@ -395,7 +370,7 @@ impl EngineInner { } async fn close(&self) { - self.opened.store(false, Ordering::SeqCst); + self.closed.store(true, Ordering::Release); self.terminate_session().await; let _ = self.engine_emitter.send(EngineEvent::Disconnected).await; } @@ -403,14 +378,18 @@ impl EngineInner { // Wait for the reconnection task to finish // Return directly if no open RTCSession async fn wait_reconnection(&self) -> EngineResult<()> { - if !self.opened.load(Ordering::SeqCst) { - Err(EngineError::Connection("not opened".to_owned()))? + if self.closed.load(Ordering::SeqCst) { + Err(EngineError::Connection("engine is closed".to_owned()))? } - while self.reconnecting.load(Ordering::Acquire) { - tokio::task::yield_now().await; // TODO(theomonnom): Remove yield + if self.reconnecting.load(Ordering::Acquire) { + // If currently reconnecting, wait for the reconnect task to finish + self.reconnect_notifier.notified().await; } + // reconnect_task is finished here, so it is fine to try to read the RwLock here (should be a short lock) + // (the reconnection logic can lock the running_handle for a long time, e.g when resuming) + if self.running_handle.read().await.is_none() { Err(EngineError::Connection("reconnection failed".to_owned()))? } @@ -419,45 +398,64 @@ impl EngineInner { } /// Start the reconnect task if not already started - fn try_reconnect(self: Arc, retry_now: bool, full_reconnect: bool) { - if !self.opened.load(Ordering::SeqCst) { + /// Ask to retry directly if `retry_now` is true + /// Ask for a full reconnect if `full_reconnect` is true + fn try_reconnect(self: &Arc, retry_now: bool, full_reconnect: bool) { + if self.closed.load(Ordering::Acquire) { return; } if self.reconnecting.load(Ordering::SeqCst) { + let inner = self.clone(); if retry_now { - self.reconnect_interval.lock().reset(); - self.full_reconnect.store(full_reconnect, Ordering::SeqCst); + tokio::spawn(async move { + inner.reconnect_interval.lock().await.reset(); // Retry directly + }); + self.full_reconnect.store(full_reconnect, Ordering::Release); } return; } - warn!("reconnecting RTCEngine..."); + log::warn!("reconnecting RTCEngine..."); - self.reconnecting.store(true, Ordering::SeqCst); - self.full_reconnect.store(full_reconnect, Ordering::SeqCst); - self.reconnect_interval.lock().reset(); tokio::spawn({ let inner = self.clone(); async move { - let res = inner.reconnect_task().await; - inner.reconnecting.store(false, Ordering::SeqCst); + // Reconnetion logic + inner.reconnect_interval.lock().await.reset(); // Retry directly + inner.reconnecting.store(true, Ordering::Release); + inner + .full_reconnect + .store(full_reconnect, Ordering::Release); + + let res = inner.reconnect_task().await; // Wait for the reconnection task to finish + inner.reconnecting.store(false, Ordering::Release); if res.is_ok() { - warn!("RTCEngine successfully reconnected") + log::warn!("RTCEngine successfully reconnected") } else { - error!("failed to reconnect after {} attemps", RECONNECT_ATTEMPTS); + log::error!("failed to reconnect after {} attemps", RECONNECT_ATTEMPTS); inner.close().await; } + + inner.reconnect_notifier.notify_waiters(); } }); } - /// Called every time the PeerConnection or the SignalClient is closed + /// Runned every time the PeerConnection or the SignalClient is closed /// We first try to resume the connection, if it fails, we start a full reconnect. async fn reconnect_task(self: &Arc) -> EngineResult<()> { + // Get the latest connection info from the signal_client (including the refreshed token because the initial join token may have expired) + let running_handle = self.running_handle.read().await; + let signal_client = running_handle.as_ref().unwrap().session.signal_client(); + let url = signal_client.url(); + let token = signal_client.token(); + let options = signal_client.options(); + drop(running_handle); + for i in 0..RECONNECT_ATTEMPTS { - if !self.opened.load(Ordering::Acquire) { + if self.closed.load(Ordering::Acquire) { // The user closed the RTCEngine, cancel the reconnection task return Ok(()); } @@ -467,9 +465,12 @@ impl EngineInner { let _ = self.engine_emitter.send(EngineEvent::Restarting).await; } - info!("restarting connection... attempt: {}", i); - if let Err(err) = self.try_restart_connection().await { - error!("restarting connection failed: {}", err); + log::info!("restarting connection... attempt: {}", i); + if let Err(err) = self + .try_restart_connection(&url, &token, options.clone()) + .await + { + log::error!("restarting connection failed: {}", err); } else { let _ = self.engine_emitter.send(EngineEvent::Restarted).await; return Ok(()); @@ -479,9 +480,9 @@ impl EngineInner { let _ = self.engine_emitter.send(EngineEvent::Resuming).await; } - info!("resuming connection... attempt: {}", i); + log::info!("resuming connection... attempt: {}", i); if let Err(err) = self.try_resume_connection().await { - error!("resuming connection failed: {}", err); + log::error!("resuming connection failed: {}", err); if let EngineError::Signal(_) = err { self.full_reconnect.store(true, Ordering::SeqCst); } @@ -491,34 +492,33 @@ impl EngineInner { } } - self.reconnect_interval.lock().tick().await; + self.reconnect_interval.lock().await.tick().await; } Err(EngineError::Connection("failed to reconnect".to_owned())) } /// Try to recover the connection by doing a full reconnect. - /// It recreates a new RTCSession - async fn try_restart_connection(self: &Arc) -> EngineResult<()> { - let info = self.session_info.lock().clone().unwrap(); + /// It recreates a new RtcSession + async fn try_restart_connection( + self: &Arc, + url: &str, + token: &str, + options: SignalOptions, + ) -> EngineResult<()> { self.terminate_session().await; - self.connect(&info.url, &info.token, info.options).await?; - self.running_handle - .read() - .await - .as_ref() - .unwrap() - .session - .wait_pc_connection() - .await + self.connect(url, token, options).await?; - // TODO(theomonnom): Resend SignalClient queue + let handle = self.running_handle.read().await; + let session = &handle.as_ref().unwrap().session; + session.wait_pc_connection().await } /// Try to restart the current session async fn try_resume_connection(&self) -> EngineResult<()> { let handle = self.running_handle.read().await; - handle.as_ref().unwrap().session.restart().await?; - handle.as_ref().unwrap().session.wait_pc_connection().await + let session = &handle.as_ref().unwrap().session; + session.restart().await?; + session.wait_pc_connection().await } } diff --git a/livekit/src/rtc_engine/peer_transport.rs b/livekit/src/rtc_engine/peer_transport.rs index 9ad8ba4..eea0da8 100644 --- a/livekit/src/rtc_engine/peer_transport.rs +++ b/livekit/src/rtc_engine/peer_transport.rs @@ -1,8 +1,8 @@ use livekit_protocol as proto; use livekit_webrtc::prelude::*; +use log::{debug, error}; use std::fmt::{Debug, Formatter}; use std::time::Duration; -use tracing::{event, Level}; const _NEGOTIATION_FREQUENCY: Duration = Duration::from_millis(150); @@ -64,7 +64,6 @@ impl PeerTransport { self.peer_connection.close(); } - #[tracing::instrument(level = Level::DEBUG)] pub async fn add_ice_candidate(&mut self, ice_candidate: IceCandidate) -> Result<(), RtcError> { if self.peer_connection.current_remote_description().is_some() && !self.restarting_ice { self.peer_connection @@ -78,7 +77,6 @@ impl PeerTransport { Ok(()) } - #[tracing::instrument(level = Level::DEBUG)] pub async fn set_remote_description( &mut self, remote_description: SessionDescription, @@ -100,13 +98,11 @@ impl PeerTransport { Ok(()) } - #[tracing::instrument(level = Level::DEBUG)] pub async fn negotiate(&mut self) -> Result<(), RtcError> { // TODO(theomonnom) Debounce here with NEGOTIATION_FREQUENCY self.create_and_send_offer(OfferOptions::default()).await } - #[tracing::instrument(level = Level::DEBUG)] pub async fn create_anwser( &mut self, offer: SessionDescription, @@ -121,14 +117,13 @@ impl PeerTransport { Ok(answer) } - #[tracing::instrument(level = Level::DEBUG)] pub async fn create_and_send_offer(&mut self, options: OfferOptions) -> Result<(), RtcError> { if self.on_offer_handler.is_none() { return Ok(()); } if options.ice_restart { - event!(Level::TRACE, "restarting ICE"); + debug!("restarting ICE"); self.restarting_ice = true; } @@ -140,10 +135,7 @@ impl PeerTransport { .set_remote_description(remote_description) .await?; } else { - event!( - Level::ERROR, - "trying to restart ICE when the pc doesn't have remote description" - ); + error!("trying to restart ICE when the pc doesn't have remote description"); } } else { self.renegotiate = true; diff --git a/livekit/src/rtc_engine/rtc_events.rs b/livekit/src/rtc_engine/rtc_events.rs index 97b811c..2cb85e3 100644 --- a/livekit/src/rtc_engine/rtc_events.rs +++ b/livekit/src/rtc_engine/rtc_events.rs @@ -2,8 +2,8 @@ use super::peer_transport::PeerTransport; use crate::rtc_engine::peer_transport::OnOfferCreated; use livekit_protocol as proto; use livekit_webrtc::{self as rtc, prelude::*}; +use log::error; use tokio::sync::mpsc; -use tracing::{error}; pub type RtcEmitter = mpsc::UnboundedSender; pub type RtcEvents = mpsc::UnboundedReceiver; diff --git a/livekit/src/rtc_engine/rtc_session.rs b/livekit/src/rtc_engine/rtc_session.rs index 599684c..e31b926 100644 --- a/livekit/src/rtc_engine/rtc_session.rs +++ b/livekit/src/rtc_engine/rtc_session.rs @@ -4,9 +4,9 @@ use crate::prelude::TrackKind; use crate::rtc_engine::lk_runtime::LkRuntime; use crate::rtc_engine::peer_transport::PeerTransport; use crate::rtc_engine::rtc_events::{RtcEvent, RtcEvents}; -use crate::signal_client; use crate::signal_client::{SignalClient, SignalEvent, SignalEvents, SignalOptions}; use crate::track::LocalTrack; +use crate::DataPacketKind; use livekit_protocol as proto; use livekit_webrtc::prelude::*; use parking_lot::Mutex; @@ -22,7 +22,6 @@ use tokio::sync::Mutex as AsyncMutex; use tokio::sync::{mpsc, oneshot, watch}; use tokio::task::JoinHandle; use tokio::time::sleep; -use tracing::{debug, error, trace, warn}; pub const ICE_CONNECT_TIMEOUT: Duration = Duration::from_secs(15); pub const TRACK_PUBLISH_TIMEOUT: Duration = Duration::from_secs(10); @@ -40,7 +39,7 @@ pub enum SessionEvent { Data { participant_sid: String, payload: Vec, - kind: proto::data_packet::Kind, + kind: DataPacketKind, }, MediaTrack { track: MediaStreamTrack, @@ -96,19 +95,10 @@ struct IceCandidateJson { pub candidate: String, } -#[derive(Debug, Clone, Default)] -pub struct SessionInfo { - pub url: String, - pub token: String, - pub options: SignalOptions, - pub join_response: proto::JoinResponse, -} - /// Fields shared with rtc_task and signal_task struct SessionInner { - info: SessionInfo, signal_client: Arc, - pc_state: AtomicU8, // PCState + pc_state: AtomicU8, // PcState has_published: AtomicBool, publisher_pc: AsyncMutex, @@ -117,7 +107,7 @@ struct SessionInner { pending_tracks: Mutex>>, // Publisher data channels - // used to send data to other participants ( The SFU forwards the messages ) + // used to send data to other participants (The SFU forwards the messages) lossy_dc: DataChannel, reliable_dc: DataChannel, @@ -132,7 +122,6 @@ struct SessionInner { impl Debug for SessionInner { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("SessionInner") - .field("info", &self.info) .field("pc_state", &self.pc_state) .field("has_published", &self.has_published) .field("closed", &self.closed) @@ -146,8 +135,6 @@ impl Debug for SessionInner { /// RTCSession is also responsable for the signaling and the negotation #[derive(Debug)] pub struct RtcSession { - #[allow(dead_code)] - lk_runtime: Arc, inner: Arc, close_tx: watch::Sender, // false = is_running signal_task: JoinHandle<()>, @@ -159,15 +146,13 @@ impl RtcSession { url: &str, token: &str, options: SignalOptions, - lk_runtime: Arc, - session_emitter: SessionEmitter, - ) -> EngineResult { - // Connect to the SignalClient - let (signal_client, mut signal_events) = SignalClient::new(); + ) -> EngineResult<(Self, proto::JoinResponse, SessionEvents)> { + let (session_emitter, session_events) = mpsc::unbounded_channel(); + + let (signal_client, join_response, signal_events) = + SignalClient::connect(url, token, options).await?; let signal_client = Arc::new(signal_client); - signal_client.connect(url, token, options.clone()).await?; - let join_response = signal_client::utils::next_join_response(&mut signal_events).await?; - debug!("received JoinResponse: {:?}", join_response); + log::debug!("received JoinResponse: {:?}", join_response); let (rtc_emitter, rtc_events) = mpsc::unbounded_channel(); let rtc_config = RtcConfiguration { @@ -186,6 +171,7 @@ impl RtcSession { ice_transport_type: IceTransportsType::All, }; + let lk_runtime = LkRuntime::instance(); let mut publisher_pc = PeerTransport::new( lk_runtime .pc_factory() @@ -223,16 +209,8 @@ impl RtcSession { rtc_events::forward_dc_events(&mut lossy_dc, rtc_emitter.clone()); rtc_events::forward_dc_events(&mut reliable_dc, rtc_emitter.clone()); - let session_info = SessionInfo { - url: url.to_owned(), - token: token.to_owned(), - options, - join_response, - }; - let (close_tx, close_rx) = watch::channel(false); let inner = Arc::new(SessionInner { - info: session_info, pc_state: AtomicU8::new(PeerState::New as u8), has_published: Default::default(), signal_client, @@ -250,19 +228,14 @@ impl RtcSession { let signal_task = tokio::spawn(inner.clone().signal_task(signal_events, close_rx.clone())); let rtc_task = tokio::spawn(inner.clone().rtc_session_task(rtc_events, close_rx.clone())); - if !inner.info.join_response.subscriber_primary { - inner.negotiate_publisher().await?; - } - let session = Self { - lk_runtime, inner: inner.clone(), close_tx, signal_task, rtc_task, }; - Ok(session) + Ok((session, join_response, session_events)) } #[inline] @@ -291,7 +264,6 @@ impl RtcSession { } /// Close the PeerConnections and the SignalClient - #[tracing::instrument] pub async fn close(self) { // Close the tasks self.inner.close().await; @@ -304,7 +276,7 @@ impl RtcSession { pub async fn publish_data( &self, data: &proto::DataPacket, - kind: proto::data_packet::Kind, + kind: DataPacketKind, ) -> Result<(), EngineError> { self.inner.publish_data(data, kind).await } @@ -324,11 +296,6 @@ impl RtcSession { self.inner.simulate_scenario(scenario).await } - #[inline] - pub fn info(&self) -> &SessionInfo { - &self.inner.info - } - #[allow(dead_code)] #[inline] pub fn state(&self) -> PeerState { @@ -359,7 +326,7 @@ impl RtcSession { #[allow(dead_code)] #[inline] - pub fn data_channel(&self, kind: proto::data_packet::Kind) -> &DataChannel { + pub fn data_channel(&self, kind: DataPacketKind) -> &DataChannel { &self.inner.data_channel(kind) } } @@ -375,11 +342,11 @@ impl SessionInner { res = rtc_events.recv() => { if let Some(event) = res { if let Err(err) = self.on_rtc_event(event).await { - error!("failed to handle rtc event: {:?}", err); + log::error!("failed to handle rtc event: {:?}", err); } } }, _ = close_rx.changed() => { - trace!("closing rtc_session_task"); + log::trace!("closing rtc_session_task"); break; } } @@ -399,7 +366,7 @@ impl SessionInner { SignalEvent::Open => {} SignalEvent::Signal(signal) => { if let Err(err) = self.on_signal_event(signal).await { - error!("failed to handle signal: {:?}", err); + log::error!("failed to handle signal: {:?}", err); } } SignalEvent::Close => { @@ -415,7 +382,7 @@ impl SessionInner { } }, _ = close_rx.changed() => { - trace!("closing signal_task"); + log::trace!("closing signal_task"); break; } } @@ -425,7 +392,7 @@ impl SessionInner { async fn on_signal_event(&self, event: proto::signal_response::Message) -> EngineResult<()> { match event { proto::signal_response::Message::Answer(answer) => { - trace!("received publisher answer: {:?}", answer); + log::debug!("received publisher answer: {:?}", answer); let answer = SessionDescription::parse(&answer.sdp, answer.r#type.parse().unwrap())?; self.publisher_pc @@ -435,7 +402,7 @@ impl SessionInner { .await?; } proto::signal_response::Message::Offer(offer) => { - trace!("received subscriber offer: {:?}", offer); + log::debug!("received subscriber offer: {:?}", offer); let offer = SessionDescription::parse(&offer.sdp, offer.r#type.parse().unwrap())?; let answer = self .subscriber_pc @@ -460,7 +427,7 @@ impl SessionInner { IceCandidate::parse(&json.sdp_mid, json.sdp_m_line_index, &json.candidate)? }; - debug!("received ice_candidate {:?} {:?}", target, ice_candidate); + log::debug!("received ice_candidate {:?} {:?}", target, ice_candidate); if target == proto::SignalTarget::Publisher { self.publisher_pc @@ -478,7 +445,7 @@ impl SessionInner { } proto::signal_response::Message::Leave(leave) => { self.on_session_disconnected( - "received leave", + "server request to leave", leave.reason(), leave.can_reconnect, true, @@ -533,11 +500,12 @@ impl SessionInner { .await; } RtcEvent::ConnectionChange { state, target } => { - debug!("connection change, {:?} {:?}", state, target); - let is_primary = self.info.join_response.subscriber_primary - && target == proto::SignalTarget::Subscriber; + log::debug!("connection change, {:?} {:?}", state, target); - if is_primary && state == PeerConnectionState::Connected { + // The subscriber is always the primary peer connection + if target == proto::SignalTarget::Subscriber + && state == PeerConnectionState::Connected + { let old_state = self .pc_state .swap(PeerState::Connected as u8, Ordering::SeqCst); @@ -565,7 +533,7 @@ impl SessionInner { } RtcEvent::Offer { offer, target: _ } => { // Send the publisher offer to the server - debug!("sending publisher offer: {:?}", offer); + log::debug!("sending publisher offer: {:?}", offer); self.signal_client .send(proto::signal_request::Message::Offer( proto::SessionDescription { @@ -589,7 +557,7 @@ impl SessionInner { receiver, }); } else { - warn!("Track event with no streams"); + log::warn!("Track event with no streams"); } } RtcEvent::Data { data, binary } => { @@ -605,7 +573,9 @@ impl SessionInner { let _ = self.emitter.send(SessionEvent::Data { participant_sid: user.participant_sid, payload: user.payload, - kind: proto::data_packet::Kind::from_i32(data.kind).unwrap(), + kind: proto::data_packet::Kind::from_i32(data.kind) + .unwrap() + .into(), }); } proto::data_packet::Value::Speaker(_) => {} @@ -735,7 +705,6 @@ impl SessionInner { }); } - #[tracing::instrument] async fn close(&self) { self.closed.store(true, Ordering::Release); self.signal_client.close().await; @@ -743,7 +712,6 @@ impl SessionInner { self.subscriber_pc.lock().await.close(); } - #[tracing::instrument] async fn simulate_scenario(&self, scenario: SimulateScenario) { match scenario { SimulateScenario::SignalReconnect => { @@ -814,11 +782,10 @@ impl SessionInner { } } - #[tracing::instrument(skip(data))] async fn publish_data( &self, data: &proto::DataPacket, - kind: proto::data_packet::Kind, + kind: DataPacketKind, ) -> Result<(), EngineError> { self.ensure_publisher_connected(kind).await?; self.data_channel(kind) @@ -827,18 +794,9 @@ impl SessionInner { } /// Try to restart the session by doing an ICE Restart (The SignalClient is also restarted) - /// This reconnection if more seemless than the full reconnection implemented in ['RTCEngine'] + /// This reconnection if more seemless compared to the full reconnection implemented in ['RTCEngine'] async fn restart_session(&self) -> EngineResult<()> { - self.signal_client.close().await; - - let mut options = self.info.options.clone(); - options.sid = self.info.join_response.participant.clone().unwrap().sid; - options.reconnect = true; - - self.signal_client - .connect(&self.info.url, &self.info.token, options) - .await?; - + self.signal_client.restart().await?; self.subscriber_pc.lock().await.prepare_ice_restart(); if self.has_published.load(Ordering::Acquire) { @@ -854,11 +812,10 @@ impl SessionInner { self.wait_pc_connection().await?; self.signal_client.flush_queue().await; - Ok(()) } - // Wait for PCState to become PCState::Connected + // Wait for PeerState to become PeerState::Connected // Timeout after ['MAX_ICE_CONNECT_TIMEOUT'] async fn wait_pc_connection(&self) -> EngineResult<()> { let wait_connected = async move { @@ -867,7 +824,7 @@ impl SessionInner { return Err(EngineError::Connection("closed".to_string())); } - tokio::task::yield_now().await; + tokio::time::sleep(Duration::from_millis(50)).await; } Ok(()) @@ -887,18 +844,14 @@ impl SessionInner { self.has_published.store(true, Ordering::Release); let res = self.publisher_pc.lock().await.negotiate().await; if let Err(err) = &res { - error!("failed to negotiate the publisher: {:?}", err); + log::error!("failed to negotiate the publisher: {:?}", err); } res.map_err(Into::into) } /// Ensure the Publisher PC is connected, if not, start the negotiation /// This is required when sending data to the server - async fn ensure_publisher_connected(&self, kind: proto::data_packet::Kind) -> EngineResult<()> { - if !self.info.join_response.subscriber_primary { - return Ok(()); - } - + async fn ensure_publisher_connected(&self, kind: DataPacketKind) -> EngineResult<()> { if !self.publisher_pc.lock().await.is_connected() && self .publisher_pc @@ -923,7 +876,7 @@ impl SessionInner { return Err(EngineError::Connection("closed".to_string())); } - tokio::task::yield_now().await; + tokio::time::sleep(Duration::from_millis(50)).await; } Ok(()) @@ -933,14 +886,14 @@ impl SessionInner { res = wait_connected => res, _ = sleep(ICE_CONNECT_TIMEOUT) => { let err = EngineError::Connection("could not establish publisher connection: timeout".to_string()); - error!(error = ?err); + log::error!("{}", err); Err(err) } } } - fn data_channel(&self, kind: proto::data_packet::Kind) -> &DataChannel { - if kind == proto::data_packet::Kind::Reliable { + fn data_channel(&self, kind: DataPacketKind) -> &DataChannel { + if kind == DataPacketKind::Reliable { &self.reliable_dc } else { &self.lossy_dc diff --git a/livekit/src/signal_client/mod.rs b/livekit/src/signal_client/mod.rs index b429435..20f9cc3 100644 --- a/livekit/src/signal_client/mod.rs +++ b/livekit/src/signal_client/mod.rs @@ -1,13 +1,12 @@ use crate::signal_client::signal_stream::SignalStream; use livekit_protocol as proto; - -use parking_lot::RwLock; +use parking_lot::Mutex; use std::fmt::Debug; use std::time::Duration; use thiserror::Error; use tokio::sync::mpsc; +use tokio::sync::RwLock as AsyncRwLock; use tokio_tungstenite::tungstenite::Error as WsError; -use tracing::{instrument, Level}; mod signal_stream; @@ -16,9 +15,12 @@ pub(crate) type SignalEvents = mpsc::Receiver; pub(crate) type SignalResult = Result; pub const JOIN_RESPONSE_TIMEOUT: Duration = Duration::from_secs(5); +pub const PROTOCOL_VERSION: u32 = 8; #[derive(Error, Debug)] pub enum SignalError { + #[error("already connected")] + AlreadyConnected, #[error("ws failure: {0}")] WsError(#[from] WsError), #[error("failed to parse the url")] @@ -39,8 +41,6 @@ pub enum SignalEvent { #[derive(Debug, Clone)] pub struct SignalOptions { - pub(crate) reconnect: bool, - pub(crate) sid: String, pub auto_subscribe: bool, pub adaptive_stream: bool, } @@ -48,9 +48,7 @@ pub struct SignalOptions { impl Default for SignalOptions { fn default() -> Self { Self { - reconnect: false, auto_subscribe: true, - sid: "".to_string(), adaptive_stream: false, } } @@ -58,100 +56,143 @@ impl Default for SignalOptions { #[derive(Debug)] pub struct SignalClient { - stream: RwLock>, + stream: AsyncRwLock>, + url: String, + token: Mutex, // TODO(theomonnom): Handle token refresh + join_response: proto::JoinResponse, + options: SignalOptions, emitter: SignalEmitter, } impl SignalClient { - pub fn new() -> (Self, SignalEvents) { - let (emitter, events) = mpsc::channel(8); - ( - Self { - stream: Default::default(), - emitter, - }, - events, - ) - } - - #[instrument(level = Level::DEBUG, skip(url, token, options))] pub async fn connect( - &self, url: &str, token: &str, options: SignalOptions, - ) -> SignalResult<()> { - let stream = SignalStream::connect(url, token, options, self.emitter.clone()).await?; - *self.stream.write() = Some(stream); + ) -> SignalResult<(Self, proto::JoinResponse, SignalEvents)> { + let (emitter, mut events) = mpsc::channel(8); + let lk_url = get_livekit_url(url, token, &options)?; + let new_stream = SignalStream::connect(lk_url, emitter.clone()).await?; + let join_response = get_join_response(&mut events).await?; + + Ok(( + Self { + stream: AsyncRwLock::new(Some(new_stream)), + url: url.to_string(), + token: Mutex::new(token.to_string()), + join_response: join_response.clone(), + options, + emitter, + }, + join_response, + events, + )) + } + + // Restart is called when trying to resume the room (RtcSession resume) + // TODO(theomonom): Should this be renamed to resume? + pub async fn restart(&self) -> SignalResult<()> { + self.close().await; + + let sid = &self.join_response.participant.as_ref().unwrap().sid; + let token = self.token.lock().clone(); + + let mut lk_url = get_livekit_url(&self.url, &token, &self.options)?; + lk_url + .query_pairs_mut() + .append_pair("reconnect", "1") + .append_pair("sid", sid); + + let new_stream = SignalStream::connect(lk_url, self.emitter.clone()).await?; + *self.stream.write().await = Some(new_stream); Ok(()) } - #[instrument(level = Level::DEBUG)] pub async fn close(&self) { - if let Some(stream) = self.stream.write().take() { + if let Some(stream) = self.stream.write().await.take() { stream.close().await; } } - #[instrument(level = Level::DEBUG)] pub async fn send(&self, signal: proto::signal_request::Message) { - if let Some(stream) = self.stream.read().as_ref() { + // TODO: Check if currently reconnecting and queue message + + if let Some(stream) = self.stream.read().await.as_ref() { if stream.send(signal).await.is_ok() { return; } } - - // TODO(theomonnom): enqueue message + // TODO(theomonnom): return result? } - /*#[allow(dead_code)] + #[allow(dead_code)] pub async fn clear_queue(&self) { - // TODO(theomonnom): impl - }*/ + // TODO(theomonnom): Clear the queue + } - #[instrument(level = Level::DEBUG)] pub async fn flush_queue(&self) { - // TODO(theomonnom): impl + // TODO(theomonnom): Send the queue + } + + pub fn join_response(&self) -> proto::JoinResponse { + self.join_response.clone() + } + + pub fn options(&self) -> SignalOptions { + self.options.clone() + } + + pub fn url(&self) -> String { + self.url.clone() + } + + pub fn token(&self) -> String { + self.token.lock().clone() } } -pub mod utils { - use crate::signal_client::{SignalError, SignalEvent, SignalResult, JOIN_RESPONSE_TIMEOUT}; - use livekit_protocol as proto; - use tokio::time::timeout; - use tokio_tungstenite::tungstenite::Error as WsError; - use tracing::{event, instrument, Level}; +fn get_livekit_url(url: &str, token: &str, options: &SignalOptions) -> SignalResult { + let mut lk_url = url::Url::parse(url)?; + lk_url.set_path("/rtc"); + lk_url + .query_pairs_mut() + .append_pair("access_token", token) + .append_pair("protocol", PROTOCOL_VERSION.to_string().as_str()) + .append_pair( + "auto_subscribe", + if options.auto_subscribe { "1" } else { "0" }, + ) + .append_pair( + "adaptive_stream", + if options.adaptive_stream { "1" } else { "0" }, + ); - use super::SignalEvents; + Ok(lk_url) +} - #[instrument(level = Level::DEBUG, skip(receiver))] - pub(crate) async fn next_join_response( - receiver: &mut SignalEvents, - ) -> SignalResult { - let join = async { - while let Some(event) = receiver.recv().await { - match event { - SignalEvent::Signal(proto::signal_response::Message::Join(join)) => { - return Ok(join) - } - SignalEvent::Close => break, - SignalEvent::Open => continue, - _ => { - event!( - Level::WARN, - "received unexpected message while waiting for JoinResponse: {:?}", - event - ); - continue; - } +async fn get_join_response(receiver: &mut SignalEvents) -> SignalResult { + let join = async { + while let Some(event) = receiver.recv().await { + match event { + SignalEvent::Signal(proto::signal_response::Message::Join(join)) => { + return Ok(join) + } + SignalEvent::Close => break, + SignalEvent::Open => continue, + _ => { + log::warn!( + "received unexpected message while waiting for JoinResponse: {:?}", + event + ); + continue; } } + } - Err(WsError::ConnectionClosed)? - }; + Err(WsError::ConnectionClosed)? + }; - timeout(JOIN_RESPONSE_TIMEOUT, join) - .await - .map_err(|_| SignalError::Timeout("failed to receive JoinResponse".to_string()))? - } + tokio::time::timeout(JOIN_RESPONSE_TIMEOUT, join) + .await + .map_err(|_| SignalError::Timeout("failed to receive JoinResponse".to_string()))? } diff --git a/livekit/src/signal_client/signal_stream.rs b/livekit/src/signal_client/signal_stream.rs index 2877e3e..4e85783 100644 --- a/livekit/src/signal_client/signal_stream.rs +++ b/livekit/src/signal_client/signal_stream.rs @@ -1,4 +1,4 @@ -use crate::signal_client::{SignalEmitter, SignalEvent, SignalOptions, SignalResult}; +use crate::signal_client::{SignalEmitter, SignalEvent, SignalResult}; use futures_util::stream::{SplitSink, SplitStream}; use futures_util::{SinkExt, StreamExt}; use livekit_protocol as proto; @@ -10,9 +10,8 @@ use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode; use tokio_tungstenite::tungstenite::protocol::CloseFrame; use tokio_tungstenite::tungstenite::Message; use tokio_tungstenite::{connect_async, MaybeTlsStream, WebSocketStream}; -use tracing::{event, Level}; -pub const PROTOCOL_VERSION: u32 = 8; +use super::SignalEvents; type WebSocket = WebSocketStream>; @@ -46,31 +45,10 @@ impl SignalStream { /// /// SignalStream will never try to reconnect if the connection has been /// closed. - pub async fn connect( - url: &str, - token: &str, - options: SignalOptions, - emitter: SignalEmitter, - ) -> SignalResult { - let mut lk_url = url::Url::parse(url)?; - lk_url.set_path("/rtc"); - lk_url - .query_pairs_mut() - .append_pair("access_token", token) - .append_pair("protocol", PROTOCOL_VERSION.to_string().as_str()) - .append_pair("reconnect", if options.reconnect { "1" } else { "0" }) - .append_pair("sid", &options.sid) - .append_pair( - "auto_subscribe", - if options.auto_subscribe { "1" } else { "0" }, - ) - .append_pair( - "adaptive_stream", - if options.adaptive_stream { "1" } else { "0" }, - ); + pub async fn connect(url: url::Url, emitter: SignalEmitter) -> SignalResult { + log::info!("connecting to SignalClient: {}", url); - event!(Level::INFO, "connecting to SignalClient: {}", lk_url); - let (ws_stream, _) = connect_async(lk_url).await?; + let (ws_stream, _) = connect_async(url).await?; let _ = emitter.send(SignalEvent::Open).await; let (ws_writer, ws_reader) = ws_stream.split(); @@ -128,7 +106,7 @@ impl SignalStream { signal, response_chn, } => { - event!(Level::TRACE, "sending SignalRequest: {:?}", signal); + log::debug!("sending SignalRequest: {:?}", signal); let data = Message::Binary( proto::SignalRequest { @@ -138,7 +116,7 @@ impl SignalStream { ); if let Err(err) = ws_writer.send(data).await { - event!(Level::ERROR, "failed to send signal: {:?}", err); + log::error!("failed to send signal: {:?}", err); let _ = response_chn.send(Err(err.into())); break; } @@ -147,7 +125,7 @@ impl SignalStream { } InternalMessage::Pong { ping_data } => { if let Err(err) = ws_writer.send(Message::Pong(ping_data)).await { - event!(Level::ERROR, "failed to send pong message: {:?}", err); + log::error!("failed to send pong message: {:?}", err); } } InternalMessage::Close { close_frame } => { @@ -180,7 +158,7 @@ impl SignalStream { .expect("failed to decode SignalResponse"); let msg = res.message.unwrap(); - event!(Level::TRACE, "received SignalResponse: {:?}", msg); + log::debug!("received SignalResponse: {:?}", msg); let _ = emitter.send(SignalEvent::Signal(msg)).await; } Ok(Message::Ping(data)) => { @@ -190,11 +168,11 @@ impl SignalStream { continue; } Ok(Message::Close(close)) => { - event!(Level::DEBUG, "server closed the connection: {:?}", close); + log::debug!("server closed the connection: {:?}", close); break; } _ => { - event!(Level::ERROR, "unhandled websocket message {:?}", msg); + log::error!("unhandled websocket message {:?}", msg); break; } } diff --git a/webrtc-sys/include/livekit/audio_track.h b/webrtc-sys/include/livekit/audio_track.h index 34e6f48..0d08de6 100644 --- a/webrtc-sys/include/livekit/audio_track.h +++ b/webrtc-sys/include/livekit/audio_track.h @@ -18,6 +18,7 @@ #include +#include "api/audio_options.h" #include "livekit/helper.h" #include "livekit/media_stream_track.h" #include "livekit/webrtc.h" @@ -78,7 +79,7 @@ std::shared_ptr new_native_audio_sink( class AudioTrackSource { class InternalSource : public webrtc::LocalAudioSource { public: - InternalSource(); + InternalSource(const cricket::AudioOptions& options); SourceState state() const override; bool remote() const override; @@ -88,6 +89,8 @@ class AudioTrackSource { void AddSink(webrtc::AudioTrackSinkInterface* sink) override; void RemoveSink(webrtc::AudioTrackSinkInterface* sink) override; + void set_options(const cricket::AudioOptions& options); + // AudioFrame should always contain 10 ms worth of data (see index.md of // acm) void on_captured_frame(rust::Slice audio_data, @@ -96,13 +99,17 @@ class AudioTrackSource { size_t number_of_frames); private: - webrtc::Mutex mutex_; + mutable webrtc::Mutex mutex_; std::vector sinks_; cricket::AudioOptions options_{}; }; public: - AudioTrackSource(); + AudioTrackSource(AudioSourceOptions options); + + AudioSourceOptions audio_options() const; + + void set_audio_options(const AudioSourceOptions& options) const; void on_captured_frame(rust::Slice audio_data, int sample_rate, @@ -115,7 +122,8 @@ class AudioTrackSource { rtc::scoped_refptr source_; }; -std::shared_ptr new_audio_track_source(); +std::shared_ptr new_audio_track_source( + AudioSourceOptions options); static std::shared_ptr audio_to_media( std::shared_ptr track) { diff --git a/webrtc-sys/include/livekit/video_track.h b/webrtc-sys/include/livekit/video_track.h index 75ce556..f60c3dd 100644 --- a/webrtc-sys/include/livekit/video_track.h +++ b/webrtc-sys/include/livekit/video_track.h @@ -87,23 +87,29 @@ std::shared_ptr new_native_video_sink( class VideoTrackSource { class InternalSource : public rtc::AdaptedVideoTrackSource { public: - InternalSource(); + InternalSource(const VideoResolution& + resolution); // (0, 0) means no resolution/optional, the + // source will guess the resolution at the + // first captured frame ~InternalSource() override; bool is_screencast() const override; absl::optional needs_denoising() const override; SourceState state() const override; bool remote() const override; - + VideoResolution video_resolution() const; bool on_captured_frame(const webrtc::VideoFrame& frame); private: - webrtc::Mutex mutex_; + mutable webrtc::Mutex mutex_; rtc::TimestampAligner timestamp_aligner_; + VideoResolution resolution_; }; public: - VideoTrackSource(); + VideoTrackSource(const VideoResolution& resolution); + + VideoResolution video_resolution() const; bool on_captured_frame(const std::unique_ptr& frame) const; // frames pushed from Rust (+interior mutability) @@ -114,7 +120,8 @@ class VideoTrackSource { rtc::scoped_refptr source_; }; -std::shared_ptr new_video_track_source(); +std::shared_ptr new_video_track_source( + const VideoResolution& resolution); static std::shared_ptr video_to_media( std::shared_ptr track) { diff --git a/webrtc-sys/src/audio_track.cpp b/webrtc-sys/src/audio_track.cpp index 989822b..eff92a4 100644 --- a/webrtc-sys/src/audio_track.cpp +++ b/webrtc-sys/src/audio_track.cpp @@ -20,6 +20,7 @@ #include #include +#include "api/audio_options.h" #include "api/media_stream_interface.h" #include "audio/remix_resample.h" #include "common_audio/include/audio_util.h" @@ -28,9 +29,28 @@ #include "rtc_base/synchronization/mutex.h" #include "rtc_base/time_utils.h" #include "rust/cxx.h" +#include "webrtc-sys/src/audio_track.rs.h" namespace livekit { +inline cricket::AudioOptions to_native_audio_options( + const AudioSourceOptions& options) { + cricket::AudioOptions rtc_options{}; + rtc_options.echo_cancellation = options.echo_cancellation; + rtc_options.noise_suppression = options.noise_suppression; + rtc_options.auto_gain_control = options.auto_gain_control; + return rtc_options; +} + +inline AudioSourceOptions to_rust_audio_options( + const cricket::AudioOptions& rtc_options) { + AudioSourceOptions options{}; + options.echo_cancellation = rtc_options.echo_cancellation.value_or(false); + options.noise_suppression = rtc_options.noise_suppression.value_or(false); + options.auto_gain_control = rtc_options.auto_gain_control.value_or(false); + return options; +} + AudioTrack::AudioTrack(std::shared_ptr rtc_runtime, rtc::scoped_refptr track) : MediaStreamTrack(rtc_runtime, std::move(track)) {} @@ -74,11 +94,8 @@ std::shared_ptr new_native_audio_sink( return std::make_shared(std::move(observer)); } -AudioTrackSource::InternalSource::InternalSource() { - options_.echo_cancellation = false; - options_.auto_gain_control = false; - options_.noise_suppression = false; -} +AudioTrackSource::InternalSource::InternalSource( + const cricket::AudioOptions& options) {} webrtc::MediaSourceInterface::SourceState AudioTrackSource::InternalSource::state() const { @@ -90,9 +107,16 @@ bool AudioTrackSource::InternalSource::remote() const { } const cricket::AudioOptions AudioTrackSource::InternalSource::options() const { + webrtc::MutexLock lock(&mutex_); return options_; } +void AudioTrackSource::InternalSource::set_options( + const cricket::AudioOptions& options) { + webrtc::MutexLock lock(&mutex_); + options_ = options; +} + void AudioTrackSource::InternalSource::AddSink( webrtc::AudioTrackSinkInterface* sink) { webrtc::MutexLock lock(&mutex_); @@ -117,8 +141,18 @@ void AudioTrackSource::InternalSource::on_captured_frame( } } -AudioTrackSource::AudioTrackSource() { - source_ = rtc::make_ref_counted(); +AudioTrackSource::AudioTrackSource(AudioSourceOptions options) { + source_ = + rtc::make_ref_counted(to_native_audio_options(options)); +} + +AudioSourceOptions AudioTrackSource::audio_options() const { + return to_rust_audio_options(source_->options()); +} + +void AudioTrackSource::set_audio_options( + const AudioSourceOptions& options) const { + source_->set_options(to_native_audio_options(options)); } void AudioTrackSource::on_captured_frame(rust::Slice audio_data, @@ -134,8 +168,9 @@ rtc::scoped_refptr AudioTrackSource::get() return source_; } -std::shared_ptr new_audio_track_source() { - return std::make_shared(); +std::shared_ptr new_audio_track_source( + AudioSourceOptions options) { + return std::make_shared(options); } } // namespace livekit diff --git a/webrtc-sys/src/audio_track.rs b/webrtc-sys/src/audio_track.rs index 715579c..9edd7bd 100644 --- a/webrtc-sys/src/audio_track.rs +++ b/webrtc-sys/src/audio_track.rs @@ -3,6 +3,13 @@ use std::sync::Arc; #[cxx::bridge(namespace = "livekit")] pub mod ffi { + + pub struct AudioSourceOptions { + pub echo_cancellation: bool, + pub noise_suppression: bool, + pub auto_gain_control: bool, + } + extern "C++" { include!("livekit/media_stream_track.h"); @@ -27,7 +34,9 @@ pub mod ffi { nb_channels: usize, nb_frames: usize, ); - fn new_audio_track_source() -> SharedPtr; + fn audio_options(self: &AudioTrackSource) -> AudioSourceOptions; + fn set_audio_options(self: &AudioTrackSource, options: &AudioSourceOptions); + fn new_audio_track_source(options: AudioSourceOptions) -> SharedPtr; fn audio_to_media(track: SharedPtr) -> SharedPtr; unsafe fn media_to_audio(track: SharedPtr) -> SharedPtr; diff --git a/webrtc-sys/src/video_track.cpp b/webrtc-sys/src/video_track.cpp index ced13d5..45e07dc 100644 --- a/webrtc-sys/src/video_track.cpp +++ b/webrtc-sys/src/video_track.cpp @@ -31,6 +31,7 @@ #include "rtc_base/ref_counted_object.h" #include "rtc_base/synchronization/mutex.h" #include "rtc_base/time_utils.h" +#include "webrtc-sys/src/video_track.rs.h" namespace livekit { @@ -103,8 +104,9 @@ std::shared_ptr new_native_video_sink( return std::make_shared(std::move(observer)); } -VideoTrackSource::InternalSource::InternalSource() - : rtc::AdaptedVideoTrackSource(4) {} +VideoTrackSource::InternalSource::InternalSource( + const VideoResolution& resolution) + : rtc::AdaptedVideoTrackSource(4), resolution_(resolution) {} VideoTrackSource::InternalSource::~InternalSource() {} @@ -125,6 +127,11 @@ bool VideoTrackSource::InternalSource::remote() const { return false; } +VideoResolution VideoTrackSource::InternalSource::video_resolution() const { + webrtc::MutexLock lock(&mutex_); + return resolution_; +} + bool VideoTrackSource::InternalSource::on_captured_frame( const webrtc::VideoFrame& frame) { webrtc::MutexLock lock(&mutex_); @@ -135,6 +142,11 @@ bool VideoTrackSource::InternalSource::on_captured_frame( rtc::scoped_refptr buffer = frame.video_frame_buffer(); + if (resolution_.height == 0 || resolution_.width == 0) { + resolution_ = VideoResolution{static_cast(buffer->width()), + static_cast(buffer->height())}; + } + int adapted_width, adapted_height, crop_width, crop_height, crop_x, crop_y; if (!AdaptFrame(buffer->width(), buffer->height(), aligned_timestamp_us, &adapted_width, &adapted_height, &crop_width, &crop_height, @@ -163,16 +175,17 @@ bool VideoTrackSource::InternalSource::on_captured_frame( return true; } -VideoTrackSource::VideoTrackSource() { - source_ = rtc::make_ref_counted(); +VideoTrackSource::VideoTrackSource(const VideoResolution& resolution) { + source_ = rtc::make_ref_counted(resolution); +} + +VideoResolution VideoTrackSource::video_resolution() const { + return source_->video_resolution(); } bool VideoTrackSource::on_captured_frame( const std::unique_ptr& frame) const { auto rtc_frame = frame->get(); - rtc_frame.set_timestamp_us( - rtc::TimeMicros()); // TODO(theomonnom): Expore capture ts to Rust - return source_->on_captured_frame(rtc_frame); } @@ -181,8 +194,9 @@ rtc::scoped_refptr VideoTrackSource::get() return source_; } -std::shared_ptr new_video_track_source() { - return std::make_shared(); +std::shared_ptr new_video_track_source( + const VideoResolution& resolution) { + return std::make_shared(resolution); } } // namespace livekit diff --git a/webrtc-sys/src/video_track.rs b/webrtc-sys/src/video_track.rs index bcb840c..e82ba00 100644 --- a/webrtc-sys/src/video_track.rs +++ b/webrtc-sys/src/video_track.rs @@ -21,6 +21,12 @@ pub mod ffi { pub max_fps: f64, } + #[derive(Debug)] + pub struct VideoResolution { + pub width: u32, + pub height: u32, + } + extern "C++" { include!("livekit/video_frame.h"); include!("livekit/media_stream_track.h"); @@ -44,9 +50,9 @@ pub mod ffi { fn set_content_hint(self: &VideoTrack, hint: ContentHint); fn new_native_video_sink(observer: Box) -> SharedPtr; + fn video_resolution(self: &VideoTrackSource) -> VideoResolution; fn on_captured_frame(self: &VideoTrackSource, frame: &UniquePtr) -> bool; - fn new_video_track_source() -> SharedPtr; - + fn new_video_track_source(resolution: &VideoResolution) -> SharedPtr; fn video_to_media(track: SharedPtr) -> SharedPtr; unsafe fn media_to_video(track: SharedPtr) -> SharedPtr; fn _shared_video_track() -> SharedPtr;