From 3630e930547c43e9d1a5a73d8685b9dc17c22c39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sat, 29 Oct 2022 00:11:10 +0200 Subject: [PATCH 1/8] Initial Room Switching computer --- Cargo.lock | 1 + crates/livekit-core/Cargo.toml | 1 + crates/livekit-core/src/event.rs | 43 -- crates/livekit-core/src/lib.rs | 11 +- crates/livekit-core/src/local_participant.rs | 43 -- crates/livekit-core/src/room.rs | 78 --- crates/livekit-core/src/room/id.rs | 38 + .../src/room/local_participant.rs | 46 ++ crates/livekit-core/src/room/mod.rs | 316 +++++++++ crates/livekit-core/src/room/participant.rs | 109 +++ .../src/room/remote_participant.rs | 34 + crates/livekit-core/src/room/track.rs | 135 ++++ .../src/room/track_publication.rs | 28 + .../src/rtc_engine/engine_internal.rs | 576 ---------------- .../src/{ => rtc_engine}/lk_runtime.rs | 0 crates/livekit-core/src/rtc_engine/mod.rs | 651 +++++++++++++++--- .../src/{ => rtc_engine}/pc_transport.rs | 13 +- .../livekit-core/src/rtc_engine/rtc_events.rs | 103 +++ crates/livekit-core/src/signal_client/mod.rs | 98 ++- .../src/signal_client/signal_stream.rs | 23 +- crates/livekit-webrtc/libwebrtc-sys/build.rs | 8 +- .../include/livekit/media_stream.h | 52 ++ .../include/livekit/media_stream_interface.h | 28 - .../include/livekit/rtp_receiver.h | 3 + .../include/livekit/rust_types.h | 1 + .../livekit-webrtc/libwebrtc-sys/src/jsep.rs | 24 +- .../livekit-webrtc/libwebrtc-sys/src/lib.rs | 2 +- .../libwebrtc-sys/src/media_stream.cpp | 41 ++ .../libwebrtc-sys/src/media_stream.rs | 36 + .../src/media_stream_interface.cpp | 12 - .../src/media_stream_interface.rs | 10 - .../libwebrtc-sys/src/peer_connection.cpp | 7 +- .../libwebrtc-sys/src/peer_connection.rs | 32 +- .../src/peer_connection_factory.rs | 6 +- .../libwebrtc-sys/src/rtp_receiver.cpp | 5 + .../libwebrtc-sys/src/rtp_receiver.rs | 8 + crates/livekit-webrtc/src/data_channel.rs | 6 + crates/livekit-webrtc/src/jsep.rs | 12 +- crates/livekit-webrtc/src/media_stream.rs | 53 +- crates/livekit-webrtc/src/peer_connection.rs | 15 +- .../src/peer_connection_factory.rs | 9 +- crates/livekit-webrtc/src/rtc_error.rs | 1 - crates/livekit-webrtc/src/rtp_receiver.rs | 19 +- examples/Cargo.lock | 2 + examples/simple_room/Cargo.toml | 3 +- examples/simple_room/src/main.rs | 19 +- src/lib.rs | 2 +- 47 files changed, 1783 insertions(+), 980 deletions(-) delete mode 100644 crates/livekit-core/src/event.rs delete mode 100644 crates/livekit-core/src/local_participant.rs delete mode 100644 crates/livekit-core/src/room.rs create mode 100644 crates/livekit-core/src/room/id.rs create mode 100644 crates/livekit-core/src/room/local_participant.rs create mode 100644 crates/livekit-core/src/room/mod.rs create mode 100644 crates/livekit-core/src/room/participant.rs create mode 100644 crates/livekit-core/src/room/remote_participant.rs create mode 100644 crates/livekit-core/src/room/track.rs create mode 100644 crates/livekit-core/src/room/track_publication.rs delete mode 100644 crates/livekit-core/src/rtc_engine/engine_internal.rs rename crates/livekit-core/src/{ => rtc_engine}/lk_runtime.rs (100%) rename crates/livekit-core/src/{ => rtc_engine}/pc_transport.rs (90%) create mode 100644 crates/livekit-core/src/rtc_engine/rtc_events.rs create mode 100644 crates/livekit-webrtc/libwebrtc-sys/include/livekit/media_stream.h delete mode 100644 crates/livekit-webrtc/libwebrtc-sys/include/livekit/media_stream_interface.h create mode 100644 crates/livekit-webrtc/libwebrtc-sys/src/media_stream.cpp create mode 100644 crates/livekit-webrtc/libwebrtc-sys/src/media_stream.rs delete mode 100644 crates/livekit-webrtc/libwebrtc-sys/src/media_stream_interface.cpp delete mode 100644 crates/livekit-webrtc/libwebrtc-sys/src/media_stream_interface.rs diff --git a/Cargo.lock b/Cargo.lock index b50b8ab..fe5be1c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -494,6 +494,7 @@ dependencies = [ "futures-util", "lazy_static", "livekit-webrtc", + "parking_lot", "prost", "prost-build", "prost-types", diff --git a/crates/livekit-core/Cargo.toml b/crates/livekit-core/Cargo.toml index ab79746..ed20382 100644 --- a/crates/livekit-core/Cargo.toml +++ b/crates/livekit-core/Cargo.toml @@ -10,6 +10,7 @@ serde_json = "1.0" tokio-tungstenite = { version = "0.17.2", features = ["native-tls"] } tokio = { version = "1", features = ["full"] } futures = "0.3" +parking_lot = "0.12.1" url = "2.2.2" futures-util = "0.3.23" thiserror = "1.0" diff --git a/crates/livekit-core/src/event.rs b/crates/livekit-core/src/event.rs deleted file mode 100644 index 680a014..0000000 --- a/crates/livekit-core/src/event.rs +++ /dev/null @@ -1,43 +0,0 @@ -use std::pin::Pin; -use std::task::{Context, Poll}; -use futures::Stream; -use tokio::sync::mpsc; - -/// Using unbounded channels to prevent users from blocking internal logic ( e.g: ws heartbeat ) -/// Users must listen to all events to avoid the process from running out of memory - -#[derive(Clone, Debug)] -pub struct Emitter { - tx: mpsc::UnboundedSender, -} - -impl Emitter { - pub fn new() -> (Self, mpsc::UnboundedReceiver) { - let (tx, rx) = mpsc::unbounded_channel(); - - (Self { tx }, rx) - } - - pub fn event(&self, event: T) { - let _ = self.tx.send(event); - } -} - -#[derive(Debug)] -pub struct Events { - rx: mpsc::UnboundedReceiver, -} - -impl Events { - pub fn new(rx: mpsc::UnboundedReceiver) -> Self { - Self { rx } - } -} - -impl Stream for Events { - type Item = T; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { - self.rx.poll_recv(cx) - } -} diff --git a/crates/livekit-core/src/lib.rs b/crates/livekit-core/src/lib.rs index 8b972d1..bfc8c9a 100644 --- a/crates/livekit-core/src/lib.rs +++ b/crates/livekit-core/src/lib.rs @@ -1,12 +1,11 @@ +extern crate core; +extern crate core; + pub mod proto { include!(concat!(env!("OUT_DIR"), "/livekit.rs")); } -mod lk_runtime; -mod signal_client; -mod pc_transport; mod rtc_engine; -mod local_participant; -mod event; +mod signal_client; -pub mod room; \ No newline at end of file +pub mod room; diff --git a/crates/livekit-core/src/local_participant.rs b/crates/livekit-core/src/local_participant.rs deleted file mode 100644 index 4ffc925..0000000 --- a/crates/livekit-core/src/local_participant.rs +++ /dev/null @@ -1,43 +0,0 @@ -use std::sync::Arc; -use tokio::sync::Mutex; - -use crate::proto::{data_packet, DataPacket, ParticipantInfo, UserPacket}; -use crate::room::RoomError; -use crate::rtc_engine::RTCEngine; - -pub struct LocalParticipant { - sid: String, - identity: String, - name: String, - - engine: Arc>, -} - -impl LocalParticipant { - pub(crate) fn from(info: ParticipantInfo, engine: Arc>) -> Self { - Self { - sid: info.sid, - identity: info.identity, - name: info.name, - engine, - } - } - - pub(crate) fn update(info: ParticipantInfo) { - // TODO(theomonnom) - } - - // TODO(theomonnom) Add the destinations parameter - pub async fn publish_data(&mut self, data: &[u8], kind: data_packet::Kind) -> Result<(), RoomError> { - let data = DataPacket { - kind: kind as i32, - value: Some(data_packet::Value::User(UserPacket { - participant_sid: self.sid.clone(), - payload: data.to_vec(), - destination_sids: vec![], // TODO(theomonnom) - })), - }; - - self.engine.lock().await.publish_data(&data, kind).await.map_err(Into::into) - } -} diff --git a/crates/livekit-core/src/room.rs b/crates/livekit-core/src/room.rs deleted file mode 100644 index d2070a2..0000000 --- a/crates/livekit-core/src/room.rs +++ /dev/null @@ -1,78 +0,0 @@ -use std::sync::Arc; - -use thiserror::Error; -use tokio::sync::Mutex; - -use crate::local_participant::LocalParticipant; -use crate::rtc_engine; -use crate::rtc_engine::{EngineError, RTCEngine}; - -#[derive(Error, Debug)] -pub enum RoomError { - #[error("internal RTCEngine failure")] - Engine(#[from] EngineError), -} - -#[derive(Debug)] -pub enum RoomEvent { - -} - -pub struct Room { - sid: String, - name: String, - local_participant: LocalParticipant, - internal: Arc -} - -#[tracing::instrument(skip(url, token))] -pub async fn connect(url: &str, token: &str) -> Result { - let engine = rtc_engine::connect(url, token).await?; - let join = engine.join_response().await; - let engine = Arc::new(Mutex::new(engine)); - let local_participant = LocalParticipant::from(join.participant.unwrap(), engine.clone()); - let internal = Arc::new(RoomInternal::new(engine)); - let room_info = join.room.unwrap(); - - tokio::spawn(async move { - - }); - - Ok(Room { - sid: room_info.sid, - name: room_info.name, - local_participant, - internal, - }) -} - -impl Room { - pub fn local_participant(&self) -> &LocalParticipant { - &self.local_participant - } - - pub fn local_participant_mut(&mut self) -> &mut LocalParticipant { - &mut self.local_participant - } - - pub fn sid(&self) -> &str { - &self.sid - } - - pub fn name(&self) -> &str { - &self.name - } -} - - -struct RoomInternal { - engine: Arc>, -} - -impl RoomInternal { - pub fn new(engine: Arc>) -> Self { - Self { - engine - } - } -} \ No newline at end of file diff --git a/crates/livekit-core/src/room/id.rs b/crates/livekit-core/src/room/id.rs new file mode 100644 index 0000000..f46198a --- /dev/null +++ b/crates/livekit-core/src/room/id.rs @@ -0,0 +1,38 @@ +macro_rules! id_str { + ($($name:ident;)*) => { + $( + impl From for $name { + fn from(str: String) -> $name { + $name(str) + } + } + + impl PartialEq<$name> for String { + fn eq(&self, u: &$name) -> bool { + *self == *u.0 + } + } + + impl From<$name> for String { + fn from(id: $name) -> String { + id.0 + } + } + )* + } +} + +#[derive(Clone, Default, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] +pub struct ParticipantSid(pub String); + +#[derive(Clone, Default, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] +pub struct ParticipantIdentity(pub String); + +#[derive(Clone, Default, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] +pub struct TrackSid(pub String); + +id_str! { + ParticipantSid; + ParticipantIdentity; + TrackSid; +} diff --git a/crates/livekit-core/src/room/local_participant.rs b/crates/livekit-core/src/room/local_participant.rs new file mode 100644 index 0000000..5f7a96e --- /dev/null +++ b/crates/livekit-core/src/room/local_participant.rs @@ -0,0 +1,46 @@ +use crate::proto::{data_packet, DataPacket, UserPacket}; +use crate::room::participant::{impl_participant_trait, ParticipantShared}; +use crate::room::RoomError; +use crate::rtc_engine::RTCEngine; +use std::sync::Arc; + +pub struct LocalParticipant { + shared: ParticipantShared, + rtc_engine: Arc, +} + +impl LocalParticipant { + pub(super) fn new(rtc_engine: Arc, info: ParticipantInfo) -> Self { + Self { + shared: ParticipantShared::new( + info.sid.into(), + info.identity.into(), + info.name, + info.metadata, + ), + rtc_engine, + } + } + + pub async fn publish_data( + &self, + data: &[u8], + kind: data_packet::Kind, + ) -> Result<(), RoomError> { + let data = DataPacket { + kind: kind as i32, + value: Some(data_packet::Value::User(UserPacket { + participant_sid: "".to_string(), /*self.sid().to_owned()*/ + payload: data.to_vec(), + destination_sids: vec![], + })), + }; + + self.rtc_engine + .publish_data(&data, kind) + .await + .map_err(Into::into) + } +} + +impl_participant_trait!(LocalParticipant); diff --git a/crates/livekit-core/src/room/mod.rs b/crates/livekit-core/src/room/mod.rs new file mode 100644 index 0000000..853d992 --- /dev/null +++ b/crates/livekit-core/src/room/mod.rs @@ -0,0 +1,316 @@ +use futures_util::future::BoxFuture; +use parking_lot::lock_api::RwLockUpgradableReadGuard; +use parking_lot::{Mutex, RwLock}; +use std::borrow::Cow; +use std::collections::HashMap; +use std::future::Future; +use std::sync::atomic::AtomicU8; +use std::sync::Arc; + +use crate::proto; +use crate::proto::{participant_info, ParticipantInfo}; +use crate::room::id::{ParticipantIdentity, ParticipantSid}; +use crate::room::local_participant::LocalParticipant; +use crate::room::participant::ParticipantTrait; +use crate::room::remote_participant::RemoteParticipant; +use thiserror::Error; +use tracing::error; + +use crate::rtc_engine::{EngineError, EngineEvent, EngineEvents, RTCEngine}; +use crate::signal_client::SignalOptions; + +mod id; +mod local_participant; +mod participant; +mod remote_participant; +mod track; +mod track_publication; + +#[derive(Error, Debug)] +pub enum RoomError { + #[error("internal RTCEngine failure")] + Engine(#[from] EngineError), + #[error("internal Room failure")] + Internal(String), +} + +type RoomResult = Result; + +pub enum ConnectionState { + Disconnected, + Connecting, + Connected, + Reconnecting, +} + +struct RoomInner { + state: AtomicU8, // ConnectionState + sid: Mutex, + name: Mutex, + participants: RwLock>>, + rtc_engine: Arc, + local_participant: Arc, +} + +type OnParticipantConnectedHandler = + Box) -> BoxFuture<'static, ()> + Send + Sync>; +type OnParticipantDisconnectedHandler = OnParticipantConnectedHandler; + +struct RoomEvents { + on_participant_connected_handler: Mutex>, + on_participant_disconnected_handler: Mutex>, +} + +pub struct Room { + inner: Option>, + events: Arc, +} + +impl Room { + pub fn new() -> Room { + Self { + inner: None, + events: Arc::new(RoomEvents { + on_participant_connected_handler: Default::default(), + on_participant_disconnected_handler: Default::default(), + }), + } + } + + pub async fn connect(&mut self, url: &str, token: &str) -> RoomResult<()> { + let (rtc_engine, engine_events) = + RTCEngine::connect(url, token, SignalOptions::default()).await?; + let rtc_engine = Arc::new(rtc_engine); + let join_response = rtc_engine.join_response(); + let local_participant = Arc::new(LocalParticipant::new( + rtc_engine.clone(), + join_response.participant.unwrap().clone(), + )); + let room_info = join_response.room.unwrap(); + let inner = Arc::new(RoomInner { + state: AtomicU8::new(ConnectionState::Connecting as u8), + sid: Mutex::new(room_info.sid), + name: Mutex::new(room_info.name), + participants: Default::default(), + rtc_engine, + local_participant, + }); + + self.inner = Some(inner.clone()); + + tokio::spawn(Self::room_task(inner, self.events.clone(), engine_events)); + + Ok(()) + } + + pub fn get_handle(&self) -> Option { + self.inner.as_ref().map(|inner| RoomHandle { + inner: inner.clone(), + }) + } + + pub fn on_participant_connected(&self, mut callback: F) + where + F: FnMut(RoomHandle, Arc) -> Fut + Send + Sync + 'static, + Fut: Future + Send + Sync + 'static, + { + *self.events.on_participant_connected_handler.lock() = + Some(Box::new(move |handle, participant| { + Box::pin(callback(handle, participant)) + })); + } + + pub fn on_participant_disconnected(&self, mut callback: F) + where + F: FnMut(RoomHandle, Arc) -> Fut + Send + Sync + 'static, + Fut: Future + Send + Sync + 'static, + { + *self.events.on_participant_disconnected_handler.lock() = + Some(Box::new(move |handle, participant| { + Box::pin(callback(handle, participant)) + })); + } + + async fn room_task( + room_inner: Arc, + room_events: Arc, + mut engine_events: EngineEvents, + ) { + while let Some(event) = engine_events.recv().await { + if let Err(err) = + Self::handle_event(room_inner.clone(), room_events.clone(), event).await + { + error!("failed to handle engine event: {:?}", err); + } + } + } + + async fn handle_event( + room_inner: Arc, + room_events: Arc, + event: EngineEvent, + ) -> RoomResult<()> { + match event { + EngineEvent::ParticipantUpdate(update) => { + Self::handle_participant_update(room_inner.clone(), room_events.clone(), update) + } + EngineEvent::AddTrack { + rtp_receiver, + streams, + } => { + if streams.is_empty() { + Err(RoomError::Internal( + "AddTrack event with empty streams".to_string(), + ))?; + } + + let first_stream_id = streams.first().unwrap().id(); + let stream_id = unpack_stream_id(&first_stream_id); + if stream_id.is_none() { + Err(RoomError::Internal(format!( + "AddTrack event with invalid track_id: {:?}", + first_stream_id + )))?; + } + + let (participant_sid, track_sid) = stream_id.unwrap(); + let remote_participant = + Self::get_participant(room_inner.clone(), &participant_sid.to_string().into()); + + if let Some(remote_participant) = remote_participant { + + } else { + // The server should send participant updates before sending a new offer + // So this should not happen. + Err(RoomError::Internal(format!( + "AddTrack event with invalid participant_sid: {:?}", + participant_sid + )))?; + } + } + } + + Ok(()) + } + + fn handle_participant_update( + room_inner: Arc, + room_events: Arc, + update: proto::ParticipantUpdate, + ) { + for pi in update.participants { + if pi.sid == room_inner.local_participant.sid() + || pi.identity == room_inner.local_participant.identity() + { + room_inner.local_participant.update_info(pi); + continue; + } + + let remote_participant = + Self::get_participant(room_inner.clone(), &pi.sid.clone().into()); + + if let Some(remote_participant) = remote_participant { + if pi.state == participant_info::State::Disconnected as i32 { + // Participant disconencted + Self::handle_participant_disconnect( + room_inner.clone(), + room_events.clone(), + remote_participant, + ) + } else { + // Participant is already connected, update the informations + remote_participant.update_info(pi); + } + } else { + // Create a new participant and call OnConnect event + let remote_participant = Self::get_or_create_participant(room_inner.clone(), pi); + let mut handler = room_events.on_participant_connected_handler.lock(); + if let Some(callback) = handler.as_mut() { + callback( + RoomHandle::from(room_inner.clone()), + remote_participant.clone(), + ); + } + } + } + } + + fn handle_participant_disconnect( + room_inner: Arc, + room_events: Arc, + remote_participant: Arc, + ) { + room_inner + .participants + .write() + .remove(&remote_participant.sid()); + + // TODO(theomonnom): Unpublish all tracks + + let mut handler = room_events.on_participant_disconnected_handler.lock(); + if let Some(callback) = handler.as_mut() { + callback( + RoomHandle::from(room_inner.clone()), + remote_participant.clone(), + ); + } + } + + fn get_participant( + room_inner: Arc, + sid: &ParticipantSid, + ) -> Option> { + room_inner.participants.read().get(sid).cloned() + } + + fn get_or_create_participant( + room_inner: Arc, + pi: proto::ParticipantInfo, + ) -> Arc { + let participants = room_inner.participants.upgradable_read(); + let sid = pi.sid.clone().into(); + if let Some(p) = participants.get(&sid) { + p.update_info(pi); + p.clone() + } else { + let mut participants = RwLockUpgradableReadGuard::upgrade(participants); + let p = Arc::new(RemoteParticipant::new(pi)); + participants.insert(sid, p.clone()); + p + } + } +} + +#[derive(Clone)] +pub struct RoomHandle { + inner: Arc, +} + +impl RoomHandle { + fn from(room_inner: Arc) -> Self { + Self { inner: room_inner } + } + + pub fn sid(&self) -> String { + self.inner.sid.lock().clone() + } + + pub fn name(&self) -> String { + self.inner.name.lock().clone() + } + + pub fn local_participant(&self) -> Arc { + self.inner.local_participant.clone() + } +} + +fn unpack_stream_id(stream_id: &str) -> Option<(&str, &str)> { + let split: Vec<&str> = stream_id.split('|').collect(); + if split.len() == 2 { + let participant_sid = split.get(0).unwrap(); + let track_sid = split.get(1).unwrap(); + Some((participant_sid, track_sid)) + } else { + None + } +} diff --git a/crates/livekit-core/src/room/participant.rs b/crates/livekit-core/src/room/participant.rs new file mode 100644 index 0000000..b36c60c --- /dev/null +++ b/crates/livekit-core/src/room/participant.rs @@ -0,0 +1,109 @@ +use crate::proto::ParticipantInfo; +use crate::room::local_participant::LocalParticipant; +use crate::room::remote_participant::RemoteParticipant; +use parking_lot::{Mutex, RwLock}; +use std::collections::HashMap; +use std::sync::Arc; + +pub(super) struct ParticipantShared { + pub(super) sid: Mutex, + pub(super) identity: Mutex, + pub(super) name: Mutex, + pub(super) metadata: Mutex, + pub(super) tracks: RwLock>>, +} + +impl ParticipantShared { + pub(super) fn new( + sid: ParticipantSid, + identity: ParticipantIdentity, + name: String, + metadata: String, + ) -> Self { + Self { + sid: Mutex::new(sid), + identity: Mutex::new(identity), + name: Mutex::new(name), + metadata: Mutex::new(metadata), + tracks: Default::default(), + } + } + + pub(crate) fn update_info(&self, info: ParticipantInfo) { + *self.sid.lock() = info.sid.into(); + *self.identity.lock() = info.identity.into(); + *self.name.lock() = info.name; + *self.metadata.lock() = info.metadata; // TODO(theomonnom): callback + } +} + +pub trait ParticipantTrait { + fn sid(&self) -> ParticipantSid; + fn identity(&self) -> ParticipantIdentity; + fn name(&self) -> String; + fn metadata(&self) -> String; + fn update_info(&self, info: ParticipantInfo); +} + +pub enum Participant { + Local(LocalParticipant), + Remote(RemoteParticipant), +} + +macro_rules! shared_method { + ($x:ident, $ret:ident) => { + fn $x(&self) -> $ret { + match self { + Participant::Local(p) => p.$x(), + Participant::Remote(p) => p.$x(), + } + } + }; +} + +impl ParticipantTrait for Participant { + shared_method!(sid, ParticipantSid); + shared_method!(identity, ParticipantIdentity); + shared_method!(name, String); + shared_method!(metadata, String); + + fn update_info(&self, info: ParticipantInfo) { + match self { + Participant::Local(p) => p.update_info(info), + Participant::Remote(p) => p.update_info(info), + } + } +} + +macro_rules! impl_participant_trait { + ($x:ident) => { + use crate::proto::ParticipantInfo; + use crate::room::id::{ParticipantIdentity, ParticipantSid}; + + impl crate::room::participant::ParticipantTrait for $x { + fn sid(&self) -> ParticipantSid { + self.shared.sid.lock().clone() + } + + fn identity(&self) -> ParticipantIdentity { + self.shared.identity.lock().clone() + } + + fn name(&self) -> String { + self.shared.name.lock().clone() + } + + fn metadata(&self) -> String { + self.shared.metadata.lock().clone() + } + + fn update_info(&self, info: ParticipantInfo) { + self.shared.update_info(info); + } + } + }; +} + +use crate::room::id::{ParticipantIdentity, ParticipantSid, TrackSid}; +use crate::room::track_publication::TrackPublication; +pub(super) use impl_participant_trait; diff --git a/crates/livekit-core/src/room/remote_participant.rs b/crates/livekit-core/src/room/remote_participant.rs new file mode 100644 index 0000000..5751cdd --- /dev/null +++ b/crates/livekit-core/src/room/remote_participant.rs @@ -0,0 +1,34 @@ +use std::sync::Arc; +use crate::room::participant::{impl_participant_trait, ParticipantShared}; +use crate::room::track_publication::RemoteTrackPublication; + +pub struct RemoteParticipant { + shared: ParticipantShared, +} + +impl RemoteParticipant { + pub(super) fn new(info: ParticipantInfo) -> Self { + Self { + shared: ParticipantShared::new( + info.sid.into(), + info.identity.into(), + info.name, + info.metadata, + ), + } + } + + pub(super) async fn add_subscribed_media_track() { + + + } + + fn get_track_publication(&self, sid: &str) -> Option { + let track = self.shared.tracks.read().get(&sid.to_string().into()).unwrap().clone(); + + + None + } +} + +impl_participant_trait!(RemoteParticipant); diff --git a/crates/livekit-core/src/room/track.rs b/crates/livekit-core/src/room/track.rs new file mode 100644 index 0000000..8b88b74 --- /dev/null +++ b/crates/livekit-core/src/room/track.rs @@ -0,0 +1,135 @@ +pub enum TrackKind { + Audio, + Video +} + +pub enum StreamState { + Active, + Paused, + Unknown +} + +pub enum TrackSource { + Camera, + Microphone, + Screenshare, + ScreenshareAudio, + Unknown, +} + +pub struct LocalVideoTrack {} +pub struct RemoteVideoTrack {} +pub struct LocalAudioTrack {} +pub struct RemoteAudioTrack {} + + +pub enum RemoteTrack { + Audio(RemoteAudioTrack), + Video(RemoteVideoTrack), +} + +pub enum LocalTrack { + Audio(LocalAudioTrack), + Video(LocalVideoTrack), +} + +pub enum VideoTrack { + Local(LocalVideoTrack), + Remote(RemoteVideoTrack), +} + +pub enum AudioTrack { + Local(LocalAudioTrack), + Remote(RemoteAudioTrack), +} + +pub enum Track { + LocalVideo(LocalVideoTrack), + LocalAudio(LocalAudioTrack), + RemoteVideo(RemoteVideoTrack), + RemoteAudio(RemoteAudioTrack), +} + +impl From for Track { + fn from(video_track: VideoTrack) -> Self { + match video_track { + VideoTrack::Local(local_video) => Self::LocalVideo(local_video), + VideoTrack::Remote(remote_video) => Self::RemoteVideo(remote_video), + } + } +} + +impl From for Track { + fn from(audio_track: AudioTrack) -> Self { + match audio_track { + AudioTrack::Local(local_audio) => Self::LocalAudio(local_audio), + AudioTrack::Remote(remote_audio) => Self::RemoteAudio(remote_audio), + } + } +} + +impl From for Track { + fn from(local_track: LocalTrack) -> Self { + match local_track { + LocalTrack::Audio(local_audio) => Self::LocalAudio(local_audio), + LocalTrack::Video(local_video) => Self::LocalVideo(local_video), + } + } +} + +impl From for Track { + fn from(remote_track: RemoteTrack) -> Self { + match remote_track { + RemoteTrack::Audio(remote_audio) => Self::RemoteAudio(remote_audio), + RemoteTrack::Video(remote_video) => Self::RemoteVideo(remote_video), + } + } +} + +impl TryFrom for VideoTrack { + type Error = &'static str; + + fn try_from(track: Track) -> Result { + match track { + Track::LocalVideo(local_video) => Ok(Self::Local(local_video)), + Track::RemoteVideo(remote_video) => Ok(Self::Remote(remote_video)), + _ => Err("not a video track"), + } + } +} + +impl TryFrom for AudioTrack { + type Error = &'static str; + + fn try_from(track: Track) -> Result { + match track { + Track::LocalAudio(local_audio) => Ok(Self::Local(local_audio)), + Track::RemoteAudio(remote_audio) => Ok(Self::Remote(remote_audio)), + _ => Err("not a audio track"), + } + } +} + +impl TryFrom for LocalTrack { + type Error = &'static str; + + fn try_from(track: Track) -> Result { + match track { + Track::LocalAudio(local_audio) => Ok(Self::Audio(local_audio)), + Track::LocalVideo(local_video) => Ok(Self::Video(local_video)), + _ => Err("not a local track"), + } + } +} + +impl TryFrom for RemoteTrack { + type Error = &'static str; + + fn try_from(track: Track) -> Result { + match track { + Track::RemoteAudio(remote_audio) => Ok(Self::Audio(remote_audio)), + Track::RemoteVideo(remote_video) => Ok(Self::Video(remote_video)), + _ => Err("not a remote track"), + } + } +} diff --git a/crates/livekit-core/src/room/track_publication.rs b/crates/livekit-core/src/room/track_publication.rs new file mode 100644 index 0000000..e2c6361 --- /dev/null +++ b/crates/livekit-core/src/room/track_publication.rs @@ -0,0 +1,28 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU8}; +use parking_lot::Mutex; +use crate::room::id::TrackSid; + +pub(super) struct TrackPublicationShared { + pub(super) name: Mutex, + pub(super) sid: Mutex, + pub(super) kind: AtomicU8, // Casted to TrackKind + pub(super) source: AtomicU8, // Casted to TrackSource + pub(super) simulcasted: AtomicBool +} + +#[derive(Clone)] +pub struct LocalTrackPublication { + shared: Arc +} + +#[derive(Clone)] +pub struct RemoteTrackPublication { + shared: Arc +} + +#[derive(Clone)] +pub enum TrackPublication { + Local(LocalTrackPublication), + Remote(RemoteTrackPublication) +} \ No newline at end of file diff --git a/crates/livekit-core/src/rtc_engine/engine_internal.rs b/crates/livekit-core/src/rtc_engine/engine_internal.rs deleted file mode 100644 index 7c98870..0000000 --- a/crates/livekit-core/src/rtc_engine/engine_internal.rs +++ /dev/null @@ -1,576 +0,0 @@ -use std::fmt::{Debug, Formatter}; -use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; -use std::sync::Arc; -use std::time::Duration; - -use prost::Message; -use serde::{Deserialize, Serialize}; -use tokio::sync::{mpsc, Mutex}; -use tokio::time; -use tracing::{event, Level}; - -use livekit_webrtc::data_channel::{DataChannel, DataChannelInit, DataState}; -use livekit_webrtc::jsep::{IceCandidate, SessionDescription}; -use livekit_webrtc::peer_connection::{ - IceConnectionState, PeerConnectionState, RTCOfferAnswerOptions, -}; -use livekit_webrtc::peer_connection_factory::{ - ContinualGatheringPolicy, ICEServer, IceTransportsType, RTCConfiguration, -}; - -use crate::lk_runtime::LKRuntime; -use crate::pc_transport::PCTransport; -use crate::proto; -use crate::proto::data_packet::Value; -use crate::proto::{ - data_packet, signal_request, signal_response, DataPacket, JoinResponse, SignalTarget, - TrickleRequest, -}; -use crate::rtc_engine::{EngineError, MAX_ICE_CONNECT_TIMEOUT}; -use crate::signal_client::SignalClient; - -const LOSSY_DC_LABEL: &str = "_lossy"; -const RELIABLE_DC_LABEL: &str = "_reliable"; - -// Used to communicate IceCandidate with the server -#[derive(Serialize, Deserialize)] -#[allow(non_snake_case)] -struct IceCandidateJSON { - sdpMid: String, - sdpMLineIndex: i32, - candidate: String, -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub(crate) enum PCState { - New, - Connected, - Disconnected, - Reconnecting, - Closed, -} - -#[derive(Debug)] -pub(crate) enum InternalMessage { - IceCandidate { - ice_candidate: IceCandidate, - publisher: bool, - }, - ConnectionChange { - state: PeerConnectionState, - primary: bool, - }, - PrimaryDataChannel { - data_channel: DataChannel, - }, - PublisherOffer { - offer: SessionDescription, - }, - Data { - data: Vec, - binary: bool, - }, -} - -pub(crate) struct EngineInternal { - pub(super) publisher_pc: Arc>, - pub(super) subscriber_pc: Arc>, - pub(super) lossy_dc: Arc>, - pub(super) reliable_dc: Arc>, - pub(super) lossy_dc_sub: Arc>>, - pub(super) reliable_dc_sub: Arc>>, - - pub(super) msg_sender: mpsc::Sender, - pub(super) join_response: Mutex, - pub(super) pc_state: AtomicU8, // casted to PCState - pub(super) has_published: AtomicBool, -} - -impl Debug for EngineInternal { - fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - write!(f, "EngineInternal") - } -} - -impl EngineInternal { - /// Configure the PeerConnections - /// - /// This is called on connect & on full reconnect. - /// Create the PeerConnections & the DataChannels. - /// Register listeners and send the internal messages - /// to the event_loop. - #[tracing::instrument] - pub(super) fn configure( - lk_runtime: Arc, - sender: mpsc::Sender, - join: JoinResponse, - ) -> Result { - let rtc_config = RTCConfiguration { - ice_servers: { - let mut servers = vec![]; - for is in join.ice_servers.clone() { - servers.push(ICEServer { - urls: is.urls, - username: is.username, - password: is.credential, - }) - } - servers - }, - continual_gathering_policy: ContinualGatheringPolicy::GatherContinually, - ice_transport_type: IceTransportsType::All, - }; - - let mut publisher_pc = PCTransport::new( - lk_runtime - .pc_factory - .create_peer_connection(rtc_config.clone())?, - ); - let mut subscriber_pc = - PCTransport::new(lk_runtime.pc_factory.create_peer_connection(rtc_config)?); - - publisher_pc.peer_connection().on_ice_candidate(Box::new({ - let sender = sender.clone(); - move |ice_candidate| { - let _ = sender.blocking_send(InternalMessage::IceCandidate { - ice_candidate, - publisher: true, - }); - } - })); - - subscriber_pc.peer_connection().on_ice_candidate(Box::new({ - let sender = sender.clone(); - move |ice_candidate| { - let _ = sender.blocking_send(InternalMessage::IceCandidate { - ice_candidate, - publisher: false, - }); - } - })); - - publisher_pc.on_offer({ - let sender = sender.clone(); - Box::new(move |offer| { - let sender = sender.clone(); - - tokio::spawn(async move { - let _ = sender.send(InternalMessage::PublisherOffer { offer }).await; - }); - - Box::pin(async move {}) - }) - }); - - let mut primary_pc = &mut publisher_pc; - let mut secondary_pc = &mut subscriber_pc; - if join.subscriber_primary { - primary_pc = &mut subscriber_pc; - secondary_pc = &mut publisher_pc; - - primary_pc.peer_connection().on_data_channel(Box::new({ - let sender = sender.clone(); - move |data_channel| { - let _ = - sender.blocking_send(InternalMessage::PrimaryDataChannel { data_channel }); - } - })); - } - - primary_pc.peer_connection().on_connection_change(Box::new({ - let sender = sender.clone(); - move |state| { - let _ = sender.blocking_send(InternalMessage::ConnectionChange { - state, - primary: true, - }); - } - })); - - secondary_pc - .peer_connection() - .on_connection_change(Box::new({ - let sender = sender.clone(); - move |state| { - let _ = sender.blocking_send(InternalMessage::ConnectionChange { - state, - primary: false, - }); - } - })); - - let mut lossy_dc = publisher_pc.peer_connection().create_data_channel( - LOSSY_DC_LABEL, - DataChannelInit { - ordered: true, - max_retransmits: Some(0), - ..DataChannelInit::default() - }, - )?; - - let mut reliable_dc = publisher_pc.peer_connection().create_data_channel( - RELIABLE_DC_LABEL, - DataChannelInit { - ordered: true, - ..DataChannelInit::default() - }, - )?; - - Self::configure_dc(&mut lossy_dc, sender.clone()); - Self::configure_dc(&mut reliable_dc, sender.clone()); - - Ok(Self { - publisher_pc: Arc::new(Mutex::new(publisher_pc)), - subscriber_pc: Arc::new(Mutex::new(subscriber_pc)), - lossy_dc: Arc::new(Mutex::new(lossy_dc)), - reliable_dc: Arc::new(Mutex::new(reliable_dc)), - lossy_dc_sub: Default::default(), - reliable_dc_sub: Default::default(), - msg_sender: sender, - join_response: Mutex::new(join), - pc_state: AtomicU8::new(PCState::New as u8), - has_published: AtomicBool::new(false), - }) - } - - /// Send InternalMessage when a datachannel receives data - #[tracing::instrument] - fn configure_dc(data_channel: &mut DataChannel, sender: mpsc::Sender) { - data_channel.on_message(Box::new(move |data, binary| { - let _ = sender.blocking_send(InternalMessage::Data { - data: data.to_vec(), - binary, - }); - })); - } - - /// Ensure the publisher PeerConnection is connected - /// - /// When subscriber_primary is enabled, only the subscriber PeerConnection is negotiated. - /// This allows for faster connection when we don't need the publisher - #[tracing::instrument] - pub(super) async fn ensure_publisher_connected( - self: &Arc, - kind: data_packet::Kind, - ) -> Result<(), EngineError> { - if !self.join_response.lock().await.subscriber_primary { - return Ok(()); - } - - let publisher = &self.publisher_pc; - { - let mut publisher = publisher.lock().await; - if !publisher.is_connected() - && publisher.peer_connection().ice_connection_state() - != IceConnectionState::IceConnectionChecking - { - tokio::spawn({ - let internal = self.clone(); - async move { - let _ = internal.negotiate_publisher().await; - } - }); - } - } - - let dc = self.data_channel(kind); - if dc.lock().await.state() == DataState::Open { - return Ok(()); - } - - let res = time::timeout(MAX_ICE_CONNECT_TIMEOUT, async move { - let mut interval = time::interval(Duration::from_millis(50)); - - loop { - if publisher.lock().await.is_connected() - && dc.lock().await.state() == DataState::Open - { - break; - } - - interval.tick().await; - } - }) - .await; - - if res.is_err() { - let err = - EngineError::Connection("could not establish publisher connection".to_string()); - event!(Level::ERROR, error = ?err); - Err(err) - } else { - Ok(()) - } - } - - /// Run the event_loop of the RTCEngine - #[tracing::instrument] - pub(super) async fn run( - self: &Arc, - mut receiver: mpsc::Receiver, - signal_client: Arc, - ) { - loop { - tokio::select! { - signal = signal_client.recv() => { - match signal { - Some(signal) => { - if let Err(err) = self.handle_signal(signal, signal_client.clone()).await { - event!( - Level::ERROR, - "failed to handle signal: {:?}", - err, - ); - } - } - None => { - // TODO(theomonnom) Trigger reconnect - } - } - }, - Some(msg) = receiver.recv() => { - if let Err(err) = self.handle_message(msg, signal_client.clone()).await { - event!( - Level::ERROR, - "failed to handle engine message: {:?}", - err, - ); - } - }, - } - } - } - - /// Handle SignalResponse messages coming from the server - /// - /// Run the needed livekit-protocol - #[tracing::instrument] - async fn handle_signal( - self: &Arc, - signal: signal_response::Message, - signal_client: Arc, - ) -> Result<(), EngineError> { - match signal { - signal_response::Message::Answer(answer) => { - event!(Level::TRACE, "received answer for publisher: {:?}", answer); - let sdp = SessionDescription::from(answer.r#type.parse().unwrap(), &answer.sdp)?; - self.publisher_pc - .lock() - .await - .set_remote_description(sdp) - .await?; - } - signal_response::Message::Offer(offer) => { - // Handle the subscriber offer & send an answer to livekit-server - // We always get an offer from the server when connecting - event!(Level::TRACE, "received offer for subscriber: {:?}", offer); - let sdp = SessionDescription::from(offer.r#type.parse().unwrap(), &offer.sdp)?; - - self.subscriber_pc - .lock() - .await - .set_remote_description(sdp) - .await?; - let answer = self - .subscriber_pc - .lock() - .await - .peer_connection() - .create_answer(RTCOfferAnswerOptions::default()) - .await?; - self.subscriber_pc - .lock() - .await - .peer_connection() - .set_local_description(answer.clone()) - .await?; - - tokio::spawn(async move { - let _ = signal_client - .send(signal_request::Message::Answer(proto::SessionDescription { - r#type: "answer".to_string(), - sdp: answer.to_string(), - })) - .await; - }); - } - signal_response::Message::Trickle(trickle) => { - // Add the IceCandidate received from the livekit-server - let json: IceCandidateJSON = serde_json::from_str(&trickle.candidate_init)?; - let ice = IceCandidate::from(&json.sdpMid, json.sdpMLineIndex, &json.candidate)?; - - event!( - Level::TRACE, - "received ice_candidate ({:?}) - {:?}", - SignalTarget::from_i32(trickle.target).unwrap(), - ice - ); - - if trickle.target == SignalTarget::Publisher as i32 { - self.publisher_pc - .lock() - .await - .add_ice_candidate(ice) - .await?; - } else { - self.subscriber_pc - .lock() - .await - .add_ice_candidate(ice) - .await?; - } - } - _ => {} - } - - Ok(()) - } - - /// Handle libwebrtc messages - /// - /// Every message used inside this function comes from libwebrtc. - /// The messages are received in [EngineInternal](#run) - /// We're not handling the messages inside the signaling_thread, to return - /// as quickly as possible. - #[tracing::instrument] - async fn handle_message( - self: &Arc, - msg: InternalMessage, - signal_client: Arc, - ) -> Result<(), EngineError> { - match msg { - InternalMessage::IceCandidate { - ice_candidate, - publisher, - } => { - // Send the IceCandidate to livekit-server - // Note that ContinualGatheringPolicy is set to GatherContinually - let json = serde_json::to_string(&IceCandidateJSON { - sdpMid: ice_candidate.sdp_mid(), - sdpMLineIndex: ice_candidate.sdp_mline_index(), - candidate: ice_candidate.candidate(), - })?; - - let target = if publisher { - SignalTarget::Publisher - } else { - SignalTarget::Subscriber - }; - - event!( - Level::TRACE, - "sending ice_candidate ({:?}) - {:?}", - target, - ice_candidate - ); - - tokio::spawn(async move { - let _ = signal_client - .send(signal_request::Message::Trickle(TrickleRequest { - candidate_init: json, - target: target as i32, - })) - .await; - }); - } - InternalMessage::ConnectionChange { state, primary } => { - // PeerConnectionState changed - // Reconnect if we've been disconnected unexpectedly - // If connected for the first time, send OnConnect event - if primary && state == PeerConnectionState::Connected { - let old_state = self.pc_state.load(Ordering::SeqCst); - self.pc_state - .store(PCState::Connected as u8, Ordering::SeqCst); - - if old_state == PCState::New as u8 { - // TODO(theomonnom) OnConnected - } - } else if state == PeerConnectionState::Failed { - self.pc_state - .store(PCState::Disconnected as u8, Ordering::SeqCst); - - // TODO(theomonnom) handle Disconnect - } - } - InternalMessage::PrimaryDataChannel { mut data_channel } => { - // Received datachannel from the primary PeerConnection. - // If subscriber_primary is enabled, the datachannel is used for downstream data - let reliable = data_channel.label() == RELIABLE_DC_LABEL; - Self::configure_dc(&mut data_channel, self.msg_sender.clone()); - - event!( - Level::TRACE, - "received primary data_channel - {:?}", - data_channel - ); - - if reliable { - *self.reliable_dc_sub.lock().await = Some(data_channel); - } else { - *self.lossy_dc_sub.lock().await = Some(data_channel); - } - } - InternalMessage::PublisherOffer { offer } => { - // Send the publisher offer to livekit-server - event!(Level::TRACE, "sending publisher offer - {:?}", offer); - - tokio::spawn(async move { - let _ = signal_client - .send(signal_request::Message::Offer(proto::SessionDescription { - r#type: "offer".to_string(), - sdp: offer.to_string(), - })) - .await; - }); - } - InternalMessage::Data { data, binary } => { - // Received data from a datachannel - // If this is a Speaker DataPacket, update the active speakers - // Send SpeakersChanged/OnData event - if !binary { - return Err(EngineError::Internal( - "text messages aren't supported".to_string(), - )); - } - - let data = DataPacket::decode(&*data)?; - match data.value.unwrap() { - Value::User(user) => { - /*let mut handler = self.on_data_handler.lock().await; - if let Some(f) = &mut *handler { - f(Packet { - data: user, - kind: data_packet::Kind::from_i32(data.kind).unwrap(), - }) - .await; - }*/ - } - Value::Speaker(_) => { - // TODO(theomonnonm) - } - } - } - } - - Ok(()) - } - - #[tracing::instrument] - async fn negotiate_publisher(self: &Arc) -> Result<(), EngineError> { - self.has_published.store(true, Ordering::SeqCst); - if let Err(err) = self.publisher_pc.lock().await.negotiate().await { - event!(Level::ERROR, "failed to negotiate the publisher: {:?}", err,); - Err(EngineError::Rtc(err)) - } else { - Ok(()) - } - } - - pub(super) fn data_channel(&self, kind: data_packet::Kind) -> Arc> { - if kind == data_packet::Kind::Reliable { - self.reliable_dc.clone() - } else { - self.lossy_dc.clone() - } - } -} diff --git a/crates/livekit-core/src/lk_runtime.rs b/crates/livekit-core/src/rtc_engine/lk_runtime.rs similarity index 100% rename from crates/livekit-core/src/lk_runtime.rs rename to crates/livekit-core/src/rtc_engine/lk_runtime.rs diff --git a/crates/livekit-core/src/rtc_engine/mod.rs b/crates/livekit-core/src/rtc_engine/mod.rs index 180097f..86fd80f 100644 --- a/crates/livekit-core/src/rtc_engine/mod.rs +++ b/crates/livekit-core/src/rtc_engine/mod.rs @@ -1,29 +1,72 @@ -use crate::lk_runtime::LKRuntime; -use crate::proto::{data_packet, signal_response, DataPacket, JoinResponse, UserPacket}; -use crate::rtc_engine::engine_internal::EngineInternal; -use crate::signal_client::{SignalClient, SignalError, SignalEvent, SignalOptions}; -use futures_util::{FutureExt, StreamExt}; -use lazy_static::lazy_static; -use livekit_webrtc::data_channel::DataSendError; -use livekit_webrtc::jsep::SdpParseError; -use livekit_webrtc::rtc_error::RTCError; -use prost::Message; +use core::num::flt2dec::Sign; +use parking_lot::Mutex; +use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; use std::sync::{Arc, Weak}; use std::time::Duration; -use thiserror::Error; -use tokio::sync::{mpsc, Mutex}; -use tokio::time; -use tracing::{event, Level}; -mod engine_internal; +use tokio::sync::{mpsc, Mutex as AsyncMutex}; + +use lazy_static::lazy_static; +use prost::Message; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use tokio::time::sleep; +use tracing::{debug, error, trace}; + +use crate::{proto, signal_client}; +use livekit_webrtc::data_channel::{DataChannel, DataChannelInit, DataSendError, DataState}; +use livekit_webrtc::jsep::{IceCandidate, SdpParseError, SessionDescription}; +use livekit_webrtc::media_stream::MediaStream; +use livekit_webrtc::peer_connection::{ + IceConnectionState, PeerConnectionState, RTCOfferAnswerOptions, +}; +use livekit_webrtc::peer_connection_factory::RTCConfiguration; +use livekit_webrtc::rtc_error::RTCError; +use livekit_webrtc::rtp_receiver::RtpReceiver; + +use crate::proto::data_packet::Value; +use crate::proto::{ + data_packet, signal_request, signal_response, DataPacket, JoinResponse, ParticipantUpdate, + SignalTarget, TrickleRequest, +}; +use crate::rtc_engine::lk_runtime::LKRuntime; +use crate::rtc_engine::pc_transport::PCTransport; +use crate::rtc_engine::rtc_events::{RTCEmitter, RTCEvent, RTCEvents}; +use crate::signal_client::{SignalClient, SignalError, SignalEvent, SignalEvents, SignalOptions}; + +mod lk_runtime; +mod pc_transport; +mod rtc_events; lazy_static! { // Share one LKRuntime across all RTCEngine instances static ref LK_RUNTIME: Mutex> = Mutex::new(Weak::new()); } +pub(crate) type EngineEmitter = mpsc::Sender; +pub(crate) type EngineEvents = mpsc::Receiver; +pub(crate) type EngineResult = Result; + pub(crate) const MAX_ICE_CONNECT_TIMEOUT: Duration = Duration::from_secs(15); -pub(crate) const JOIN_RESPONSE_TIMEOUT: Duration = Duration::from_secs(5); +pub(crate) const LOSSY_DC_LABEL: &str = "_lossy"; +pub(crate) const RELIABLE_DC_LABEL: &str = "_reliable"; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum PCState { + New, + Connected, + Disconnected, + Reconnecting, + Closed, +} + +#[derive(Serialize, Deserialize)] +#[allow(non_snake_case)] +struct IceCandidateJSON { + sdpMid: String, + sdpMLineIndex: i32, + candidate: String, +} #[derive(Error, Debug)] pub enum EngineError { @@ -46,108 +89,542 @@ pub enum EngineError { } #[derive(Debug)] -pub struct Packet { - pub data: UserPacket, - pub kind: data_packet::Kind, +pub(crate) enum EngineEvent { + ParticipantUpdate(ParticipantUpdate), + AddTrack { + rtp_receiver: RtpReceiver, + streams: Vec, + }, } #[derive(Debug)] -pub enum EngineEvent { - DataReceived(Packet), +struct EngineInner { + has_published: AtomicBool, + join_response: Mutex, + pc_state: AtomicU8, // Casted to PCState enum + + publisher_pc: AsyncMutex, + subscriber_pc: AsyncMutex, + + // Publisher data channels + // Used to send data to other participants ( The SFU forward the messages ) + lossy_dc: Mutex, + reliable_dc: Mutex, + + // Subscriber data channels + // These fields are never used, we just keep a strong reference to them, + // so we can receive data from other participants + sub_reliable_dc: Mutex>, + sub_lossy_dc: Mutex>, } #[derive(Debug)] pub struct RTCEngine { signal_client: Arc, - internal: Arc, + engine_inner: Arc, #[allow(unused)] lk_runtime: Arc, // Keep a reference while we're using the RTCEngine } -#[tracing::instrument(skip(url, token))] -pub async fn connect( - url: &str, - token: &str, - options: SignalOptions, -) -> Result { - // Acquire an existing/a new LKRuntime - let mut lk_runtime_ref = LK_RUNTIME.lock().await; - let mut lk_runtime = lk_runtime_ref.upgrade(); +impl RTCEngine { + #[tracing::instrument(skip(url, token))] + pub(crate) async fn connect( + url: &str, + token: &str, + options: SignalOptions, + ) -> EngineResult<(RTCEngine, EngineEvents)> { + let mut lk_runtime = None; + { + let mut lk_runtime_ref = LK_RUNTIME.lock(); + lk_runtime = lk_runtime_ref.upgrade(); - if lk_runtime.is_none() { - let new_runtime = Arc::new(LKRuntime::new()); - *lk_runtime_ref = Arc::downgrade(&new_runtime); - lk_runtime = Some(new_runtime); - } - let lk_runtime = lk_runtime.unwrap(); - let (signal_client, mut signal_events) = SignalClient::connect(url, token, options).await?; - let signal_client = Arc::new(signal_client); - - let join_response = time::timeout(JOIN_RESPONSE_TIMEOUT, async move { - while let Some(event) = signal_events.next().await { - match event { - SignalEvent::Signal(signal_response::Message::Join(join)) => return join, - _ => { - // Should we try a reconnect on close here? - continue; - } + if lk_runtime.is_none() { + let new_runtime = Arc::new(LKRuntime::new()); + *lk_runtime_ref = Arc::downgrade(&new_runtime); + lk_runtime = Some(new_runtime); } } - unreachable!(); - }) - .await - .map_err(|_| EngineError::Internal("failed to receive JoinResponse".to_string()))?; + let lk_runtime = lk_runtime.unwrap(); + let (signal_client, mut signal_events) = SignalClient::connect(url, token, options).await?; - event!(Level::DEBUG, "received JoinResponse: {:?}", join_response); + let join_response = signal_client::utils::next_join_response(&mut signal_events).await?; + debug!("received JoinResponse: {:?}", join_response); - let (sender, receiver) = mpsc::channel(8); - let internal = Arc::new(EngineInternal::configure( - lk_runtime.clone(), - sender, - join_response.clone(), - )?); + let (engine_inner, rtc_events) = + Self::configure_engine(lk_runtime.clone(), join_response.clone())?; + let engine_inner = Arc::new(engine_inner); + let signal_client = Arc::new(signal_client); - if !join_response.subscriber_primary { - internal.publisher_pc.lock().await.negotiate().await?; + let (emitter, events) = mpsc::channel(8); + + tokio::spawn(Self::signal_task( + signal_client.clone(), + engine_inner.clone(), + signal_events, + emitter.clone(), + )); + + tokio::spawn(Self::engine_task( + signal_client.clone(), + engine_inner.clone(), + rtc_events, + emitter.clone(), + )); + + let rtc_engine = Self { + signal_client, + engine_inner, + lk_runtime, + }; + + if !join_response.subscriber_primary { + rtc_engine.negotiate_publisher().await?; + } + + Ok((rtc_engine, events)) } - tokio::spawn({ - let signal_client = signal_client.clone(); - let internal = internal.clone(); - - async move { - internal.run(receiver, signal_client).await; - } - }); - - Ok(RTCEngine { - lk_runtime, - signal_client, - internal, - }) -} - -impl RTCEngine { - /// Send data to other participants in the Room #[tracing::instrument] pub async fn publish_data( - &mut self, + &self, data: &DataPacket, kind: data_packet::Kind, ) -> Result<(), EngineError> { - self.internal.ensure_publisher_connected(kind).await?; - self.internal - .data_channel(kind) + self.ensure_publisher_connected(kind).await?; + self.data_channel(kind) .lock() - .await .send(&data.encode_to_vec(), true) .map_err(Into::into) } - /// Return the last received JoinResponse - pub async fn join_response(&self) -> JoinResponse { - self.internal.join_response.lock().await.clone() + pub fn join_response(&self) -> JoinResponse { + self.engine_inner.join_response.lock().clone() + } + + async fn engine_task( + signal_client: Arc, + engine_inner: Arc, + mut rtc_events: RTCEvents, + emitter: EngineEmitter, + ) { + while let Some(event) = rtc_events.recv().await { + if let Err(err) = Self::handle_rtc( + event, + signal_client.clone(), + engine_inner.clone(), + emitter.clone(), + ) + .await + { + error!("failed to handle rtc event: {:?}", err); + } + } + } + + async fn signal_task( + signal_client: Arc, + engine_inner: Arc, + mut signal_events: SignalEvents, + emitter: EngineEmitter, + ) { + while let Some(signal) = signal_events.recv().await { + match signal { + SignalEvent::Open => {} + SignalEvent::Signal(signal) => { + if let Err(err) = Self::handle_signal( + signal, + signal_client.clone(), + engine_inner.clone(), + emitter.clone(), + ) + .await + { + error!("failed to handle signal: {:?}", err); + } + } + SignalEvent::Close => { + // Try reconnect if this isn't expected + } + } + } + } + + async fn handle_rtc( + event: RTCEvent, + signal_client: Arc, + engine_inner: Arc, + emitter: EngineEmitter, + ) -> EngineResult<()> { + match event { + RTCEvent::IceCandidate { + ice_candidate, + target, + } => { + let json = serde_json::to_string(&IceCandidateJSON { + sdpMid: ice_candidate.sdp_mid(), + sdpMLineIndex: ice_candidate.sdp_mline_index(), + candidate: ice_candidate.candidate(), + })?; + + trace!("sending ice_candidate ({:?}) - {:?}", target, ice_candidate); + + tokio::spawn(async move { + signal_client + .send(signal_request::Message::Trickle(TrickleRequest { + candidate_init: json, + target: target as i32, + })) + .await; + }); + } + RTCEvent::ConnectionChange { state, target } => { + // Reconnect if we've been disconnected unexpectedly + let subscriber_primary = engine_inner.join_response.lock().subscriber_primary; + let is_primary = subscriber_primary && target == SignalTarget::Subscriber; + + if is_primary && state == PeerConnectionState::Disconnected { + let old_state = engine_inner + .pc_state + .swap(PCState::Connected as u8, Ordering::SeqCst); + if old_state == PCState::New as u8 { + // TODO(theomonnom) Handle disconnect + } + } else if state == PeerConnectionState::Failed { + engine_inner + .pc_state + .store(PCState::Disconnected as u8, Ordering::SeqCst); + // TODO(theomonnom) Handle disconnect + } + } + RTCEvent::DataChannel { + data_channel, + target, + } => { + if target == SignalTarget::Subscriber { + if data_channel.label() == RELIABLE_DC_LABEL { + *engine_inner.sub_reliable_dc.lock() = Some(data_channel); + } else { + *engine_inner.sub_lossy_dc.lock() = Some(data_channel); + } + } + } + RTCEvent::Offer { offer, target } => { + if target == SignalTarget::Publisher { + // Send the publisher offer to the server + tokio::spawn(async move { + signal_client + .send(signal_request::Message::Offer(proto::SessionDescription { + r#type: "offer".to_string(), + sdp: offer.to_string(), + })) + .await; + }); + } + } + RTCEvent::AddTrack { + rtp_receiver, + streams, + target, + } => { + if target == SignalTarget::Subscriber { + let _ = emitter.send(EngineEvent::AddTrack { + rtp_receiver, + streams, + }); + } + } + RTCEvent::Data { data, binary } => { + if !binary { + Err(EngineError::Internal( + "text messages aren't supported".to_string(), + ))?; + } + + let data = DataPacket::decode(&*data)?; + match data.value.unwrap() { + Value::User(user) => { + // TODO(theomonnom) Send event + } + Value::Speaker(_) => { + // TODO(theomonnonm) + } + } + } + } + + Ok(()) + } + + async fn handle_signal( + event: signal_response::Message, + signal_client: Arc, + engine_inner: Arc, + emitter: EngineEmitter, + ) -> EngineResult<()> { + match event { + signal_response::Message::Answer(answer) => { + trace!("received answer from the publisher: {:?}", answer); + + let sdp = SessionDescription::from(answer.r#type.parse().unwrap(), &answer.sdp)?; + engine_inner + .publisher_pc + .lock() + .await + .set_remote_description(sdp) + .await?; + } + signal_response::Message::Offer(offer) => { + // Handle the subscriber offer & send an answer to livekit-server + // We always get an offer from the server when connecting + trace!("received offer from the publisher: {:?}", offer); + let sdp = SessionDescription::from(offer.r#type.parse().unwrap(), &offer.sdp)?; + + engine_inner + .subscriber_pc + .lock() + .await + .set_remote_description(sdp) + .await?; + let answer = engine_inner + .subscriber_pc + .lock() + .await + .peer_connection() + .create_answer(RTCOfferAnswerOptions::default()) + .await?; + engine_inner + .subscriber_pc + .lock() + .await + .peer_connection() + .set_local_description(answer.clone()) + .await?; + + tokio::spawn(async move { + signal_client + .send(signal_request::Message::Answer(proto::SessionDescription { + r#type: "answer".to_string(), + sdp: answer.to_string(), + })) + .await; + }); + } + signal_response::Message::Trickle(trickle) => { + // Add the IceCandidate received from the livekit-server + let json: IceCandidateJSON = serde_json::from_str(&trickle.candidate_init)?; + let ice = IceCandidate::from(&json.sdpMid, json.sdpMLineIndex, &json.candidate)?; + + trace!( + "received ice_candidate {:?} - {:?}", + SignalTarget::from_i32(trickle.target).unwrap(), + ice + ); + + if trickle.target == SignalTarget::Publisher as i32 { + engine_inner + .publisher_pc + .lock() + .await + .add_ice_candidate(ice) + .await?; + } else { + engine_inner + .subscriber_pc + .lock() + .await + .add_ice_candidate(ice) + .await?; + } + } + signal_response::Message::Update(update) => { + let _ = emitter.send(EngineEvent::ParticipantUpdate(update)); + } + _ => {} + } + + Ok(()) + } + + async fn ensure_publisher_connected(&self, kind: data_packet::Kind) -> EngineResult<()> { + if !self.join_response().subscriber_primary { + return Ok(()); + } + + let publisher = &self.engine_inner.publisher_pc; + { + let mut publisher = publisher.lock().await; + if !publisher.is_connected() + && publisher.peer_connection().ice_connection_state() + != IceConnectionState::IceConnectionChecking + { + let _ = self.negotiate_publisher().await; + } + } + + let dc = self.data_channel(kind); + if dc.lock().state() == DataState::Open { + return Ok(()); + } + + // Wait until the PeerConnection is connected + let wait_connected = async move { + while publisher.lock().await.is_connected() && dc.lock().state() == DataState::Open { + sleep(Duration::from_millis(50)).await; + } + }; + + tokio::select! { + _ = wait_connected => Ok(()), + _ = sleep(MAX_ICE_CONNECT_TIMEOUT) => { + let err = EngineError::Connection("could not establish publisher connection: timeout".to_string()); + error!(error = ?err); + Err(err) + } + } + } + + async fn negotiate_publisher(&self) -> EngineResult<()> { + self.engine_inner + .has_published + .store(true, Ordering::SeqCst); + if let Err(err) = self + .engine_inner + .publisher_pc + .lock() + .await + .negotiate() + .await + { + error!("failed to negotiate the publisher: {:?}", err); + Err(err)? + } else { + Ok(()) + } + } + + fn configure_engine( + lk_runtime: Arc, + join_response: JoinResponse, + ) -> EngineResult<(EngineInner, RTCEvents)> { + let (rtc_emitter, events) = mpsc::unbounded_channel(); + let rtc_config = RTCConfiguration::from(join_response.clone()); + + let mut publisher_pc = PCTransport::new( + lk_runtime + .pc_factory + .create_peer_connection(rtc_config.clone())?, + ); + + let mut subscriber_pc = PCTransport::new( + lk_runtime + .pc_factory + .create_peer_connection(rtc_config.clone())?, + ); + + let mut lossy_dc = publisher_pc.peer_connection().create_data_channel( + LOSSY_DC_LABEL, + DataChannelInit { + ordered: true, + max_retransmits: Some(0), + ..DataChannelInit::default() + }, + )?; + + let mut reliable_dc = publisher_pc.peer_connection().create_data_channel( + RELIABLE_DC_LABEL, + DataChannelInit { + ordered: true, + ..DataChannelInit::default() + }, + )?; + + publisher_pc + .peer_connection() + .on_ice_candidate(rtc_events::on_ice_candidate( + SignalTarget::Publisher, + rtc_emitter.clone(), + )); + subscriber_pc + .peer_connection() + .on_ice_candidate(rtc_events::on_ice_candidate( + SignalTarget::Subscriber, + rtc_emitter.clone(), + )); + + publisher_pc.on_offer(rtc_events::on_offer( + SignalTarget::Publisher, + rtc_emitter.clone(), + )); + subscriber_pc.on_offer(rtc_events::on_offer( + SignalTarget::Subscriber, + rtc_emitter.clone(), + )); + + publisher_pc + .peer_connection() + .on_data_channel(rtc_events::on_data_channel( + SignalTarget::Publisher, + rtc_emitter.clone(), + )); + subscriber_pc + .peer_connection() + .on_data_channel(rtc_events::on_data_channel( + SignalTarget::Subscriber, + rtc_emitter.clone(), + )); + + publisher_pc + .peer_connection() + .on_add_track(rtc_events::on_add_track( + SignalTarget::Publisher, + rtc_emitter.clone(), + )); + subscriber_pc + .peer_connection() + .on_add_track(rtc_events::on_add_track( + SignalTarget::Subscriber, + rtc_emitter.clone(), + )); + + publisher_pc + .peer_connection() + .on_connection_change(rtc_events::on_connection_change( + SignalTarget::Publisher, + rtc_emitter.clone(), + )); + subscriber_pc + .peer_connection() + .on_connection_change(rtc_events::on_connection_change( + SignalTarget::Subscriber, + rtc_emitter.clone(), + )); + + lossy_dc.on_message(rtc_events::on_message(rtc_emitter.clone())); + reliable_dc.on_message(rtc_events::on_message(rtc_emitter.clone())); + + Ok(( + EngineInner { + has_published: AtomicBool::new(false), + join_response: Mutex::new(join_response), + pc_state: AtomicU8::new(PCState::New as u8), + publisher_pc: AsyncMutex::new(publisher_pc), + subscriber_pc: AsyncMutex::new(subscriber_pc), + lossy_dc: Mutex::new(lossy_dc), + reliable_dc: Mutex::new(reliable_dc), + sub_lossy_dc: Mutex::new(None), + sub_reliable_dc: Mutex::new(None), + }, + events, + )) + } + + fn data_channel(&self, kind: data_packet::Kind) -> &Mutex { + if kind == data_packet::Kind::Reliable { + &self.engine_inner.reliable_dc + } else { + &self.engine_inner.lossy_dc + } } } diff --git a/crates/livekit-core/src/pc_transport.rs b/crates/livekit-core/src/rtc_engine/pc_transport.rs similarity index 90% rename from crates/livekit-core/src/pc_transport.rs rename to crates/livekit-core/src/rtc_engine/pc_transport.rs index ca4f2b2..8d9676a 100644 --- a/crates/livekit-core/src/pc_transport.rs +++ b/crates/livekit-core/src/rtc_engine/pc_transport.rs @@ -13,7 +13,11 @@ use livekit_webrtc::rtc_error::RTCError; const NEGOTIATION_FREQUENCY: Duration = Duration::from_millis(150); -pub type OnOfferHandler = Box Pin + Send + 'static>>) + Send + Sync>; +pub type OnOfferHandler = Box< + dyn (FnMut(SessionDescription) -> Pin + Send + 'static>>) + + Send + + Sync, +>; pub struct PCTransport { peer_connection: PeerConnection, @@ -43,7 +47,7 @@ impl PCTransport { pub fn is_connected(&self) -> bool { self.peer_connection.ice_connection_state() == IceConnectionState::IceConnectionConnected || self.peer_connection.ice_connection_state() - == IceConnectionState::IceConnectionCompleted + == IceConnectionState::IceConnectionCompleted } pub fn peer_connection(&mut self) -> &mut PeerConnection { @@ -118,7 +122,10 @@ impl PCTransport { .set_remote_description(remote_description) .await?; } else { - event!(Level::ERROR, "trying to restart ICE when the pc doesn't have remote description"); + event!( + Level::ERROR, + "trying to restart ICE when the pc doesn't have remote description" + ); } } else { self.renegotiate = true; diff --git a/crates/livekit-core/src/rtc_engine/rtc_events.rs b/crates/livekit-core/src/rtc_engine/rtc_events.rs new file mode 100644 index 0000000..78712d7 --- /dev/null +++ b/crates/livekit-core/src/rtc_engine/rtc_events.rs @@ -0,0 +1,103 @@ +use livekit_webrtc::data_channel::{DataChannel, OnMessageHandler}; +use livekit_webrtc::jsep::{IceCandidate, SessionDescription}; +use livekit_webrtc::media_stream::MediaStream; +use livekit_webrtc::peer_connection::{ + OnAddTrackHandler, OnConnectionChangeHandler, OnDataChannelHandler, OnIceCandidateHandler, + PeerConnectionState, +}; +use livekit_webrtc::rtp_receiver::RtpReceiver; +use tokio::sync::mpsc; + +use crate::proto::SignalTarget; +use crate::rtc_engine::pc_transport::OnOfferHandler; + +pub(super) type RTCEmitter = mpsc::UnboundedSender; +pub(super) type RTCEvents = mpsc::UnboundedReceiver; + +#[derive(Debug)] +pub(super) enum RTCEvent { + IceCandidate { + ice_candidate: IceCandidate, + target: SignalTarget, + }, + ConnectionChange { + state: PeerConnectionState, + target: SignalTarget, + }, + DataChannel { + data_channel: DataChannel, + target: SignalTarget, + }, + Offer { + offer: SessionDescription, + target: SignalTarget, + }, + AddTrack { + rtp_receiver: RtpReceiver, + streams: Vec, + target: SignalTarget, + }, + Data { + data: Vec, + binary: bool, + }, +} + +/// Handlers used to forward event to a channel +/// Every callback here is called on the signaling thread + +pub(super) fn on_connection_change( + target: SignalTarget, + emitter: RTCEmitter, +) -> OnConnectionChangeHandler { + Box::new(move |state| { + let _ = emitter.send(RTCEvent::ConnectionChange { state, target }); + }) +} + +pub(super) fn on_ice_candidate(target: SignalTarget, emitter: RTCEmitter) -> OnIceCandidateHandler { + Box::new(move |ice_candidate| { + let _ = emitter.send(RTCEvent::IceCandidate { + ice_candidate, + target, + }); + }) +} + +pub(super) fn on_offer(target: SignalTarget, emitter: RTCEmitter) -> OnOfferHandler { + Box::new(move |offer| { + let _ = emitter.send(RTCEvent::Offer { offer, target }); + + Box::pin(async {}) + }) +} + +pub(super) fn on_data_channel(target: SignalTarget, emitter: RTCEmitter) -> OnDataChannelHandler { + Box::new(move |mut data_channel| { + data_channel.on_message(on_message(emitter.clone())); + + let _ = emitter.send(RTCEvent::DataChannel { + data_channel, + target, + }); + }) +} + +pub(super) fn on_add_track(target: SignalTarget, emitter: RTCEmitter) -> OnAddTrackHandler { + Box::new(move |rtp_receiver, streams| { + let _ = emitter.send(RTCEvent::AddTrack { + rtp_receiver, + streams, + target, + }); + }) +} + +pub(super) fn on_message(emitter: RTCEmitter) -> OnMessageHandler { + Box::new(move |data, binary| { + let _ = emitter.send(RTCEvent::Data { + data: data.to_vec(), + binary, + }); + }) +} diff --git a/crates/livekit-core/src/signal_client/mod.rs b/crates/livekit-core/src/signal_client/mod.rs index b02e526..9e9d7c4 100644 --- a/crates/livekit-core/src/signal_client/mod.rs +++ b/crates/livekit-core/src/signal_client/mod.rs @@ -1,18 +1,23 @@ -use core::num::flt2dec::Sign; use std::fmt::Debug; +use std::time::Duration; +use livekit_webrtc::peer_connection_factory::{ + ContinualGatheringPolicy, ICEServer, IceTransportsType, RTCConfiguration, +}; use thiserror::Error; +use tokio::sync::mpsc; use tokio_tungstenite::tungstenite::Error as WsError; -use crate::event::{Emitter, Events}; -use crate::proto::{signal_request, signal_response}; +use crate::proto::{signal_request, signal_response, JoinResponse}; use crate::signal_client::signal_stream::SignalStream; mod signal_stream; -type SignalEmitter = Emitter; -type SignalEvents = Events; -type SignalResult = Result; +pub(crate) type SignalEmitter = mpsc::Sender; +pub(crate) type SignalEvents = mpsc::Receiver; +pub(crate) type SignalResult = Result; + +pub const JOIN_RESPONSE_TIMEOUT: Duration = Duration::from_secs(5); #[derive(Error, Debug)] pub enum SignalError { @@ -22,10 +27,12 @@ pub enum SignalError { UrlParse(#[from] url::ParseError), #[error("failed to decode messages from server")] ProtoParse(#[from] prost::DecodeError), + #[error("{0}")] + Timeout(String), } /// Events used by the RTCEngine who will handle the reconnection logic -#[derive(Clone, Debug)] +#[derive(Debug)] pub(crate) enum SignalEvent { Open, Signal(signal_response::Message), @@ -40,6 +47,17 @@ pub(crate) struct SignalOptions { adaptive_stream: bool, } +impl Default for SignalOptions { + fn default() -> Self { + Self { + reconnect: false, + auto_subscribe: true, + sid: "".to_string(), + adaptive_stream: true, + } + } +} + #[derive(Debug)] pub struct SignalClient { stream: SignalStream, @@ -47,15 +65,16 @@ pub struct SignalClient { } impl SignalClient { - pub async fn connect( + pub(crate) async fn connect( url: &str, token: &str, options: SignalOptions, ) -> SignalResult<(Self, SignalEvents)> { - // TODO(theomonnom) Retry initial connection - let (emitter, receiver) = SignalEmitter::new(); - let events = SignalEvents::new(receiver); + let (emitter, events) = mpsc::channel(8); let stream = SignalStream::connect(url, token, options, emitter.clone()).await?; + + // TODO(theomonnom) Retry initial connection + Ok((Self { stream, emitter }, events)) } @@ -69,3 +88,60 @@ impl SignalClient { // TODO(theomonnom) Close & recreate SignalStream, also send the queue if needed } } + +impl From for RTCConfiguration { + fn from(join_response: JoinResponse) -> Self { + Self { + ice_servers: { + let mut servers = vec![]; + for ice_server in join_response.ice_servers.clone() { + servers.push(ICEServer { + urls: ice_server.urls, + username: ice_server.username, + password: ice_server.credential, + }) + } + servers + }, + continual_gathering_policy: ContinualGatheringPolicy::GatherContinually, + ice_transport_type: IceTransportsType::All, + } + } +} + +pub mod utils { + use crate::proto::{signal_response, JoinResponse}; + use crate::signal_client::{SignalError, SignalEvent, SignalResult, JOIN_RESPONSE_TIMEOUT}; + use tokio::sync::mpsc; + use tokio::time::timeout; + use tokio_tungstenite::tungstenite::Error as WsError; + use tracing::{event, Level}; + + pub(crate) async fn next_join_response( + receiver: &mut mpsc::Receiver, + ) -> SignalResult { + let join = async { + while let Some(event) = receiver.recv().await { + match event { + SignalEvent::Signal(signal_response::Message::Join(join)) => return Ok(join), + SignalEvent::Close => break, + SignalEvent::Open => continue, + _ => { + event!( + Level::WARN, + "received unexpected message while waiting for JoinResponse: {:?}", + event + ); + continue; + } + } + } + + Err(WsError::ConnectionClosed)? + }; + + timeout(JOIN_RESPONSE_TIMEOUT, join) + .await + .map_err(|_| SignalError::Timeout("failed to receive JoinResponse".to_string()))? + } +} diff --git a/crates/livekit-core/src/signal_client/signal_stream.rs b/crates/livekit-core/src/signal_client/signal_stream.rs index 5c0d7bd..eb0680d 100644 --- a/crates/livekit-core/src/signal_client/signal_stream.rs +++ b/crates/livekit-core/src/signal_client/signal_stream.rs @@ -1,13 +1,13 @@ -use futures_util::{SinkExt, StreamExt}; use futures_util::stream::{SplitSink, SplitStream}; +use futures_util::{SinkExt, StreamExt}; use prost::Message as ProstMessage; use tokio::net::TcpStream; use tokio::sync::{mpsc, oneshot}; use tokio::task::JoinHandle; -use tokio_tungstenite::{connect_async, MaybeTlsStream, WebSocketStream}; -use tokio_tungstenite::tungstenite::Message; -use tokio_tungstenite::tungstenite::protocol::CloseFrame; use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode; +use tokio_tungstenite::tungstenite::protocol::CloseFrame; +use tokio_tungstenite::tungstenite::Message; +use tokio_tungstenite::{connect_async, MaybeTlsStream, WebSocketStream}; use tracing::{event, Level}; use crate::proto::{signal_request, SignalRequest, SignalResponse}; @@ -72,7 +72,7 @@ impl SignalStream { event!(Level::DEBUG, "connecting to websocket: {}", lk_url); let (ws_stream, _) = connect_async(lk_url).await?; event!(Level::DEBUG, "connected to websocket"); - emitter.event(SignalEvent::Open); + let _ = emitter.send(SignalEvent::Open).await; let (ws_writer, ws_reader) = ws_stream.split(); let (internal_tx, internal_rx) = mpsc::channel::(8); @@ -119,7 +119,7 @@ impl SignalStream { /// This task is used to send messages to the websocket /// It is also responsible for closing the connection - pub async fn handle_write( + async fn handle_write( mut internal_rx: mpsc::Receiver, mut ws_writer: SplitSink, emitter: SignalEmitter, @@ -136,7 +136,7 @@ impl SignalStream { SignalRequest { message: Some(signal), } - .encode_to_vec(), + .encode_to_vec(), ); if let Err(err) = ws_writer.send(data).await { @@ -163,14 +163,14 @@ impl SignalStream { } let _ = ws_writer.close().await; - emitter.event(SignalEvent::Close); + let _ = emitter.send(SignalEvent::Close).await; } /// This task is used to read incoming messages from the websocket /// and dispatch them through the EventEmitter. /// /// It can also send messages to [handle_write] task ( Used e.g. answer to pings ) - pub async fn handle_read( + async fn handle_read( internal_tx: mpsc::Sender, mut ws_reader: SplitStream, emitter: SignalEmitter, @@ -181,8 +181,9 @@ impl SignalStream { let res = SignalResponse::decode(data.as_slice()) .expect("failed to decode SignalResponse"); - event!(Level::TRACE, "received SignalResponse: {:?}", res); - emitter.event(SignalEvent::Signal(res.message.unwrap())); + let msg = res.message.unwrap(); + event!(Level::TRACE, "received SignalResponse: {:?}", msg); + let _ = emitter.send(SignalEvent::Signal(msg)).await; } Ok(Message::Ping(data)) => { let _ = internal_tx diff --git a/crates/livekit-webrtc/libwebrtc-sys/build.rs b/crates/livekit-webrtc/libwebrtc-sys/build.rs index 909f297..c6d95cc 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/build.rs +++ b/crates/livekit-webrtc/libwebrtc-sys/build.rs @@ -71,7 +71,7 @@ fn main() { let mut builder = cxx_build::bridges(&[ "src/peer_connection.rs", "src/peer_connection_factory.rs", - "src/media_stream_interface.rs", + "src/media_stream.rs", "src/data_channel.rs", "src/jsep.rs", "src/candidate.rs", @@ -83,7 +83,7 @@ fn main() { builder.file("src/peer_connection.cpp"); builder.file("src/peer_connection_factory.cpp"); - builder.file("src/media_stream_interface.cpp"); + builder.file("src/media_stream.cpp"); builder.file("src/data_channel.cpp"); builder.file("src/jsep.cpp"); builder.file("src/candidate.cpp"); @@ -203,7 +203,7 @@ fn main() { let jni_regex = Regex::new(r"(Java_org_webrtc.*)").unwrap(); let content = &String::from_utf8_lossy(&readelf_output.stdout); let mut jni_symbols = Vec::new(); - jni_regex.captures_iter(&content).for_each(|cap| { + jni_regex.captures_iter(content).for_each(|cap| { jni_symbols.push(cap.get(1).unwrap().as_str()); }); @@ -215,7 +215,7 @@ fn main() { write!(vs_file, "JNI_WEBRTC {{\n\tglobal: ").unwrap(); write!(vs_file, "JNI_OnLoad; ").unwrap(); - for x in &jni_symbols { + for x in jni_symbols { println!("cargo:rustc-link-arg=-Wl,--undefined={}", x); write!(vs_file, "{}; ", x).unwrap(); } diff --git a/crates/livekit-webrtc/libwebrtc-sys/include/livekit/media_stream.h b/crates/livekit-webrtc/libwebrtc-sys/include/livekit/media_stream.h new file mode 100644 index 0000000..c2b3164 --- /dev/null +++ b/crates/livekit-webrtc/libwebrtc-sys/include/livekit/media_stream.h @@ -0,0 +1,52 @@ +// +// Created by Théo Monnom on 31/08/2022. +// + +#ifndef CLIENT_SDK_NATIVE_MEDIA_STREAM_INTERFACE_H +#define CLIENT_SDK_NATIVE_MEDIA_STREAM_INTERFACE_H + +#include + +#include "api/media_stream_interface.h" +#include "livekit/rust_types.h" +#include "rust/cxx.h" + +namespace livekit { + +class MediaStreamTrack { + public: + explicit MediaStreamTrack( + rtc::scoped_refptr track); + + rust::String kind() const; + rust::String id() const; + + bool enabled() const; + bool set_enabled(bool enable); + + TrackState state() const; + + private: + rtc::scoped_refptr track_; +}; + +static std::unique_ptr _unique_media_stream_track() { + return nullptr; // Ignore +} + +class MediaStream { + public: + explicit MediaStream(rtc::scoped_refptr stream); + + rust::String id() const; + + private: + rtc::scoped_refptr media_stream_; +}; + +static std::unique_ptr _unique_media_stream() { + return nullptr; // Ignore +} +} // namespace livekit + +#endif // CLIENT_SDK_NATIVE_MEDIA_STREAM_INTERFACE_H diff --git a/crates/livekit-webrtc/libwebrtc-sys/include/livekit/media_stream_interface.h b/crates/livekit-webrtc/libwebrtc-sys/include/livekit/media_stream_interface.h deleted file mode 100644 index 41f71c4..0000000 --- a/crates/livekit-webrtc/libwebrtc-sys/include/livekit/media_stream_interface.h +++ /dev/null @@ -1,28 +0,0 @@ -// -// Created by Théo Monnom on 31/08/2022. -// - -#ifndef CLIENT_SDK_NATIVE_MEDIA_STREAM_INTERFACE_H -#define CLIENT_SDK_NATIVE_MEDIA_STREAM_INTERFACE_H - -#include - -#include "api/media_stream_interface.h" - -namespace livekit { - -class MediaStreamInterface { - public: - explicit MediaStreamInterface( - rtc::scoped_refptr stream); - - private: - rtc::scoped_refptr media_stream_; -}; - -static std::unique_ptr _unique_media_stream() { - return nullptr; // Ignore -} -} // namespace livekit - -#endif // CLIENT_SDK_NATIVE_MEDIA_STREAM_INTERFACE_H diff --git a/crates/livekit-webrtc/libwebrtc-sys/include/livekit/rtp_receiver.h b/crates/livekit-webrtc/libwebrtc-sys/include/livekit/rtp_receiver.h index 636fb79..f541cf5 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/include/livekit/rtp_receiver.h +++ b/crates/livekit-webrtc/libwebrtc-sys/include/livekit/rtp_receiver.h @@ -8,6 +8,7 @@ #include #include "api/rtp_receiver_interface.h" +#include "livekit/media_stream.h" namespace livekit { @@ -16,6 +17,8 @@ class RtpReceiver { explicit RtpReceiver( rtc::scoped_refptr receiver); + std::unique_ptr track() const; + private: rtc::scoped_refptr receiver_; }; diff --git a/crates/livekit-webrtc/libwebrtc-sys/include/livekit/rust_types.h b/crates/livekit-webrtc/libwebrtc-sys/include/livekit/rust_types.h index 4f766a1..1d9d7ef 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/include/livekit/rust_types.h +++ b/crates/livekit-webrtc/libwebrtc-sys/include/livekit/rust_types.h @@ -23,6 +23,7 @@ enum class IceConnectionState; enum class IceGatheringState; enum class SdpType; enum class DataState; +enum class TrackState; struct SdpParseError; struct RTCOfferAnswerOptions; struct RTCError; diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/jsep.rs b/crates/livekit-webrtc/libwebrtc-sys/src/jsep.rs index 7b15d29..4ba65c2 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/src/jsep.rs +++ b/crates/livekit-webrtc/libwebrtc-sys/src/jsep.rs @@ -68,11 +68,18 @@ pub mod ffi { observer: Box, ) -> UniquePtr; - fn create_ice_candidate(sdp_mid: String, sdp_mline_index: i32, sdp: String) -> Result>; - fn create_session_description(sdp_type: SdpType, sdp: String) -> Result>; + fn create_ice_candidate( + sdp_mid: String, + sdp_mline_index: i32, + sdp: String, + ) -> Result>; + fn create_session_description( + sdp_type: SdpType, + sdp: String, + ) -> Result>; fn _unique_ice_candidate() -> UniquePtr; // Ignore - fn _unique_session_description() -> UniquePtr; // Ignore + fn _unique_session_description() -> UniquePtr; // Ignore } } @@ -80,7 +87,11 @@ impl Error for ffi::SdpParseError {} impl Display for ffi::SdpParseError { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - write!(f, "SdpParseError occurred {}: {}", self.line, self.description) + write!( + f, + "SdpParseError occurred {}: {}", + self.line, self.description + ) } } @@ -101,10 +112,7 @@ impl ffi::SdpParseError { let line = String::from(&value[8..line_length]); let description = String::from(&value[line_length..]); - Self { - line, - description, - } + Self { line, description } } } diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/lib.rs b/crates/livekit-webrtc/libwebrtc-sys/src/lib.rs index 9959239..ecd1f3e 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/src/lib.rs +++ b/crates/livekit-webrtc/libwebrtc-sys/src/lib.rs @@ -1,7 +1,7 @@ pub mod candidate; pub mod data_channel; pub mod jsep; -pub mod media_stream_interface; +pub mod media_stream; pub mod peer_connection; pub mod peer_connection_factory; pub mod rtc_error; diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/media_stream.cpp b/crates/livekit-webrtc/libwebrtc-sys/src/media_stream.cpp new file mode 100644 index 0000000..04eb1f9 --- /dev/null +++ b/crates/livekit-webrtc/libwebrtc-sys/src/media_stream.cpp @@ -0,0 +1,41 @@ +// +// Created by Théo Monnom on 31/08/2022. +// + +#include "livekit/media_stream.h" + +namespace livekit { + +MediaStreamTrack::MediaStreamTrack( + rtc::scoped_refptr track) + : track_(std::move(track)) {} + +rust::String MediaStreamTrack::kind() const { + return track_->kind(); +} + +rust::String MediaStreamTrack::id() const { + return track_->id(); +} + +bool MediaStreamTrack::enabled() const { + return track_->enabled(); +} + +bool MediaStreamTrack::set_enabled(bool enable) { + return track_->set_enabled(enable); +} + +TrackState MediaStreamTrack::state() const { + return static_cast(track_->state()); +} + +MediaStream::MediaStream( + rtc::scoped_refptr stream) + : media_stream_(std::move(stream)) {} + +rust::String MediaStream::id() const { + return media_stream_->id(); +} + +} // namespace livekit \ No newline at end of file diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/media_stream.rs b/crates/livekit-webrtc/libwebrtc-sys/src/media_stream.rs new file mode 100644 index 0000000..0f46ebf --- /dev/null +++ b/crates/livekit-webrtc/libwebrtc-sys/src/media_stream.rs @@ -0,0 +1,36 @@ +#[cxx::bridge(namespace = "livekit")] +pub mod ffi { + + #[derive(Debug)] + #[repr(i32)] + pub enum TrackState { + Live, + Ended, + } + + unsafe extern "C++" { + include!("livekit/media_stream.h"); + + type MediaStreamTrack; + type MediaStream; + + fn kind(self: &MediaStreamTrack) -> String; + fn id(self: &MediaStreamTrack) -> String; + fn enabled(self: &MediaStreamTrack) -> bool; + fn set_enabled(self: Pin<&mut MediaStreamTrack>, enable: bool) -> bool; + fn state(self: &MediaStreamTrack) -> TrackState; + + fn id(self: &MediaStream) -> String; + + fn _unique_media_stream_track() -> UniquePtr; // Ignore + fn _unique_media_stream() -> UniquePtr; // Ignore + } +} + +unsafe impl Sync for ffi::MediaStreamTrack {} + +unsafe impl Send for ffi::MediaStreamTrack {} + +unsafe impl Sync for ffi::MediaStream {} + +unsafe impl Send for ffi::MediaStream {} \ No newline at end of file diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/media_stream_interface.cpp b/crates/livekit-webrtc/libwebrtc-sys/src/media_stream_interface.cpp deleted file mode 100644 index 8f5936c..0000000 --- a/crates/livekit-webrtc/libwebrtc-sys/src/media_stream_interface.cpp +++ /dev/null @@ -1,12 +0,0 @@ -// -// Created by Théo Monnom on 31/08/2022. -// - -#include "livekit/media_stream_interface.h" - -namespace livekit { - -MediaStreamInterface::MediaStreamInterface( - rtc::scoped_refptr stream) - : media_stream_(std::move(stream)) {} -} // namespace livekit \ No newline at end of file diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/media_stream_interface.rs b/crates/livekit-webrtc/libwebrtc-sys/src/media_stream_interface.rs deleted file mode 100644 index 1999583..0000000 --- a/crates/livekit-webrtc/libwebrtc-sys/src/media_stream_interface.rs +++ /dev/null @@ -1,10 +0,0 @@ -#[cxx::bridge(namespace = "livekit")] -pub mod ffi { - unsafe extern "C++" { - include!("livekit/media_stream_interface.h"); - - type MediaStreamInterface; - - fn _unique_media_stream() -> UniquePtr; // Ignore - } -} diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/peer_connection.cpp b/crates/livekit-webrtc/libwebrtc-sys/src/peer_connection.cpp index 203c164..1ad9738 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/src/peer_connection.cpp +++ b/crates/livekit-webrtc/libwebrtc-sys/src/peer_connection.cpp @@ -3,6 +3,7 @@ // #include "livekit/peer_connection.h" +#include "livekit/media_stream.h" #include "libwebrtc-sys/src/peer_connection.rs.h" #include "livekit/rtc_error.h" @@ -142,12 +143,12 @@ void NativePeerConnectionObserver::OnSignalingChange( void NativePeerConnectionObserver::OnAddStream( rtc::scoped_refptr stream) { - observer_->on_add_stream(std::make_unique(stream)); + observer_->on_add_stream(std::make_unique(stream)); } void NativePeerConnectionObserver::OnRemoveStream( rtc::scoped_refptr stream) { - observer_->on_remove_stream(std::make_unique(stream)); + observer_->on_remove_stream(std::make_unique(stream)); } void NativePeerConnectionObserver::OnDataChannel( @@ -241,7 +242,7 @@ void NativePeerConnectionObserver::OnAddTrack( rust::Vec vec; for (const auto& item : streams) { - vec.push_back(MediaStreamPtr{std::make_unique(item)}); + vec.push_back(MediaStreamPtr{std::make_unique(item)}); } observer_->on_add_track(std::make_unique(receiver), diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/peer_connection.rs b/crates/livekit-webrtc/libwebrtc-sys/src/peer_connection.rs index 6b82f5f..f6b914f 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/src/peer_connection.rs +++ b/crates/livekit-webrtc/libwebrtc-sys/src/peer_connection.rs @@ -6,7 +6,7 @@ use cxx::UniquePtr; use crate::candidate::ffi::Candidate; use crate::data_channel::ffi::DataChannel; use crate::jsep::ffi::IceCandidate; -use crate::media_stream_interface::ffi::MediaStreamInterface; +use crate::media_stream::ffi::MediaStream; use crate::rtc_error::ffi::RTCError; use crate::rtp_receiver::ffi::RtpReceiver; use crate::rtp_transceiver::ffi::RtpTransceiver; @@ -83,7 +83,7 @@ pub mod ffi { // Wrapper to opaque C++ objects // https://github.com/dtolnay/cxx/issues/741 struct MediaStreamPtr { - pub ptr: UniquePtr, + pub ptr: UniquePtr, } struct CandidatePtr { @@ -96,7 +96,7 @@ pub mod ffi { include!("livekit/data_channel.h"); include!("livekit/rtp_receiver.h"); include!("livekit/rtp_transceiver.h"); - include!("livekit/media_stream_interface.h"); + include!("livekit/media_stream.h"); include!("livekit/candidate.h"); include!("libwebrtc-sys/src/rtc_error.rs.h"); @@ -106,7 +106,7 @@ pub mod ffi { type DataChannel = crate::data_channel::ffi::DataChannel; type RtpReceiver = crate::rtp_receiver::ffi::RtpReceiver; type RtpTransceiver = crate::rtp_transceiver::ffi::RtpTransceiver; - type MediaStreamInterface = crate::media_stream_interface::ffi::MediaStreamInterface; + type MediaStream = crate::media_stream::ffi::MediaStream; type NativeCreateSdpObserverHandle = crate::jsep::ffi::NativeCreateSdpObserverHandle; type NativeSetLocalSdpObserverHandle = crate::jsep::ffi::NativeSetLocalSdpObserverHandle; type NativeSetRemoteSdpObserverHandle = crate::jsep::ffi::NativeSetRemoteSdpObserverHandle; @@ -194,14 +194,8 @@ pub mod ffi { type PeerConnectionObserverWrapper; fn on_signaling_change(self: &PeerConnectionObserverWrapper, new_state: SignalingState); - fn on_add_stream( - self: &PeerConnectionObserverWrapper, - stream: UniquePtr, - ); - fn on_remove_stream( - self: &PeerConnectionObserverWrapper, - stream: UniquePtr, - ); + fn on_add_stream(self: &PeerConnectionObserverWrapper, stream: UniquePtr); + fn on_remove_stream(self: &PeerConnectionObserverWrapper, stream: UniquePtr); fn on_data_channel( self: &PeerConnectionObserverWrapper, data_channel: UniquePtr, @@ -317,8 +311,8 @@ impl AddIceCandidateObserverWrapper { pub trait PeerConnectionObserver: Send + Sync { fn on_signaling_change(&self, new_state: ffi::SignalingState); - fn on_add_stream(&self, stream: UniquePtr); - fn on_remove_stream(&self, stream: UniquePtr); + fn on_add_stream(&self, stream: UniquePtr); + fn on_remove_stream(&self, stream: UniquePtr); fn on_data_channel(&self, data_channel: UniquePtr); fn on_renegotiation_needed(&self); fn on_negotiation_needed_event(&self, event: u32); @@ -338,11 +332,7 @@ pub trait PeerConnectionObserver: Send + Sync { fn on_ice_candidates_removed(&self, removed: Vec>); fn on_ice_connection_receiving_change(&self, receiving: bool); fn on_ice_selected_candidate_pair_changed(&self, event: ffi::CandidatePairChangeEvent); - fn on_add_track( - &self, - receiver: UniquePtr, - streams: Vec>, - ); + fn on_add_track(&self, receiver: UniquePtr, streams: Vec>); fn on_track(&self, transceiver: UniquePtr); fn on_remove_track(&self, receiver: UniquePtr); fn on_interesting_usage(&self, usage_pattern: i32); @@ -366,13 +356,13 @@ impl PeerConnectionObserverWrapper { } } - fn on_add_stream(&self, stream: UniquePtr) { + fn on_add_stream(&self, stream: UniquePtr) { unsafe { (*self.observer).on_add_stream(stream); } } - fn on_remove_stream(&self, stream: UniquePtr) { + fn on_remove_stream(&self, stream: UniquePtr) { unsafe { (*self.observer).on_remove_stream(stream); } diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/peer_connection_factory.rs b/crates/livekit-webrtc/libwebrtc-sys/src/peer_connection_factory.rs index 690f605..6266242 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/src/peer_connection_factory.rs +++ b/crates/livekit-webrtc/libwebrtc-sys/src/peer_connection_factory.rs @@ -35,12 +35,14 @@ pub mod ffi { type PeerConnection = crate::peer_connection::ffi::PeerConnection; type NativePeerConnectionObserver = - crate::peer_connection::ffi::NativePeerConnectionObserver; + crate::peer_connection::ffi::NativePeerConnectionObserver; type PeerConnectionFactory; type NativeRTCConfiguration; type RTCRuntime = crate::webrtc::ffi::RTCRuntime; - fn create_peer_connection_factory(runtime: SharedPtr) -> UniquePtr; + fn create_peer_connection_factory( + runtime: SharedPtr, + ) -> UniquePtr; fn create_rtc_configuration(conf: RTCConfiguration) -> UniquePtr; /// SAFETY diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/rtp_receiver.cpp b/crates/livekit-webrtc/libwebrtc-sys/src/rtp_receiver.cpp index b413a32..e9095c0 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/src/rtp_receiver.cpp +++ b/crates/livekit-webrtc/libwebrtc-sys/src/rtp_receiver.cpp @@ -8,4 +8,9 @@ namespace livekit { RtpReceiver::RtpReceiver( rtc::scoped_refptr receiver) : receiver_(std::move(receiver)) {} + +std::unique_ptr RtpReceiver::track() const { + return std::make_unique(receiver_->track()); +} + } // namespace livekit \ No newline at end of file diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/rtp_receiver.rs b/crates/livekit-webrtc/libwebrtc-sys/src/rtp_receiver.rs index 6b6cc92..34327de 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/src/rtp_receiver.rs +++ b/crates/livekit-webrtc/libwebrtc-sys/src/rtp_receiver.rs @@ -2,9 +2,17 @@ pub mod ffi { unsafe extern "C++" { include!("livekit/rtp_receiver.h"); + include!("livekit/media_stream.h"); + type MediaStreamTrack = crate::media_stream::ffi::MediaStreamTrack; type RtpReceiver; + fn track(self: &RtpReceiver) -> UniquePtr; + fn _unique_rtp_receiver() -> UniquePtr; // Ignore } } + +unsafe impl Sync for ffi::RtpReceiver {} + +unsafe impl Send for ffi::RtpReceiver {} \ No newline at end of file diff --git a/crates/livekit-webrtc/src/data_channel.rs b/crates/livekit-webrtc/src/data_channel.rs index 9af7620..a45c6ca 100644 --- a/crates/livekit-webrtc/src/data_channel.rs +++ b/crates/livekit-webrtc/src/data_channel.rs @@ -99,6 +99,12 @@ impl DataChannel { } } +impl Drop for DataChannel { + fn drop(&mut self) { + self.cxx_handle.pin_mut().unregister_observer(); + } +} + pub type OnStateChangeHandler = Box; pub type OnMessageHandler = Box; // data, is_binary diff --git a/crates/livekit-webrtc/src/jsep.rs b/crates/livekit-webrtc/src/jsep.rs index 8117830..228ba62 100644 --- a/crates/livekit-webrtc/src/jsep.rs +++ b/crates/livekit-webrtc/src/jsep.rs @@ -17,8 +17,16 @@ impl Debug for IceCandidate { } impl IceCandidate { - pub fn from(sdp_mid: &str, sdp_mline_index: i32, sdp: &str) -> Result { - let res = sys_jsep::ffi::create_ice_candidate(sdp_mid.to_string(), sdp_mline_index, sdp.to_string()); + pub fn from( + sdp_mid: &str, + sdp_mline_index: i32, + sdp: &str, + ) -> Result { + let res = sys_jsep::ffi::create_ice_candidate( + sdp_mid.to_string(), + sdp_mline_index, + sdp.to_string(), + ); match res { Ok(cxx_handle) => Ok(IceCandidate::new(cxx_handle)), diff --git a/crates/livekit-webrtc/src/media_stream.rs b/crates/livekit-webrtc/src/media_stream.rs index 9a693e7..3fff376 100644 --- a/crates/livekit-webrtc/src/media_stream.rs +++ b/crates/livekit-webrtc/src/media_stream.rs @@ -1,2 +1,51 @@ -#[derive(Debug)] -pub struct MediaStream {} +use cxx::UniquePtr; + +use libwebrtc_sys::media_stream as sys_ms; + +pub use sys_ms::ffi::TrackState; + +pub struct MediaStreamTrack { + cxx_handle: UniquePtr, +} + +impl MediaStreamTrack { + pub(crate) fn new(cxx_handle: UniquePtr) -> Self { + Self { cxx_handle } + } + + fn kind(&self) -> String { + self.cxx_handle.kind() + } + + fn id(&self) -> String { + self.cxx_handle.id() + } + + fn enabled(&self) -> bool { + self.cxx_handle.enabled() + } + + fn set_enabled(&mut self, enable: bool) -> bool { + self.cxx_handle.pin_mut().set_enabled(enable) + } + + fn state(&self) -> TrackState { + self.cxx_handle.state() + } +} + +pub struct MediaStream { + cxx_handle: UniquePtr, +} + +impl MediaStream { + pub(crate) fn new(cxx_handle: UniquePtr) -> Self { + Self { + cxx_handle + } + } + + pub fn id(&self) -> String { + self.cxx_handle.id() + } +} \ No newline at end of file diff --git a/crates/livekit-webrtc/src/peer_connection.rs b/crates/livekit-webrtc/src/peer_connection.rs index 1b0dedd..4d44676 100644 --- a/crates/livekit-webrtc/src/peer_connection.rs +++ b/crates/livekit-webrtc/src/peer_connection.rs @@ -402,10 +402,7 @@ impl sys_pc::PeerConnectionObserver for InternalObserver { } } - fn on_add_stream( - &self, - stream: UniquePtr, - ) { + fn on_add_stream(&self, stream: UniquePtr) { trace!("on_add_stream"); let mut handler = self.on_add_stream_handler.lock().unwrap(); if let Some(f) = handler.as_mut() { @@ -413,10 +410,7 @@ impl sys_pc::PeerConnectionObserver for InternalObserver { } } - fn on_remove_stream( - &self, - stream: UniquePtr, - ) { + fn on_remove_stream(&self, stream: UniquePtr) { trace!("on_remove_stream"); let mut handler = self.on_remove_stream_handler.lock().unwrap(); if let Some(f) = handler.as_mut() { @@ -548,12 +542,13 @@ impl sys_pc::PeerConnectionObserver for InternalObserver { fn on_add_track( &self, receiver: UniquePtr, - streams: Vec>, + streams: Vec>, ) { trace!("on_add_track"); let mut handler = self.on_add_track_handler.lock().unwrap(); if let Some(f) = handler.as_mut() { - // TODO(theomonnom) + let streams = streams.into_iter().map(MediaStream::new).collect(); + f(RtpReceiver::new(receiver), streams) } } diff --git a/crates/livekit-webrtc/src/peer_connection_factory.rs b/crates/livekit-webrtc/src/peer_connection_factory.rs index 09f06ff..90291b3 100644 --- a/crates/livekit-webrtc/src/peer_connection_factory.rs +++ b/crates/livekit-webrtc/src/peer_connection_factory.rs @@ -18,7 +18,9 @@ pub struct PeerConnectionFactory { impl PeerConnectionFactory { pub fn new(rtc_runtime: RTCRuntime) -> Self { Self { - cxx_handle: sys_factory::ffi::create_peer_connection_factory(rtc_runtime.clone().release()), + cxx_handle: sys_factory::ffi::create_peer_connection_factory( + rtc_runtime.clone().release(), + ), rtc_runtime, } } @@ -31,8 +33,9 @@ impl PeerConnectionFactory { unsafe { let mut observer = Box::new(InternalObserver::default()); - let mut native_observer = sys_pc::ffi::create_native_peer_connection_observer(self.rtc_runtime.clone().release(), - Box::new(sys_pc::PeerConnectionObserverWrapper::new(&mut *observer)), + let mut native_observer = sys_pc::ffi::create_native_peer_connection_observer( + self.rtc_runtime.clone().release(), + Box::new(sys_pc::PeerConnectionObserverWrapper::new(&mut *observer)), ); let res = self diff --git a/crates/livekit-webrtc/src/rtc_error.rs b/crates/livekit-webrtc/src/rtc_error.rs index e054cf4..86c81e1 100644 --- a/crates/livekit-webrtc/src/rtc_error.rs +++ b/crates/livekit-webrtc/src/rtc_error.rs @@ -1,3 +1,2 @@ // TODO(theomonnom) Wrap the RTCError ffi so we can use Option(u16) pub use libwebrtc_sys::rtc_error::ffi::RTCError; - diff --git a/crates/livekit-webrtc/src/rtp_receiver.rs b/crates/livekit-webrtc/src/rtp_receiver.rs index d7f431b..0bcfee3 100644 --- a/crates/livekit-webrtc/src/rtp_receiver.rs +++ b/crates/livekit-webrtc/src/rtp_receiver.rs @@ -1,2 +1,17 @@ -#[derive(Debug)] -pub struct RtpReceiver {} +use crate::media_stream::MediaStreamTrack; +use cxx::UniquePtr; +use libwebrtc_sys::rtp_receiver as sys_rec; + +pub struct RtpReceiver { + cxx_handle: UniquePtr, +} + +impl RtpReceiver { + pub(crate) fn new(cxx_handle: UniquePtr) -> Self { + Self { cxx_handle } + } + + pub fn track(&self) -> MediaStreamTrack { + MediaStreamTrack::new(self.cxx_handle.track()) + } +} diff --git a/examples/Cargo.lock b/examples/Cargo.lock index 9a656ed..c82ce3c 100644 --- a/examples/Cargo.lock +++ b/examples/Cargo.lock @@ -472,6 +472,7 @@ dependencies = [ "futures-util", "lazy_static", "livekit-webrtc", + "parking_lot", "prost", "prost-build", "prost-types", @@ -931,6 +932,7 @@ dependencies = [ name = "simple_room" version = "0.1.0" dependencies = [ + "futures", "livekit", "tokio", "tracing", diff --git a/examples/simple_room/Cargo.toml b/examples/simple_room/Cargo.toml index 85f7276..1f13b23 100644 --- a/examples/simple_room/Cargo.toml +++ b/examples/simple_room/Cargo.toml @@ -7,4 +7,5 @@ edition = "2021" tokio = { version = "1", features = ["full"] } tracing = "0.1" tracing-subscriber = "0.3" -livekit = { path = "../.." } \ No newline at end of file +livekit = { path = "../.." } +futures = "0.3" \ No newline at end of file diff --git a/examples/simple_room/src/main.rs b/examples/simple_room/src/main.rs index 1c09fbe..52b161c 100644 --- a/examples/simple_room/src/main.rs +++ b/examples/simple_room/src/main.rs @@ -1,7 +1,6 @@ -use std::time::Duration; -use tokio::time::sleep; -use livekit::proto::data_packet; -use livekit::room; +use livekit::room::Room; +use std::sync::{Arc, Mutex}; +use tracing::{info, trace}; const URL: &str = "ws://localhost:7880"; const TOKEN : &str = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjIzODQ4MDY0NzMsImlzcyI6IkFQSXpLYkFTaUNWYWtnSiIsIm5hbWUiOiJuYXRpdmUiLCJuYmYiOjE2NjQ4MDY0NzMsInN1YiI6Im5hdGl2ZSIsInZpZGVvIjp7InJvb21DcmVhdGUiOnRydWUsInJvb21Kb2luIjp0cnVlfX0.BgVdBnq3XFD3_BQHoe1azqjifYysubgFl6Qlzu9IQGI"; @@ -9,14 +8,12 @@ const TOKEN : &str = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjIzODQ4MDY0N // eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjIzODQ4MDY3MzAsImlzcyI6IkFQSXpLYkFTaUNWYWtnSiIsIm5hbWUiOiJ3ZWIiLCJuYmYiOjE2NjQ4MDY3MzAsInN1YiI6IndlYiIsInZpZGVvIjp7InJvb21DcmVhdGUiOnRydWUsInJvb21Kb2luIjp0cnVlfX0.VbDoULjX1CVGZu2sPy3SvWYlVZUBXxQVPmdB9BnmlN4 #[tokio::main] -async fn main() -> Result<(), room::RoomError> { +async fn main() { tracing_subscriber::fmt::init(); - let mut room = room::connect(URL, TOKEN).await?; - room.local_participant() - .publish_data(b"some data", data_packet::Kind::Reliable) - .await?; + let room = Room::new(); + room.on_participant_connected(async |participant| { - sleep(Duration::from_secs(120)).await; - Ok(()) + + }) } diff --git a/src/lib.rs b/src/lib.rs index fe7ff25..ef89eee 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,2 +1,2 @@ // export everything inside livekit-core -pub use livekit_core::*; \ No newline at end of file +pub use livekit_core::*; From 64979309125f06a1d46bf41c053332b838e8f301 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sat, 29 Oct 2022 15:00:38 +0200 Subject: [PATCH 2/8] implement Debug on WebRTC structs --- crates/livekit-core/src/lib.rs | 3 -- crates/livekit-core/src/room/id.rs | 30 ++++++++++++++++---- crates/livekit-core/src/rtc_engine/mod.rs | 1 - crates/livekit-webrtc/src/data_channel.rs | 4 ++- crates/livekit-webrtc/src/media_stream.rs | 27 ++++++++++++++---- crates/livekit-webrtc/src/peer_connection.rs | 13 +++++++++ crates/livekit-webrtc/src/rtp_receiver.rs | 9 ++++++ 7 files changed, 71 insertions(+), 16 deletions(-) diff --git a/crates/livekit-core/src/lib.rs b/crates/livekit-core/src/lib.rs index bfc8c9a..15669d3 100644 --- a/crates/livekit-core/src/lib.rs +++ b/crates/livekit-core/src/lib.rs @@ -1,6 +1,3 @@ -extern crate core; -extern crate core; - pub mod proto { include!(concat!(env!("OUT_DIR"), "/livekit.rs")); } diff --git a/crates/livekit-core/src/room/id.rs b/crates/livekit-core/src/room/id.rs index f46198a..9bd35eb 100644 --- a/crates/livekit-core/src/room/id.rs +++ b/crates/livekit-core/src/room/id.rs @@ -1,21 +1,39 @@ +use std::fmt; + macro_rules! id_str { ($($name:ident;)*) => { $( + impl $name { + pub fn new(str: String) -> Self { + Self(str) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + } + impl From for $name { fn from(str: String) -> $name { $name(str) } } + impl From<$name> for String { + fn from(id: $name) -> String { + id.0 + } + } + impl PartialEq<$name> for String { fn eq(&self, u: &$name) -> bool { *self == *u.0 } } - impl From<$name> for String { - fn from(id: $name) -> String { - id.0 + impl fmt::Display for $name { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str(&self.0) } } )* @@ -23,13 +41,13 @@ macro_rules! id_str { } #[derive(Clone, Default, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] -pub struct ParticipantSid(pub String); +pub struct ParticipantSid(String); #[derive(Clone, Default, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] -pub struct ParticipantIdentity(pub String); +pub struct ParticipantIdentity(String); #[derive(Clone, Default, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] -pub struct TrackSid(pub String); +pub struct TrackSid(String); id_str! { ParticipantSid; diff --git a/crates/livekit-core/src/rtc_engine/mod.rs b/crates/livekit-core/src/rtc_engine/mod.rs index 86fd80f..1410b59 100644 --- a/crates/livekit-core/src/rtc_engine/mod.rs +++ b/crates/livekit-core/src/rtc_engine/mod.rs @@ -1,4 +1,3 @@ -use core::num::flt2dec::Sign; use parking_lot::Mutex; use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; use std::sync::{Arc, Weak}; diff --git a/crates/livekit-webrtc/src/data_channel.rs b/crates/livekit-webrtc/src/data_channel.rs index a45c6ca..a231ee3 100644 --- a/crates/livekit-webrtc/src/data_channel.rs +++ b/crates/livekit-webrtc/src/data_channel.rs @@ -18,7 +18,9 @@ pub struct DataChannel { impl Debug for DataChannel { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - write!(f, "DataChannel[{}]", self.label()) + f.debug_struct("DataChannel") + .field("label", &self.label()) + .finish() } } diff --git a/crates/livekit-webrtc/src/media_stream.rs b/crates/livekit-webrtc/src/media_stream.rs index 3fff376..38bfed9 100644 --- a/crates/livekit-webrtc/src/media_stream.rs +++ b/crates/livekit-webrtc/src/media_stream.rs @@ -1,13 +1,24 @@ use cxx::UniquePtr; +use std::fmt::{Debug, Formatter}; use libwebrtc_sys::media_stream as sys_ms; - pub use sys_ms::ffi::TrackState; pub struct MediaStreamTrack { cxx_handle: UniquePtr, } +impl Debug for MediaStreamTrack { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + f.debug_struct("MediaStreamTrack") + .field("id", &self.id()) + .field("kind", &self.kind()) + .field("enabled", &self.enabled()) + .field("state", &self.state()) + .finish() + } +} + impl MediaStreamTrack { pub(crate) fn new(cxx_handle: UniquePtr) -> Self { Self { cxx_handle } @@ -38,14 +49,20 @@ pub struct MediaStream { cxx_handle: UniquePtr, } +impl Debug for MediaStream { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + f.debug_struct("MediaStream") + .field("id", &self.id()) + .finish() + } +} + impl MediaStream { pub(crate) fn new(cxx_handle: UniquePtr) -> Self { - Self { - cxx_handle - } + Self { cxx_handle } } pub fn id(&self) -> String { self.cxx_handle.id() } -} \ No newline at end of file +} diff --git a/crates/livekit-webrtc/src/peer_connection.rs b/crates/livekit-webrtc/src/peer_connection.rs index 4d44676..b95005a 100644 --- a/crates/livekit-webrtc/src/peer_connection.rs +++ b/crates/livekit-webrtc/src/peer_connection.rs @@ -1,3 +1,4 @@ +use std::fmt::{Debug, Formatter}; use std::mem::ManuallyDrop; use std::sync::{Arc, Mutex}; @@ -31,6 +32,18 @@ pub struct PeerConnection { native_observer: UniquePtr, } +impl Debug for PeerConnection { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + f.debug_struct("PeerConnection") + .field("signaling_state", &self.signaling_state()) + .field("ice_connection_state", &self.ice_connection_state()) + .field("ice_gathering_state", &self.ice_gathering_state()) + .field("local_description", &self.local_description()) + .field("remote_description", &self.remote_description()) + .finish() + } +} + impl PeerConnection { pub(crate) fn new( cxx_handle: UniquePtr, diff --git a/crates/livekit-webrtc/src/rtp_receiver.rs b/crates/livekit-webrtc/src/rtp_receiver.rs index 0bcfee3..6a4ea5b 100644 --- a/crates/livekit-webrtc/src/rtp_receiver.rs +++ b/crates/livekit-webrtc/src/rtp_receiver.rs @@ -1,11 +1,20 @@ use crate::media_stream::MediaStreamTrack; use cxx::UniquePtr; use libwebrtc_sys::rtp_receiver as sys_rec; +use std::fmt::{Debug, Formatter}; pub struct RtpReceiver { cxx_handle: UniquePtr, } +impl Debug for RtpReceiver { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + f.debug_struct("RtpReceiver") + .field("track", &self.track()) + .finish() + } +} + impl RtpReceiver { pub(crate) fn new(cxx_handle: UniquePtr) -> Self { Self { cxx_handle } From 82ec808dee4e0c052079d91cbb944f581f2594b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Tue, 8 Nov 2022 10:31:09 +0100 Subject: [PATCH 3/8] changing computer ( started tracks ) I used my Mac on the company trip --- crates/livekit-core/protocol | 2 +- crates/livekit-core/src/room/mod.rs | 2 +- crates/livekit-core/src/room/participant.rs | 12 +-- .../src/room/remote_participant.rs | 65 +++++++++++++++-- .../src/room/{track.rs => track/mod.rs} | 50 ++++++++++++- .../src/room/track_publication.rs | 73 +++++++++++++++++-- crates/livekit-webrtc/libwebrtc-sys/build.rs | 2 +- 7 files changed, 178 insertions(+), 28 deletions(-) rename crates/livekit-core/src/room/{track.rs => track/mod.rs} (81%) diff --git a/crates/livekit-core/protocol b/crates/livekit-core/protocol index dc2a7bc..8449c11 160000 --- a/crates/livekit-core/protocol +++ b/crates/livekit-core/protocol @@ -1 +1 @@ -Subproject commit dc2a7bc3a05fe37f5c64cdb83ac36aeb00672778 +Subproject commit 8449c1106929b3f5cf9fdcc97ab3e63a1c5e6a76 diff --git a/crates/livekit-core/src/room/mod.rs b/crates/livekit-core/src/room/mod.rs index 853d992..84f9285 100644 --- a/crates/livekit-core/src/room/mod.rs +++ b/crates/livekit-core/src/room/mod.rs @@ -1,7 +1,6 @@ use futures_util::future::BoxFuture; use parking_lot::lock_api::RwLockUpgradableReadGuard; use parking_lot::{Mutex, RwLock}; -use std::borrow::Cow; use std::collections::HashMap; use std::future::Future; use std::sync::atomic::AtomicU8; @@ -36,6 +35,7 @@ pub enum RoomError { type RoomResult = Result; +#[derive(Debug)] pub enum ConnectionState { Disconnected, Connecting, diff --git a/crates/livekit-core/src/room/participant.rs b/crates/livekit-core/src/room/participant.rs index b36c60c..425d32e 100644 --- a/crates/livekit-core/src/room/participant.rs +++ b/crates/livekit-core/src/room/participant.rs @@ -10,7 +10,7 @@ pub(super) struct ParticipantShared { pub(super) identity: Mutex, pub(super) name: Mutex, pub(super) metadata: Mutex, - pub(super) tracks: RwLock>>, + pub(super) tracks: RwLock>, } impl ParticipantShared { @@ -50,7 +50,7 @@ pub enum Participant { Remote(RemoteParticipant), } -macro_rules! shared_method { +macro_rules! shared_getter { ($x:ident, $ret:ident) => { fn $x(&self) -> $ret { match self { @@ -62,10 +62,10 @@ macro_rules! shared_method { } impl ParticipantTrait for Participant { - shared_method!(sid, ParticipantSid); - shared_method!(identity, ParticipantIdentity); - shared_method!(name, String); - shared_method!(metadata, String); + shared_getter!(sid, ParticipantSid); + shared_getter!(identity, ParticipantIdentity); + shared_getter!(name, String); + shared_getter!(metadata, String); fn update_info(&self, info: ParticipantInfo) { match self { diff --git a/crates/livekit-core/src/room/remote_participant.rs b/crates/livekit-core/src/room/remote_participant.rs index 5751cdd..341c731 100644 --- a/crates/livekit-core/src/room/remote_participant.rs +++ b/crates/livekit-core/src/room/remote_participant.rs @@ -1,7 +1,18 @@ -use std::sync::Arc; +use crate::room::id::TrackSid; use crate::room::participant::{impl_participant_trait, ParticipantShared}; -use crate::room::track_publication::RemoteTrackPublication; +use crate::room::track::{RemoteAudioTrack, RemoteTrack, RemoteVideoTrack, TrackKind}; +use crate::room::track_publication::{ + RemoteTrackPublication, TrackPublication, TrackPublicationTrait, +}; +use livekit_webrtc::media_stream::MediaStreamTrack; +use std::time::Duration; +use tokio::time::{sleep, timeout}; +const ADD_TRACK_TIMEOUT: Duration = Duration::from_secs(5); + + +// It should be fine to add event listeners in this structure +// Registering after should be ParticipantConnected is fine to avoid missing events pub struct RemoteParticipant { shared: ParticipantShared, } @@ -18,16 +29,54 @@ impl RemoteParticipant { } } - pub(super) async fn add_subscribed_media_track() { + pub(super) async fn add_subscribed_media_track( + &self, + sid: &TrackSid, + media_track: MediaStreamTrack, + ) { + let wait_publication = async { + loop { + let publication = self.get_track_publication(sid); + if let Some(publication) = publication { + return publication; + } + + sleep(Duration::from_millis(50)).await; + } + }; + + let res = timeout(ADD_TRACK_TIMEOUT, wait_publication).await; + + if let Ok(remote_publication) = res { + let track = match remote_publication.kind() { + TrackKind::Audio => { + let audio_track = RemoteAudioTrack::new(); + RemoteTrack::Audio(audio_track) + } + TrackKind::Video => { + let video_track = RemoteVideoTrack::new(); + RemoteTrack::Video(video_track) + } + _ => unreachable!(), + }; + + // TODO(theomonnom): call OnTrackSubscribed here + + } else { + // TODO(theomonnom): send error + } } - fn get_track_publication(&self, sid: &str) -> Option { - let track = self.shared.tracks.read().get(&sid.to_string().into()).unwrap().clone(); - - - None + fn get_track_publication(&self, sid: &TrackSid) -> Option { + self.shared.tracks.read().get(sid).map(|track| { + if let TrackPublication::Remote(remote) = track { + remote.clone() + } else { + unreachable!() + } + }) } } diff --git a/crates/livekit-core/src/room/track.rs b/crates/livekit-core/src/room/track/mod.rs similarity index 81% rename from crates/livekit-core/src/room/track.rs rename to crates/livekit-core/src/room/track/mod.rs index 8b88b74..1627418 100644 --- a/crates/livekit-core/src/room/track.rs +++ b/crates/livekit-core/src/room/track/mod.rs @@ -1,28 +1,70 @@ +#[derive(Debug)] pub enum TrackKind { + Unknown, Audio, - Video + Video, } +impl From for TrackKind { + fn from(val: u8) -> Self { + match val { + 1 => Self::Audio, + 2 => Self::Video, + _ => Self::Unknown, + } + } +} + +#[derive(Debug)] pub enum StreamState { + Unknown, Active, Paused, - Unknown } +#[derive(Debug)] pub enum TrackSource { + Unknown, Camera, Microphone, Screenshare, ScreenshareAudio, - Unknown, +} + +impl From for TrackSource { + fn from(val: u8) -> Self { + match val { + 1 => Self::Camera, + 2 => Self::Microphone, + 3 => Self::Screenshare, + 4 => Self::ScreenshareAudio, + _ => Self::Unknown, + } + } } pub struct LocalVideoTrack {} pub struct RemoteVideoTrack {} pub struct LocalAudioTrack {} -pub struct RemoteAudioTrack {} +pub struct RemoteAudioTrack { + + +} + +impl RemoteVideoTrack { + pub(crate) fn new() -> Self { + Self {} + } +} + +impl RemoteAudioTrack { + pub(crate) fn new() -> Self { + Self {} + } +} + pub enum RemoteTrack { Audio(RemoteAudioTrack), Video(RemoteVideoTrack), diff --git a/crates/livekit-core/src/room/track_publication.rs b/crates/livekit-core/src/room/track_publication.rs index e2c6361..2c351db 100644 --- a/crates/livekit-core/src/room/track_publication.rs +++ b/crates/livekit-core/src/room/track_publication.rs @@ -1,7 +1,16 @@ use std::sync::Arc; -use std::sync::atomic::{AtomicBool, AtomicU8}; +use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; use parking_lot::Mutex; -use crate::room::id::TrackSid; +use crate::room::id::{ParticipantIdentity, ParticipantSid, TrackSid}; +use crate::room::track::{TrackKind, TrackSource}; + +pub trait TrackPublicationTrait { + fn name(&self) -> String; + fn sid(&self) -> TrackSid; + fn kind(&self) -> TrackKind; + fn source(&self) -> TrackSource; + fn simulcasted(&self) -> bool; +} pub(super) struct TrackPublicationShared { pub(super) name: Mutex, @@ -11,6 +20,59 @@ pub(super) struct TrackPublicationShared { pub(super) simulcasted: AtomicBool } +#[derive(Clone)] +pub enum TrackPublication { + Local(LocalTrackPublication), + Remote(RemoteTrackPublication) +} + +macro_rules! shared_getter { + ($x:ident, $ret:ident) => { + fn $x(&self) -> $ret { + match self { + TrackPublication::Local(p) => p.$x(), + TrackPublication::Remote(p) => p.$x(), + } + } + }; +} + +impl TrackPublicationTrait for TrackPublication { + shared_getter!(name, String); + shared_getter!(sid, TrackSid); + shared_getter!(kind, TrackKind); + shared_getter!(source, TrackSource); + shared_getter!(simulcasted, bool); +} + +macro_rules! impl_publication_trait { + ($x:ident) => { + impl TrackPublicationTrait for $x { + fn name(&self) -> String { + self.shared.name.lock().clone() + } + + fn sid(&self) -> TrackSid { + self.shared.sid.lock().clone() + } + + fn kind(&self) -> TrackKind { + self.shared.kind.load(Ordering::SeqCst).into() + } + + fn source(&self) -> TrackSource { + self.shared.source.load(Ordering::SeqCst).into() + } + + fn simulcasted(&self) -> bool { + self.shared.simulcasted.load(Ordering::SeqCst) + } + } + } +} + + + #[derive(Clone)] pub struct LocalTrackPublication { shared: Arc @@ -21,8 +83,5 @@ pub struct RemoteTrackPublication { shared: Arc } -#[derive(Clone)] -pub enum TrackPublication { - Local(LocalTrackPublication), - Remote(RemoteTrackPublication) -} \ No newline at end of file +impl_publication_trait!(LocalTrackPublication); +impl_publication_trait!(RemoteTrackPublication); diff --git a/crates/livekit-webrtc/libwebrtc-sys/build.rs b/crates/livekit-webrtc/libwebrtc-sys/build.rs index c6d95cc..a65a667 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/build.rs +++ b/crates/livekit-webrtc/libwebrtc-sys/build.rs @@ -52,7 +52,7 @@ fn macos_link_search_path() -> Option { fn main() { // TODO Download precompiled binaries of WebRTC for the target_os - let target_os = "windows"; + let target_os = "macos"; //let target_arch = "arm64"; let libwebrtc_dir = path::PathBuf::from("libwebrtc"); From ae776f04d0352fd45b7e2d7d6d834c9e8a83d472 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Tue, 22 Nov 2022 17:59:55 +0100 Subject: [PATCH 4/8] Initial downstream tracks --- crates/livekit-core/Cargo.toml | 2 +- crates/livekit-core/src/events.rs | 140 +++++++++ crates/livekit-core/src/lib.rs | 4 + crates/livekit-core/src/room/mod.rs | 104 +++---- .../{ => participant}/local_participant.rs | 3 +- .../{participant.rs => participant/mod.rs} | 88 ++++-- .../room/participant/remote_participant.rs | 149 ++++++++++ .../mod.rs} | 56 +++- .../src/room/remote_participant.rs | 83 ------ .../src/room/track/audio_track.rs | 31 ++ crates/livekit-core/src/room/track/events.rs | 1 + .../src/room/track/local_audio_track.rs | 7 + .../src/room/track/local_track.rs | 31 ++ .../src/room/track/local_video_track.rs | 7 + crates/livekit-core/src/room/track/mod.rs | 242 +++++++-------- .../src/room/track/remote_audio_track.rs | 30 ++ .../src/room/track/remote_track.rs | 49 ++++ .../src/room/track/remote_video_track.rs | 31 ++ .../src/room/track/video_track.rs | 31 ++ crates/livekit-core/src/utils.rs | 26 ++ .../libwebrtc-sys/CMakeLists.txt | 23 -- crates/livekit-webrtc/libwebrtc-sys/build.rs | 4 +- .../libwebrtc-sys/compile_flags.txt | 8 + .../include/livekit/media_stream.h | 111 +++++-- .../include/livekit/rust_types.h | 4 + .../include/livekit/video_frame.h | 48 +++ .../include/livekit/video_frame_buffer.h | 93 ++++++ .../libwebrtc-sys/src/candidate.cpp | 2 +- .../libwebrtc-sys/src/data_channel.cpp | 2 +- .../livekit-webrtc/libwebrtc-sys/src/jsep.cpp | 2 +- .../livekit-webrtc/libwebrtc-sys/src/lib.rs | 6 + .../libwebrtc-sys/src/media_stream.cpp | 72 ++++- .../libwebrtc-sys/src/media_stream.rs | 101 ++++++- .../libwebrtc-sys/src/peer_connection.rs | 2 +- .../src/peer_connection_factory.rs | 2 +- .../libwebrtc-sys/src/rtp_receiver.cpp | 2 +- .../libwebrtc-sys/src/rtp_receiver.rs | 2 +- .../libwebrtc-sys/src/video_frame.rs | 32 ++ .../libwebrtc-sys/src/video_frame_buffer.rs | 43 +++ .../src/.media_stream.rs.rustfmt | 277 ++++++++++++++++++ crates/livekit-webrtc/src/data_channel.rs | 18 +- crates/livekit-webrtc/src/jsep.rs | 2 +- crates/livekit-webrtc/src/lib.rs | 2 + crates/livekit-webrtc/src/media_stream.rs | 243 +++++++++++++-- crates/livekit-webrtc/src/peer_connection.rs | 2 +- crates/livekit-webrtc/src/rtp_receiver.rs | 10 +- crates/livekit-webrtc/src/video_frame.rs | 50 ++++ .../livekit-webrtc/src/video_frame_buffer.rs | 18 ++ examples/simple_room/src/main.rs | 20 +- 49 files changed, 1910 insertions(+), 406 deletions(-) create mode 100644 crates/livekit-core/src/events.rs rename crates/livekit-core/src/room/{ => participant}/local_participant.rs (93%) rename crates/livekit-core/src/room/{participant.rs => participant/mod.rs} (52%) create mode 100644 crates/livekit-core/src/room/participant/remote_participant.rs rename crates/livekit-core/src/room/{track_publication.rs => publication/mod.rs} (61%) delete mode 100644 crates/livekit-core/src/room/remote_participant.rs create mode 100644 crates/livekit-core/src/room/track/audio_track.rs create mode 100644 crates/livekit-core/src/room/track/events.rs create mode 100644 crates/livekit-core/src/room/track/local_audio_track.rs create mode 100644 crates/livekit-core/src/room/track/local_track.rs create mode 100644 crates/livekit-core/src/room/track/local_video_track.rs create mode 100644 crates/livekit-core/src/room/track/remote_audio_track.rs create mode 100644 crates/livekit-core/src/room/track/remote_track.rs create mode 100644 crates/livekit-core/src/room/track/remote_video_track.rs create mode 100644 crates/livekit-core/src/room/track/video_track.rs create mode 100644 crates/livekit-core/src/utils.rs delete mode 100644 crates/livekit-webrtc/libwebrtc-sys/CMakeLists.txt create mode 100644 crates/livekit-webrtc/libwebrtc-sys/compile_flags.txt create mode 100644 crates/livekit-webrtc/libwebrtc-sys/include/livekit/video_frame.h create mode 100644 crates/livekit-webrtc/libwebrtc-sys/include/livekit/video_frame_buffer.h create mode 100644 crates/livekit-webrtc/libwebrtc-sys/src/video_frame.rs create mode 100644 crates/livekit-webrtc/libwebrtc-sys/src/video_frame_buffer.rs create mode 100644 crates/livekit-webrtc/src/.media_stream.rs.rustfmt create mode 100644 crates/livekit-webrtc/src/video_frame.rs create mode 100644 crates/livekit-webrtc/src/video_frame_buffer.rs diff --git a/crates/livekit-core/Cargo.toml b/crates/livekit-core/Cargo.toml index ed20382..a9ce7cd 100644 --- a/crates/livekit-core/Cargo.toml +++ b/crates/livekit-core/Cargo.toml @@ -10,7 +10,7 @@ serde_json = "1.0" tokio-tungstenite = { version = "0.17.2", features = ["native-tls"] } tokio = { version = "1", features = ["full"] } futures = "0.3" -parking_lot = "0.12.1" +parking_lot = { version = "0.12.1", features = ["send_guard"] } url = "2.2.2" futures-util = "0.3.23" thiserror = "1.0" diff --git a/crates/livekit-core/src/events.rs b/crates/livekit-core/src/events.rs new file mode 100644 index 0000000..5dfecd7 --- /dev/null +++ b/crates/livekit-core/src/events.rs @@ -0,0 +1,140 @@ +use futures_util::future::BoxFuture; +use thiserror::Error; + +type EventHandler = Box BoxFuture<'static, ()> + Send + Sync>; + +macro_rules! event_setter { + ($fnc:ident, $event:ty) => { + pub fn $fnc(&self, mut callback: F) + where + F: FnMut($event) -> Fut + Send + Sync + 'static, + Fut: Future + Send + Sync + 'static, + { + *self.$fnc.lock() = Some(Box::new(move |event| Box::pin(callback(event)))); + } + }; +} + +#[derive(Error, Debug, Clone)] +pub enum TrackError { + #[error("could not find published track with sid: {0}")] + TrackNotFound(String), +} + +pub mod room { + use super::{EventHandler, TrackError}; + use crate::room::id::TrackSid; + use crate::room::participant::remote_participant::RemoteParticipant; + use crate::room::publication::RemoteTrackPublication; + use crate::room::track::remote_track::RemoteTrackHandle; + use crate::room::RoomHandle; + use futures::future::Future; + use parking_lot::Mutex; + use std::sync::Arc; + + #[derive(Clone)] + pub struct ParticipantConnectedEvent { + pub room_handle: RoomHandle, + pub participant: Arc, + } + + #[derive(Clone)] + pub struct ParticipantDisconnectedEvent { + pub room_handle: RoomHandle, + pub participant: Arc, + } + + #[derive(Clone)] + pub struct TrackSubscribedEvent { + pub room_handle: RoomHandle, + pub track: RemoteTrackHandle, + pub publication: RemoteTrackPublication, + pub participant: Arc, + } + + #[derive(Clone)] + pub struct TrackPublishedEvent { + pub room_handle: RoomHandle, + pub publication: RemoteTrackPublication, + pub participant: Arc, + } + + #[derive(Clone)] + pub struct TrackSubscriptionFailedEvent { + pub room_handle: RoomHandle, + pub error: TrackError, + pub sid: TrackSid, + pub participant: Arc, + } + + pub(crate) type OnParticipantConnectedHandler = EventHandler; + pub(crate) type OnParticipantDisconnectedHandler = EventHandler; + pub(crate) type OnTrackSubscribedEventHandler = EventHandler; + pub(crate) type OnTrackPublishedEventHandler = EventHandler; + pub(crate) type OnTrackSubscriptionFailedHandler = EventHandler; + + #[derive(Default)] + pub struct RoomEvents { + pub(crate) on_participant_connected: Mutex>, + pub(crate) on_participant_disconnected: Mutex>, + pub(crate) on_track_subscribed: Mutex>, + pub(crate) on_track_published: Mutex>, + pub(crate) on_track_subscription_failed: Mutex>, + } + + impl RoomEvents { + event_setter!(on_participant_connected, ParticipantConnectedEvent); + event_setter!(on_participant_disconnected, ParticipantDisconnectedEvent); + event_setter!(on_track_subscribed, TrackSubscribedEvent); + event_setter!(on_track_published, TrackPublishedEvent); + event_setter!(on_track_subscription_failed, TrackSubscriptionFailedEvent); + } +} + +pub mod participant { + use super::{EventHandler, TrackError}; + use crate::room::id::TrackSid; + use crate::room::participant::remote_participant::RemoteParticipant; + use crate::room::publication::RemoteTrackPublication; + use crate::room::track::remote_track::RemoteTrackHandle; + use futures::future::Future; + use parking_lot::Mutex; + use std::sync::Arc; + + #[derive(Clone)] + pub struct TrackPublishedEvent { + pub publication: RemoteTrackPublication, + pub participant: Arc, + } + + #[derive(Clone)] + pub struct TrackSubscribedEvent { + pub track: RemoteTrackHandle, + pub publication: RemoteTrackPublication, + pub participant: Arc, + } + + #[derive(Clone)] + pub struct TrackSubscriptionFailedEvent { + pub sid: TrackSid, + pub error: TrackError, + pub participant: Arc, + } + + pub(crate) type TrackPublishedHandler = EventHandler; + pub(crate) type TrackSubscribedHandler = EventHandler; + pub(crate) type TrackSubscriptionFailedHandler = EventHandler; + + #[derive(Default)] + pub struct ParticipantEvents { + pub(crate) on_track_published: Mutex>, + pub(crate) on_track_subscribed: Mutex>, + pub(crate) on_track_subscription_failed: Mutex>, + } + + impl ParticipantEvents { + event_setter!(on_track_published, TrackPublishedEvent); + event_setter!(on_track_subscribed, TrackSubscribedEvent); + event_setter!(on_track_subscription_failed, TrackSubscriptionFailedEvent); + } +} diff --git a/crates/livekit-core/src/lib.rs b/crates/livekit-core/src/lib.rs index 15669d3..a9770a7 100644 --- a/crates/livekit-core/src/lib.rs +++ b/crates/livekit-core/src/lib.rs @@ -1,8 +1,12 @@ +extern crate core; + pub mod proto { include!(concat!(env!("OUT_DIR"), "/livekit.rs")); } +mod events; mod rtc_engine; mod signal_client; +mod utils; pub mod room; diff --git a/crates/livekit-core/src/room/mod.rs b/crates/livekit-core/src/room/mod.rs index 84f9285..c293d29 100644 --- a/crates/livekit-core/src/room/mod.rs +++ b/crates/livekit-core/src/room/mod.rs @@ -1,29 +1,27 @@ -use futures_util::future::BoxFuture; use parking_lot::lock_api::RwLockUpgradableReadGuard; use parking_lot::{Mutex, RwLock}; use std::collections::HashMap; -use std::future::Future; use std::sync::atomic::AtomicU8; use std::sync::Arc; +use self::id::ParticipantSid; +use self::participant::local_participant::LocalParticipant; +use self::participant::remote_participant::RemoteParticipant; +use self::participant::ParticipantInternalTrait; +use self::participant::ParticipantTrait; +use crate::events::room::{ParticipantConnectedEvent, ParticipantDisconnectedEvent, RoomEvents}; use crate::proto; -use crate::proto::{participant_info, ParticipantInfo}; -use crate::room::id::{ParticipantIdentity, ParticipantSid}; -use crate::room::local_participant::LocalParticipant; -use crate::room::participant::ParticipantTrait; -use crate::room::remote_participant::RemoteParticipant; +use crate::proto::participant_info; use thiserror::Error; use tracing::error; use crate::rtc_engine::{EngineError, EngineEvent, EngineEvents, RTCEngine}; use crate::signal_client::SignalOptions; -mod id; -mod local_participant; -mod participant; -mod remote_participant; -mod track; -mod track_publication; +pub mod id; +pub mod participant; +pub mod publication; +pub mod track; #[derive(Error, Debug)] pub enum RoomError { @@ -52,15 +50,6 @@ struct RoomInner { local_participant: Arc, } -type OnParticipantConnectedHandler = - Box) -> BoxFuture<'static, ()> + Send + Sync>; -type OnParticipantDisconnectedHandler = OnParticipantConnectedHandler; - -struct RoomEvents { - on_participant_connected_handler: Mutex>, - on_participant_disconnected_handler: Mutex>, -} - pub struct Room { inner: Option>, events: Arc, @@ -70,13 +59,14 @@ impl Room { pub fn new() -> Room { Self { inner: None, - events: Arc::new(RoomEvents { - on_participant_connected_handler: Default::default(), - on_participant_disconnected_handler: Default::default(), - }), + events: Default::default(), } } + pub fn events(&self) -> Arc { + self.events.clone() + } + pub async fn connect(&mut self, url: &str, token: &str) -> RoomResult<()> { let (rtc_engine, engine_events) = RTCEngine::connect(url, token, SignalOptions::default()).await?; @@ -109,28 +99,6 @@ impl Room { }) } - pub fn on_participant_connected(&self, mut callback: F) - where - F: FnMut(RoomHandle, Arc) -> Fut + Send + Sync + 'static, - Fut: Future + Send + Sync + 'static, - { - *self.events.on_participant_connected_handler.lock() = - Some(Box::new(move |handle, participant| { - Box::pin(callback(handle, participant)) - })); - } - - pub fn on_participant_disconnected(&self, mut callback: F) - where - F: FnMut(RoomHandle, Arc) -> Fut + Send + Sync + 'static, - Fut: Future + Send + Sync + 'static, - { - *self.events.on_participant_disconnected_handler.lock() = - Some(Box::new(move |handle, participant| { - Box::pin(callback(handle, participant)) - })); - } - async fn room_task( room_inner: Arc, room_events: Arc, @@ -178,7 +146,10 @@ impl Room { Self::get_participant(room_inner.clone(), &participant_sid.to_string().into()); if let Some(remote_participant) = remote_participant { - + remote_participant.add_subscribed_media_track( + track_sid.to_string().into(), + rtp_receiver.track(), + ); } else { // The server should send participant updates before sending a new offer // So this should not happen. @@ -223,13 +194,14 @@ impl Room { } } else { // Create a new participant and call OnConnect event - let remote_participant = Self::get_or_create_participant(room_inner.clone(), pi); - let mut handler = room_events.on_participant_connected_handler.lock(); - if let Some(callback) = handler.as_mut() { - callback( - RoomHandle::from(room_inner.clone()), - remote_participant.clone(), - ); + let remote_participant = + Self::get_or_create_participant(room_inner.clone(), room_events.clone(), pi); + let mut handler = room_events.on_participant_connected.lock(); + if let Some(cb) = handler.as_mut() { + cb(ParticipantConnectedEvent { + room_handle: RoomHandle::from(room_inner.clone()), + participant: remote_participant.clone(), + }); } } } @@ -247,12 +219,12 @@ impl Room { // TODO(theomonnom): Unpublish all tracks - let mut handler = room_events.on_participant_disconnected_handler.lock(); - if let Some(callback) = handler.as_mut() { - callback( - RoomHandle::from(room_inner.clone()), - remote_participant.clone(), - ); + let mut handler = room_events.on_participant_disconnected.lock(); + if let Some(cb) = handler.as_mut() { + cb(ParticipantDisconnectedEvent { + room_handle: RoomHandle::from(room_inner.clone()), + participant: remote_participant.clone(), + }); } } @@ -265,6 +237,7 @@ impl Room { fn get_or_create_participant( room_inner: Arc, + room_events: Arc, pi: proto::ParticipantInfo, ) -> Arc { let participants = room_inner.participants.upgradable_read(); @@ -275,6 +248,13 @@ impl Room { } else { let mut participants = RwLockUpgradableReadGuard::upgrade(participants); let p = Arc::new(RemoteParticipant::new(pi)); + + // Forward participantevents to room events + p.internal_events().on_track_published({ + let room_events = room_events.clone(); + |event| async move {} + }); + participants.insert(sid, p.clone()); p } diff --git a/crates/livekit-core/src/room/local_participant.rs b/crates/livekit-core/src/room/participant/local_participant.rs similarity index 93% rename from crates/livekit-core/src/room/local_participant.rs rename to crates/livekit-core/src/room/participant/local_participant.rs index 5f7a96e..ebc877f 100644 --- a/crates/livekit-core/src/room/local_participant.rs +++ b/crates/livekit-core/src/room/participant/local_participant.rs @@ -2,7 +2,6 @@ use crate::proto::{data_packet, DataPacket, UserPacket}; use crate::room::participant::{impl_participant_trait, ParticipantShared}; use crate::room::RoomError; use crate::rtc_engine::RTCEngine; -use std::sync::Arc; pub struct LocalParticipant { shared: ParticipantShared, @@ -10,7 +9,7 @@ pub struct LocalParticipant { } impl LocalParticipant { - pub(super) fn new(rtc_engine: Arc, info: ParticipantInfo) -> Self { + pub(crate) fn new(rtc_engine: Arc, info: ParticipantInfo) -> Self { Self { shared: ParticipantShared::new( info.sid.into(), diff --git a/crates/livekit-core/src/room/participant.rs b/crates/livekit-core/src/room/participant/mod.rs similarity index 52% rename from crates/livekit-core/src/room/participant.rs rename to crates/livekit-core/src/room/participant/mod.rs index 425d32e..9158452 100644 --- a/crates/livekit-core/src/room/participant.rs +++ b/crates/livekit-core/src/room/participant/mod.rs @@ -1,11 +1,23 @@ +use crate::events::participant::ParticipantEvents; use crate::proto::ParticipantInfo; -use crate::room::local_participant::LocalParticipant; -use crate::room::remote_participant::RemoteParticipant; +use crate::room::id::{ParticipantIdentity, ParticipantSid, TrackSid}; +use crate::room::participant::local_participant::LocalParticipant; +use crate::room::participant::remote_participant::RemoteParticipant; +use crate::room::publication::{TrackPublication, TrackPublicationTrait}; +use crate::utils::wrap_variants; +use futures_util::future::BoxFuture; use parking_lot::{Mutex, RwLock}; use std::collections::HashMap; use std::sync::Arc; +pub mod local_participant; +pub mod remote_participant; + +type OnTrackSubscribed = Box BoxFuture<'static, ()> + Send + Sync>; + pub(super) struct ParticipantShared { + pub(super) events: Arc, + pub(super) internal_events: Arc, pub(super) sid: Mutex, pub(super) identity: Mutex, pub(super) name: Mutex, @@ -21,6 +33,8 @@ impl ParticipantShared { metadata: String, ) -> Self { Self { + events: Default::default(), + internal_events: Default::default(), sid: Mutex::new(sid), identity: Mutex::new(identity), name: Mutex::new(name), @@ -33,11 +47,20 @@ impl ParticipantShared { *self.sid.lock() = info.sid.into(); *self.identity.lock() = info.identity.into(); *self.name.lock() = info.name; - *self.metadata.lock() = info.metadata; // TODO(theomonnom): callback + *self.metadata.lock() = info.metadata; // TODO(theomonnom): callback MetadataChanged + } + + pub(crate) fn add_track_publication(&self, publication: TrackPublication) { + self.tracks.write().insert(publication.sid(), publication); } } +pub(crate) trait ParticipantInternalTrait { + fn internal_events(&self) -> Arc; +} + pub trait ParticipantTrait { + fn events(&self) -> Arc; fn sid(&self) -> ParticipantSid; fn identity(&self) -> ParticipantIdentity; fn name(&self) -> String; @@ -45,42 +68,49 @@ pub trait ParticipantTrait { fn update_info(&self, info: ParticipantInfo); } -pub enum Participant { - Local(LocalParticipant), - Remote(RemoteParticipant), +#[derive(Clone)] +pub enum ParticipantHandle { + Local(Arc), + Remote(Arc), } -macro_rules! shared_getter { - ($x:ident, $ret:ident) => { - fn $x(&self) -> $ret { - match self { - Participant::Local(p) => p.$x(), - Participant::Remote(p) => p.$x(), - } - } - }; +impl ParticipantInternalTrait for ParticipantHandle { + wrap_variants!( + [Local, Remote] + fnc!(internal_events, Arc, []); + ); } -impl ParticipantTrait for Participant { - shared_getter!(sid, ParticipantSid); - shared_getter!(identity, ParticipantIdentity); - shared_getter!(name, String); - shared_getter!(metadata, String); - - fn update_info(&self, info: ParticipantInfo) { - match self { - Participant::Local(p) => p.update_info(info), - Participant::Remote(p) => p.update_info(info), - } - } +impl ParticipantTrait for ParticipantHandle { + wrap_variants!( + [Local, Remote] + fnc!(events, Arc, []); + fnc!(sid, ParticipantSid, []); + fnc!(identity, ParticipantIdentity, []); + fnc!(name, String, []); + fnc!(metadata, String, []); + fnc!(update_info, (), [info: ParticipantInfo]); + ); } macro_rules! impl_participant_trait { - ($x:ident) => { + ($x:ty) => { + use crate::events::participant::ParticipantEvents; use crate::proto::ParticipantInfo; use crate::room::id::{ParticipantIdentity, ParticipantSid}; + use std::sync::Arc; + + impl crate::room::participant::ParticipantInternalTrait for $x { + fn internal_events(&self) -> Arc { + self.shared.internal_events.clone() + } + } impl crate::room::participant::ParticipantTrait for $x { + fn events(&self) -> Arc { + self.shared.events.clone() + } + fn sid(&self) -> ParticipantSid { self.shared.sid.lock().clone() } @@ -104,6 +134,4 @@ macro_rules! impl_participant_trait { }; } -use crate::room::id::{ParticipantIdentity, ParticipantSid, TrackSid}; -use crate::room::track_publication::TrackPublication; pub(super) use impl_participant_trait; diff --git a/crates/livekit-core/src/room/participant/remote_participant.rs b/crates/livekit-core/src/room/participant/remote_participant.rs new file mode 100644 index 0000000..1266504 --- /dev/null +++ b/crates/livekit-core/src/room/participant/remote_participant.rs @@ -0,0 +1,149 @@ +use crate::events::participant::{TrackSubscribedEvent, TrackSubscriptionFailedEvent}; +use crate::events::TrackError; +use crate::room::id::TrackSid; +use crate::room::participant::{impl_participant_trait, ParticipantShared}; +use crate::room::publication::{RemoteTrackPublication, TrackPublication, TrackPublicationTrait}; +use crate::room::track::remote_audio_track::RemoteAudioTrack; +use crate::room::track::remote_track::RemoteTrackHandle; +use crate::room::track::remote_video_track::RemoteVideoTrack; +use crate::room::track::{TrackTrait, TrackKind}; +use livekit_webrtc::media_stream::MediaStreamTrackHandle; +use std::time::Duration; +use tokio::time::{sleep, timeout}; +use tracing::error; + +const ADD_TRACK_TIMEOUT: Duration = Duration::from_secs(5); + +pub struct RemoteParticipant { + shared: ParticipantShared, +} + +impl RemoteParticipant { + pub(crate) fn new(info: ParticipantInfo) -> Self { + Self { + shared: ParticipantShared::new( + info.sid.into(), + info.identity.into(), + info.name, + info.metadata, + ), + } + } + + pub(crate) fn add_subscribed_media_track( + self: Arc, + sid: TrackSid, + media_track: MediaStreamTrackHandle, + ) { + tokio::spawn(async move { + let wait_publication = { + let participant = self.clone(); + let sid = sid.clone(); + async move { + loop { + let publication = participant.get_track_publication(&sid); + if let Some(publication) = publication { + return publication; + } + + sleep(Duration::from_millis(50)).await; + } + } + }; + + if let Ok(remote_publication) = timeout(ADD_TRACK_TIMEOUT, wait_publication).await { + let track = match remote_publication.kind() { + TrackKind::Audio => { + if let MediaStreamTrackHandle::Audio(rtc_track) = media_track { + let audio_track = RemoteAudioTrack::new( + remote_publication.sid().into(), + remote_publication.name(), + rtc_track, + ); + RemoteTrackHandle::Audio(Arc::new(audio_track)) + } else { + unreachable!(); + } + } + TrackKind::Video => { + if let MediaStreamTrackHandle::Video(rtc_track) = media_track { + let video_track = RemoteVideoTrack::new( + remote_publication.sid().into(), + remote_publication.name(), + rtc_track, + ); + RemoteTrackHandle::Video(Arc::new(video_track)) + } else { + unreachable!() + } + } + _ => unreachable!(), + }; + + self.shared + .add_track_publication(TrackPublication::Remote(remote_publication.clone())); + track.start(); + + let event = TrackSubscribedEvent { + track, + publication: remote_publication, + participant: self.clone(), + }; + + if let Some(cb) = self + .shared + .internal_events + .on_track_subscribed + .lock() + .as_mut() + { + cb(event.clone()).await; + } + + if let Some(cb) = self.shared.events.on_track_subscribed.lock().as_mut() { + cb(event).await; + } + } else { + error!("could not find published track with sid: {:?}", sid); + + let event = TrackSubscriptionFailedEvent { + sid: sid.clone(), + error: TrackError::TrackNotFound(sid.clone().to_string()), + participant: self.clone(), + }; + + if let Some(cb) = self + .shared + .internal_events + .on_track_subscription_failed + .lock() + .as_mut() + { + cb(event.clone()).await; + } + + if let Some(cb) = self + .shared + .events + .on_track_subscription_failed + .lock() + .as_mut() + { + cb(event).await; + } + } + }); + } + + fn get_track_publication(&self, sid: &TrackSid) -> Option { + self.shared.tracks.read().get(sid).map(|track| { + if let TrackPublication::Remote(remote) = track { + remote.clone() + } else { + unreachable!() + } + }) + } +} + +impl_participant_trait!(RemoteParticipant); diff --git a/crates/livekit-core/src/room/track_publication.rs b/crates/livekit-core/src/room/publication/mod.rs similarity index 61% rename from crates/livekit-core/src/room/track_publication.rs rename to crates/livekit-core/src/room/publication/mod.rs index 2c351db..5a36bbb 100644 --- a/crates/livekit-core/src/room/track_publication.rs +++ b/crates/livekit-core/src/room/publication/mod.rs @@ -1,8 +1,10 @@ -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; +use crate::room::id::TrackSid; +use crate::room::track::local_track::LocalTrackHandle; +use crate::room::track::remote_track::RemoteTrackHandle; +use crate::room::track::{TrackHandle, TrackKind, TrackSource}; use parking_lot::Mutex; -use crate::room::id::{ParticipantIdentity, ParticipantSid, TrackSid}; -use crate::room::track::{TrackKind, TrackSource}; +use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; +use std::sync::Arc; pub trait TrackPublicationTrait { fn name(&self) -> String; @@ -13,21 +15,22 @@ pub trait TrackPublicationTrait { } pub(super) struct TrackPublicationShared { + pub(super) track: Mutex>, pub(super) name: Mutex, pub(super) sid: Mutex, - pub(super) kind: AtomicU8, // Casted to TrackKind + pub(super) kind: AtomicU8, // Casted to TrackKind pub(super) source: AtomicU8, // Casted to TrackSource - pub(super) simulcasted: AtomicBool + pub(super) simulcasted: AtomicBool, } #[derive(Clone)] pub enum TrackPublication { Local(LocalTrackPublication), - Remote(RemoteTrackPublication) + Remote(RemoteTrackPublication), } macro_rules! shared_getter { - ($x:ident, $ret:ident) => { + ($x:ident, $ret:ty) => { fn $x(&self) -> $ret { match self { TrackPublication::Local(p) => p.$x(), @@ -37,6 +40,15 @@ macro_rules! shared_getter { }; } +impl TrackPublication { + pub fn track(&self) -> Option { + match self { + TrackPublication::Local(p) => p.shared.track.lock().clone(), + TrackPublication::Remote(p) => p.shared.track.lock().clone(), + } + } +} + impl TrackPublicationTrait for TrackPublication { shared_getter!(name, String); shared_getter!(sid, TrackSid); @@ -68,19 +80,37 @@ macro_rules! impl_publication_trait { self.shared.simulcasted.load(Ordering::SeqCst) } } - } + }; } - - #[derive(Clone)] pub struct LocalTrackPublication { - shared: Arc + shared: Arc, +} + +impl LocalTrackPublication { + pub fn track(&self) -> Option { + self.shared + .track + .lock() + .clone() + .map(|local_track| local_track.try_into().unwrap()) + } } #[derive(Clone)] pub struct RemoteTrackPublication { - shared: Arc + shared: Arc, +} + +impl RemoteTrackPublication { + pub fn track(&self) -> Option { + self.shared + .track + .lock() + .clone() + .map(|track| track.try_into().unwrap()) + } } impl_publication_trait!(LocalTrackPublication); diff --git a/crates/livekit-core/src/room/remote_participant.rs b/crates/livekit-core/src/room/remote_participant.rs deleted file mode 100644 index 341c731..0000000 --- a/crates/livekit-core/src/room/remote_participant.rs +++ /dev/null @@ -1,83 +0,0 @@ -use crate::room::id::TrackSid; -use crate::room::participant::{impl_participant_trait, ParticipantShared}; -use crate::room::track::{RemoteAudioTrack, RemoteTrack, RemoteVideoTrack, TrackKind}; -use crate::room::track_publication::{ - RemoteTrackPublication, TrackPublication, TrackPublicationTrait, -}; -use livekit_webrtc::media_stream::MediaStreamTrack; -use std::time::Duration; -use tokio::time::{sleep, timeout}; - -const ADD_TRACK_TIMEOUT: Duration = Duration::from_secs(5); - - -// It should be fine to add event listeners in this structure -// Registering after should be ParticipantConnected is fine to avoid missing events -pub struct RemoteParticipant { - shared: ParticipantShared, -} - -impl RemoteParticipant { - pub(super) fn new(info: ParticipantInfo) -> Self { - Self { - shared: ParticipantShared::new( - info.sid.into(), - info.identity.into(), - info.name, - info.metadata, - ), - } - } - - pub(super) async fn add_subscribed_media_track( - &self, - sid: &TrackSid, - media_track: MediaStreamTrack, - ) { - let wait_publication = async { - loop { - let publication = self.get_track_publication(sid); - if let Some(publication) = publication { - return publication; - } - - sleep(Duration::from_millis(50)).await; - } - }; - - let res = timeout(ADD_TRACK_TIMEOUT, wait_publication).await; - - if let Ok(remote_publication) = res { - let track = match remote_publication.kind() { - TrackKind::Audio => { - let audio_track = RemoteAudioTrack::new(); - RemoteTrack::Audio(audio_track) - } - TrackKind::Video => { - let video_track = RemoteVideoTrack::new(); - RemoteTrack::Video(video_track) - } - _ => unreachable!(), - }; - - - - // TODO(theomonnom): call OnTrackSubscribed here - - } else { - // TODO(theomonnom): send error - } - } - - fn get_track_publication(&self, sid: &TrackSid) -> Option { - self.shared.tracks.read().get(sid).map(|track| { - if let TrackPublication::Remote(remote) = track { - remote.clone() - } else { - unreachable!() - } - }) - } -} - -impl_participant_trait!(RemoteParticipant); diff --git a/crates/livekit-core/src/room/track/audio_track.rs b/crates/livekit-core/src/room/track/audio_track.rs new file mode 100644 index 0000000..cea9bfb --- /dev/null +++ b/crates/livekit-core/src/room/track/audio_track.rs @@ -0,0 +1,31 @@ +use crate::room::track::local_audio_track::LocalAudioTrack; +use crate::room::track::remote_audio_track::RemoteAudioTrack; +use crate::room::track::TrackHandle; +use std::sync::Arc; + +#[derive(Clone)] +pub enum AudioTrackHandle { + Local(Arc), + Remote(Arc), +} + +impl From for TrackHandle { + fn from(audio_track: AudioTrackHandle) -> Self { + match audio_track { + AudioTrackHandle::Local(local_audio) => Self::LocalAudio(local_audio), + AudioTrackHandle::Remote(remote_audio) => Self::RemoteAudio(remote_audio), + } + } +} + +impl TryFrom for AudioTrackHandle { + type Error = &'static str; + + fn try_from(track: TrackHandle) -> Result { + match track { + TrackHandle::LocalAudio(local_audio) => Ok(Self::Local(local_audio)), + TrackHandle::RemoteAudio(remote_audio) => Ok(Self::Remote(remote_audio)), + _ => Err("not a audio track"), + } + } +} diff --git a/crates/livekit-core/src/room/track/events.rs b/crates/livekit-core/src/room/track/events.rs new file mode 100644 index 0000000..86d8cad --- /dev/null +++ b/crates/livekit-core/src/room/track/events.rs @@ -0,0 +1 @@ +pub struct TrackEvents {} diff --git a/crates/livekit-core/src/room/track/local_audio_track.rs b/crates/livekit-core/src/room/track/local_audio_track.rs new file mode 100644 index 0000000..c194c34 --- /dev/null +++ b/crates/livekit-core/src/room/track/local_audio_track.rs @@ -0,0 +1,7 @@ +use crate::room::track::{impl_track_trait, TrackShared}; + +pub struct LocalAudioTrack { + shared: TrackShared, +} + +impl_track_trait!(LocalAudioTrack); diff --git a/crates/livekit-core/src/room/track/local_track.rs b/crates/livekit-core/src/room/track/local_track.rs new file mode 100644 index 0000000..64cc127 --- /dev/null +++ b/crates/livekit-core/src/room/track/local_track.rs @@ -0,0 +1,31 @@ +use crate::room::track::local_audio_track::LocalAudioTrack; +use crate::room::track::local_video_track::LocalVideoTrack; +use crate::room::track::TrackHandle; +use std::sync::Arc; + +#[derive(Clone)] +pub enum LocalTrackHandle { + Audio(Arc), + Video(Arc), +} + +impl From for TrackHandle { + fn from(local_track: LocalTrackHandle) -> Self { + match local_track { + LocalTrackHandle::Audio(local_audio) => Self::LocalAudio(local_audio), + LocalTrackHandle::Video(local_video) => Self::LocalVideo(local_video), + } + } +} + +impl TryFrom for LocalTrackHandle { + type Error = &'static str; + + fn try_from(track: TrackHandle) -> Result { + match track { + TrackHandle::LocalAudio(local_audio) => Ok(Self::Audio(local_audio)), + TrackHandle::LocalVideo(local_video) => Ok(Self::Video(local_video)), + _ => Err("not a local track"), + } + } +} diff --git a/crates/livekit-core/src/room/track/local_video_track.rs b/crates/livekit-core/src/room/track/local_video_track.rs new file mode 100644 index 0000000..cb3f0d1 --- /dev/null +++ b/crates/livekit-core/src/room/track/local_video_track.rs @@ -0,0 +1,7 @@ +use crate::room::track::{impl_track_trait, TrackShared}; + +pub struct LocalVideoTrack { + shared: TrackShared, +} + +impl_track_trait!(LocalVideoTrack); diff --git a/crates/livekit-core/src/room/track/mod.rs b/crates/livekit-core/src/room/track/mod.rs index 1627418..ff53f4d 100644 --- a/crates/livekit-core/src/room/track/mod.rs +++ b/crates/livekit-core/src/room/track/mod.rs @@ -1,3 +1,24 @@ +use crate::room::id::TrackSid; +use crate::room::track::local_audio_track::LocalAudioTrack; +use crate::room::track::local_video_track::LocalVideoTrack; +use crate::room::track::remote_audio_track::RemoteAudioTrack; +use crate::room::track::remote_video_track::RemoteVideoTrack; +use crate::utils::wrap_variants; +use livekit_webrtc::media_stream::{MediaStreamTrackHandle, MediaStreamTrackTrait}; +use parking_lot::Mutex; +use std::sync::atomic::AtomicU8; +use std::sync::Arc; + +pub mod audio_track; +pub mod events; +pub mod local_audio_track; +pub mod local_track; +pub mod local_video_track; +pub mod remote_audio_track; +pub mod remote_track; +pub mod remote_video_track; +pub mod video_track; + #[derive(Debug)] pub enum TrackKind { Unknown, @@ -22,6 +43,16 @@ pub enum StreamState { Paused, } +impl From for StreamState { + fn from(val: u8) -> Self { + match val { + 1 => Self::Active, + 2 => Self::Paused, + _ => Self::Unknown, + } + } +} + #[derive(Debug)] pub enum TrackSource { Unknown, @@ -43,135 +74,114 @@ impl From for TrackSource { } } -pub struct LocalVideoTrack {} -pub struct RemoteVideoTrack {} -pub struct LocalAudioTrack {} - - -pub struct RemoteAudioTrack { - - +pub trait TrackTrait { + fn sid(&self) -> TrackSid; + fn name(&self) -> String; + fn kind(&self) -> TrackKind; + fn stream_state(&self) -> StreamState; + fn start(&self); + fn stop(&self); } -impl RemoteVideoTrack { - pub(crate) fn new() -> Self { - Self {} +pub(super) struct TrackShared { + pub(super) sid: Mutex, + pub(super) name: Mutex, + pub(super) kind: AtomicU8, // TrackKind + pub(super) stream_state: AtomicU8, // StreamState + pub(super) rtc_track: MediaStreamTrackHandle, +} + +impl TrackShared { + pub(crate) fn new( + sid: TrackSid, + name: String, + kind: TrackKind, + rtc_track: MediaStreamTrackHandle, + ) -> Self { + Self { + sid: Mutex::new(sid), + name: Mutex::new(name), + kind: AtomicU8::new(kind as u8), + stream_state: AtomicU8::new(StreamState::Active as u8), + rtc_track: rtc_track, + } + } + + pub(crate) fn start(&self) { + self.rtc_track.set_enabled(true); + } + + pub(crate) fn stop(&self) { + self.rtc_track.set_enabled(false); } } -impl RemoteAudioTrack { - pub(crate) fn new() -> Self { - Self {} - } +#[derive(Clone)] +pub enum TrackHandle { + LocalVideo(Arc), + LocalAudio(Arc), + RemoteVideo(Arc), + RemoteAudio(Arc), } -pub enum RemoteTrack { - Audio(RemoteAudioTrack), - Video(RemoteVideoTrack), +impl TrackTrait for TrackHandle { + wrap_variants!( + [LocalVideo, LocalAudio, RemoteVideo, RemoteAudio] + fnc!(sid, TrackSid, []); + fnc!(name, String, []); + fnc!(kind, TrackKind, []); + fnc!(stream_state, StreamState, []); + fnc!(start, (), []); + fnc!(stop, (), []); + ); } -pub enum LocalTrack { - Audio(LocalAudioTrack), - Video(LocalVideoTrack), -} - -pub enum VideoTrack { - Local(LocalVideoTrack), - Remote(RemoteVideoTrack), -} - -pub enum AudioTrack { - Local(LocalAudioTrack), - Remote(RemoteAudioTrack), -} - -pub enum Track { - LocalVideo(LocalVideoTrack), - LocalAudio(LocalAudioTrack), - RemoteVideo(RemoteVideoTrack), - RemoteAudio(RemoteAudioTrack), -} - -impl From for Track { - fn from(video_track: VideoTrack) -> Self { - match video_track { - VideoTrack::Local(local_video) => Self::LocalVideo(local_video), - VideoTrack::Remote(remote_video) => Self::RemoteVideo(remote_video), +impl TrackHandle { + pub fn rtc_track(&self) -> MediaStreamTrackHandle { + match self { + Self::RemoteVideo(remote_video) => { + MediaStreamTrackHandle::Video(remote_video.rtc_track()) + } + Self::RemoteAudio(remote_audio) => { + MediaStreamTrackHandle::Audio(remote_audio.rtc_track()) + } + _ => todo!(), } } } -impl From for Track { - fn from(audio_track: AudioTrack) -> Self { - match audio_track { - AudioTrack::Local(local_audio) => Self::LocalAudio(local_audio), - AudioTrack::Remote(remote_audio) => Self::RemoteAudio(remote_audio), +macro_rules! impl_track_trait { + ($x:ident) => { + use crate::room::id::TrackSid; + use crate::room::track::{StreamState, TrackKind, TrackTrait}; + use std::sync::atomic::Ordering; + + impl TrackTrait for $x { + fn sid(&self) -> TrackSid { + self.shared.sid.lock().clone() + } + + fn name(&self) -> String { + self.shared.name.lock().clone() + } + + fn kind(&self) -> TrackKind { + self.shared.kind.load(Ordering::SeqCst).into() + } + + fn stream_state(&self) -> StreamState { + self.shared.stream_state.load(Ordering::SeqCst).into() + } + + fn start(&self) { + self.shared.start(); + } + + fn stop(&self) { + self.shared.stop(); + } } - } + }; } -impl From for Track { - fn from(local_track: LocalTrack) -> Self { - match local_track { - LocalTrack::Audio(local_audio) => Self::LocalAudio(local_audio), - LocalTrack::Video(local_video) => Self::LocalVideo(local_video), - } - } -} - -impl From for Track { - fn from(remote_track: RemoteTrack) -> Self { - match remote_track { - RemoteTrack::Audio(remote_audio) => Self::RemoteAudio(remote_audio), - RemoteTrack::Video(remote_video) => Self::RemoteVideo(remote_video), - } - } -} - -impl TryFrom for VideoTrack { - type Error = &'static str; - - fn try_from(track: Track) -> Result { - match track { - Track::LocalVideo(local_video) => Ok(Self::Local(local_video)), - Track::RemoteVideo(remote_video) => Ok(Self::Remote(remote_video)), - _ => Err("not a video track"), - } - } -} - -impl TryFrom for AudioTrack { - type Error = &'static str; - - fn try_from(track: Track) -> Result { - match track { - Track::LocalAudio(local_audio) => Ok(Self::Local(local_audio)), - Track::RemoteAudio(remote_audio) => Ok(Self::Remote(remote_audio)), - _ => Err("not a audio track"), - } - } -} - -impl TryFrom for LocalTrack { - type Error = &'static str; - - fn try_from(track: Track) -> Result { - match track { - Track::LocalAudio(local_audio) => Ok(Self::Audio(local_audio)), - Track::LocalVideo(local_video) => Ok(Self::Video(local_video)), - _ => Err("not a local track"), - } - } -} - -impl TryFrom for RemoteTrack { - type Error = &'static str; - - fn try_from(track: Track) -> Result { - match track { - Track::RemoteAudio(remote_audio) => Ok(Self::Audio(remote_audio)), - Track::RemoteVideo(remote_video) => Ok(Self::Video(remote_video)), - _ => Err("not a remote track"), - } - } -} +pub(super) use impl_track_trait; diff --git a/crates/livekit-core/src/room/track/remote_audio_track.rs b/crates/livekit-core/src/room/track/remote_audio_track.rs new file mode 100644 index 0000000..933f307 --- /dev/null +++ b/crates/livekit-core/src/room/track/remote_audio_track.rs @@ -0,0 +1,30 @@ +use crate::room::track::{impl_track_trait, TrackShared}; +use livekit_webrtc::media_stream::{AudioTrack, MediaStreamTrackHandle}; +use std::sync::Arc; + +pub struct RemoteAudioTrack { + shared: TrackShared, +} + +impl RemoteAudioTrack { + pub(crate) fn new(sid: TrackSid, name: String, track: Arc) -> Self { + Self { + shared: TrackShared::new( + sid, + name, + TrackKind::Audio, + MediaStreamTrackHandle::Audio(track), + ), + } + } + + pub fn rtc_track(&self) -> Arc { + if let MediaStreamTrackHandle::Audio(audio) = &self.shared.rtc_track { + audio.clone() + } else { + unreachable!() + } + } +} + +impl_track_trait!(RemoteAudioTrack); diff --git a/crates/livekit-core/src/room/track/remote_track.rs b/crates/livekit-core/src/room/track/remote_track.rs new file mode 100644 index 0000000..591b443 --- /dev/null +++ b/crates/livekit-core/src/room/track/remote_track.rs @@ -0,0 +1,49 @@ +use std::sync::Arc; + +use super::{StreamState, TrackKind}; +use crate::room::id::TrackSid; +use crate::room::track::remote_audio_track::RemoteAudioTrack; +use crate::room::track::remote_video_track::RemoteVideoTrack; +use crate::room::track::TrackHandle; +use crate::utils::wrap_variants; + +use super::TrackTrait; + +#[derive(Clone)] +pub enum RemoteTrackHandle { + Audio(Arc), + Video(Arc), +} + +impl TrackTrait for RemoteTrackHandle { + wrap_variants!( + [Audio, Video] + fnc!(sid, TrackSid, []); + fnc!(name, String, []); + fnc!(kind, TrackKind, []); + fnc!(stream_state, StreamState, []); + fnc!(start, (), []); + fnc!(stop, (), []); + ); +} + +impl From for TrackHandle { + fn from(remote_track: RemoteTrackHandle) -> Self { + match remote_track { + RemoteTrackHandle::Audio(remote_audio) => Self::RemoteAudio(remote_audio), + RemoteTrackHandle::Video(remote_video) => Self::RemoteVideo(remote_video), + } + } +} + +impl TryFrom for RemoteTrackHandle { + type Error = &'static str; + + fn try_from(track: TrackHandle) -> Result { + match track { + TrackHandle::RemoteAudio(remote_audio) => Ok(Self::Audio(remote_audio)), + TrackHandle::RemoteVideo(remote_video) => Ok(Self::Video(remote_video)), + _ => Err("not a remote track"), + } + } +} diff --git a/crates/livekit-core/src/room/track/remote_video_track.rs b/crates/livekit-core/src/room/track/remote_video_track.rs new file mode 100644 index 0000000..57caf67 --- /dev/null +++ b/crates/livekit-core/src/room/track/remote_video_track.rs @@ -0,0 +1,31 @@ +use livekit_webrtc::media_stream::{MediaStreamTrackHandle, VideoTrack}; +use std::sync::Arc; + +use crate::room::track::{impl_track_trait, TrackShared}; + +pub struct RemoteVideoTrack { + shared: TrackShared, +} + +impl RemoteVideoTrack { + pub(crate) fn new(sid: TrackSid, name: String, track: Arc) -> Self { + Self { + shared: TrackShared::new( + sid, + name, + TrackKind::Video, + MediaStreamTrackHandle::Video(track), + ), + } + } + + pub fn rtc_track(&self) -> Arc { + if let MediaStreamTrackHandle::Video(video) = &self.shared.rtc_track { + video.clone() + } else { + unreachable!() + } + } +} + +impl_track_trait!(RemoteVideoTrack); diff --git a/crates/livekit-core/src/room/track/video_track.rs b/crates/livekit-core/src/room/track/video_track.rs new file mode 100644 index 0000000..4294e5e --- /dev/null +++ b/crates/livekit-core/src/room/track/video_track.rs @@ -0,0 +1,31 @@ +use crate::room::track::local_video_track::LocalVideoTrack; +use crate::room::track::remote_video_track::RemoteVideoTrack; +use crate::room::track::TrackHandle; +use std::sync::Arc; + +#[derive(Clone)] +pub enum VideoTrackHandle { + Local(Arc), + Remote(Arc), +} + +impl From for TrackHandle { + fn from(video_track: VideoTrackHandle) -> Self { + match video_track { + VideoTrackHandle::Local(local_video) => Self::LocalVideo(local_video), + VideoTrackHandle::Remote(remote_video) => Self::RemoteVideo(remote_video), + } + } +} + +impl TryFrom for VideoTrackHandle { + type Error = &'static str; + + fn try_from(track: TrackHandle) -> Result { + match track { + TrackHandle::LocalVideo(local_video) => Ok(Self::Local(local_video)), + TrackHandle::RemoteVideo(remote_video) => Ok(Self::Remote(remote_video)), + _ => Err("not a video track"), + } + } +} diff --git a/crates/livekit-core/src/utils.rs b/crates/livekit-core/src/utils.rs new file mode 100644 index 0000000..c250092 --- /dev/null +++ b/crates/livekit-core/src/utils.rs @@ -0,0 +1,26 @@ +macro_rules! wrap_variants { + // This arm is used to avoid nested loops with the arguments + // The arguments are transformed to $combined_args TokenTree + (@match $self:ident $fnc:ident $combined_args:tt [$($variant:ident),+]) => { + match $self { + $( + Self::$variant(inner) => inner.$fnc$combined_args, + )+ + } + }; + + ($fnc:ident, $ret:ty, [$($arg:ident: $t:ty),*], [$($variant:ident),+]) => { + fn $fnc(&self, $($arg: $t),*) -> $ret { + wrap_variants!(@match self $fnc ($($arg,)*) [$($variant),+]) + } + }; + + ($variants:tt $(fnc!($fnc:ident, $ret:ty, $args:tt);)+) => { + $( + wrap_variants!($fnc, $ret, $args, $variants); + )* + }; +} + +pub(crate) use wrap_variants; + diff --git a/crates/livekit-webrtc/libwebrtc-sys/CMakeLists.txt b/crates/livekit-webrtc/libwebrtc-sys/CMakeLists.txt deleted file mode 100644 index b2b7e04..0000000 --- a/crates/livekit-webrtc/libwebrtc-sys/CMakeLists.txt +++ /dev/null @@ -1,23 +0,0 @@ -# IMPORTANT NOTE -# This file is just used because some IDEs need to understand how to do autocompletion. -# This file is completely ignored by the library ( See build.rs for the build system ) - -cmake_minimum_required(VERSION 3.22) -project(livekit-webrtc) - -set(CMAKE_CXX_STANDARD 17) - -add_definitions(-DWEBRTC_WIN) - -include_directories(libwebrtc/include) -include_directories(libwebrtc/include/third_party/abseil-cpp/) -include_directories(libwebrtc/include/third_party/libc++/) -include_directories(include/) -include_directories(../../../target/cxxbridge) # Can be different - -file(GLOB_RECURSE SRC src/*.cpp) -add_library(livekit-webrtc ${SRC}) - -#include_directories(/Users/theomonnom/Library/Android/sdk/ndk/25.0.8775105/toolchains/llvm/prebuilt/darwin-x86_64/sysroot/usr/include) -#find_library(ANDROID_LIB_ANDROID android) -#target_link_libraries(client_sdk_native PRIVATE android) \ No newline at end of file diff --git a/crates/livekit-webrtc/libwebrtc-sys/build.rs b/crates/livekit-webrtc/libwebrtc-sys/build.rs index a65a667..b1fdd96 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/build.rs +++ b/crates/livekit-webrtc/libwebrtc-sys/build.rs @@ -52,7 +52,7 @@ fn macos_link_search_path() -> Option { fn main() { // TODO Download precompiled binaries of WebRTC for the target_os - let target_os = "macos"; + let target_os = "windows"; //let target_arch = "arm64"; let libwebrtc_dir = path::PathBuf::from("libwebrtc"); @@ -79,6 +79,8 @@ fn main() { "src/rtp_transceiver.rs", "src/rtc_error.rs", "src/webrtc.rs", + "src/video_frame.rs", + "src/video_frame_buffer.rs", ]); builder.file("src/peer_connection.cpp"); diff --git a/crates/livekit-webrtc/libwebrtc-sys/compile_flags.txt b/crates/livekit-webrtc/libwebrtc-sys/compile_flags.txt new file mode 100644 index 0000000..e0fb4f0 --- /dev/null +++ b/crates/livekit-webrtc/libwebrtc-sys/compile_flags.txt @@ -0,0 +1,8 @@ +-xc++ +-std=c++17 +-Iinclude +-Ilibwebrtc/include +-Ilibwebrtc/include/third_party/abseil-cpp +-Ilibwebrtc/include/third_party/libc++ +-I../../../target/cxxbridge +-DWEBRTC_WIN diff --git a/crates/livekit-webrtc/libwebrtc-sys/include/livekit/media_stream.h b/crates/livekit-webrtc/libwebrtc-sys/include/livekit/media_stream.h index c2b3164..bf5d652 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/include/livekit/media_stream.h +++ b/crates/livekit-webrtc/libwebrtc-sys/include/livekit/media_stream.h @@ -13,26 +13,7 @@ namespace livekit { -class MediaStreamTrack { - public: - explicit MediaStreamTrack( - rtc::scoped_refptr track); - - rust::String kind() const; - rust::String id() const; - - bool enabled() const; - bool set_enabled(bool enable); - - TrackState state() const; - - private: - rtc::scoped_refptr track_; -}; - -static std::unique_ptr _unique_media_stream_track() { - return nullptr; // Ignore -} +class NativeVideoFrameSink; class MediaStream { public: @@ -47,6 +28,96 @@ class MediaStream { static std::unique_ptr _unique_media_stream() { return nullptr; // Ignore } + +class MediaStreamTrack { + protected: + explicit MediaStreamTrack( + rtc::scoped_refptr track); + + public: + static std::unique_ptr from( + rtc::scoped_refptr track); + + rust::String kind() const; + rust::String id() const; + + bool enabled() const; + bool set_enabled(bool enable); + + TrackState state() const; + + protected: + rtc::scoped_refptr track_; +}; + +static std::unique_ptr _unique_media_stream_track() { + return nullptr; // Ignore +} + +class AudioTrack : public MediaStreamTrack { + public: + explicit AudioTrack(rtc::scoped_refptr track); +}; + +static std::unique_ptr _unique_audio_track() { + return nullptr; // Ignore +} + +class VideoTrack : public MediaStreamTrack { + public: + explicit VideoTrack(rtc::scoped_refptr track); + + void add_sink(NativeVideoFrameSink& sink); + void remove_sink(NativeVideoFrameSink& sink); + + void set_should_receive(bool should_receive); + bool should_receive() const; + ContentHint content_hint() const; + void set_content_hint(ContentHint hint); + + private: + webrtc::VideoTrackInterface* track() const { + return static_cast(track_.get()); + } +}; + +static std::unique_ptr _unique_video_track() { + return nullptr; // Ignore +} + +class NativeVideoFrameSink + : public rtc::VideoSinkInterface { + public: + explicit NativeVideoFrameSink(rust::Box observer); + + void OnFrame(const webrtc::VideoFrame& frame) override; + void OnDiscardedFrame() override; + void OnConstraintsChanged( + const webrtc::VideoTrackSourceConstraints& constraints) override; + + private: + rust::Box observer_; +}; + +std::unique_ptr create_native_video_frame_sink( + rust::Box observer); + +const MediaStreamTrack* video_to_media(const VideoTrack* track) { + return track; +} + +const MediaStreamTrack* audio_to_media(const AudioTrack* track) { + return track; +} + +const VideoTrack* media_to_video(const MediaStreamTrack* track) { + return static_cast(track); +} + +const AudioTrack* media_to_audio(const MediaStreamTrack* track) { + return static_cast(track); +} + } // namespace livekit #endif // CLIENT_SDK_NATIVE_MEDIA_STREAM_INTERFACE_H diff --git a/crates/livekit-webrtc/libwebrtc-sys/include/livekit/rust_types.h b/crates/livekit-webrtc/libwebrtc-sys/include/livekit/rust_types.h index 1d9d7ef..f342734 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/include/livekit/rust_types.h +++ b/crates/livekit-webrtc/libwebrtc-sys/include/livekit/rust_types.h @@ -15,6 +15,7 @@ struct SetLocalSdpObserverWrapper; struct SetRemoteSdpObserverWrapper; struct DataChannelObserverWrapper; struct AddIceCandidateObserverWrapper; +struct VideoFrameSinkWrapper; // Shared types enum class PeerConnectionState; @@ -24,6 +25,9 @@ enum class IceGatheringState; enum class SdpType; enum class DataState; enum class TrackState; +enum class ContentHint; +enum class VideoRotation; +enum class VideoFrameBufferType; struct SdpParseError; struct RTCOfferAnswerOptions; struct RTCError; diff --git a/crates/livekit-webrtc/libwebrtc-sys/include/livekit/video_frame.h b/crates/livekit-webrtc/libwebrtc-sys/include/livekit/video_frame.h new file mode 100644 index 0000000..9c719f4 --- /dev/null +++ b/crates/livekit-webrtc/libwebrtc-sys/include/livekit/video_frame.h @@ -0,0 +1,48 @@ +// +// Created by theom on 14/11/2022. +// + +#ifndef LIVEKIT_WEBRTC_VIDEO_FRAME_H +#define LIVEKIT_WEBRTC_VIDEO_FRAME_H + +#include "api/video/video_frame.h" +#include "livekit/rust_types.h" +#include "livekit/video_frame_buffer.h" + +namespace livekit { + +class VideoFrame { + public: + explicit VideoFrame(const webrtc::VideoFrame& frame) + : frame_(std::move(frame)) {} + + int width() const { return frame_.width(); } + int height() const { return frame_.height(); } + uint32_t size() const { return frame_.size(); } + uint16_t id() const { return frame_.id(); } + int64_t timestamp_us() const { return frame_.timestamp_us(); } + int64_t ntp_time_ms() const { return frame_.ntp_time_ms(); } + uint32_t transport_frame_id() const { return frame_.transport_frame_id(); } + uint32_t timestamp() const { return frame_.timestamp(); } + + VideoRotation rotation() const { + return static_cast(frame_.rotation()); + } + + // TODO(theomonnom) This shouldn't create a new shared_ptr at each call + std::shared_ptr video_frame_buffer() const { + return std::make_shared(frame_.video_frame_buffer()); + } + + private: + webrtc::VideoFrame frame_; +}; + +static std::unique_ptr _unique_video_frame() { + return nullptr; // Ignore +} + + +} // namespace livekit + +#endif // LIVEKIT_WEBRTC_VIDEO_FRAME_H diff --git a/crates/livekit-webrtc/libwebrtc-sys/include/livekit/video_frame_buffer.h b/crates/livekit-webrtc/libwebrtc-sys/include/livekit/video_frame_buffer.h new file mode 100644 index 0000000..3b4909c --- /dev/null +++ b/crates/livekit-webrtc/libwebrtc-sys/include/livekit/video_frame_buffer.h @@ -0,0 +1,93 @@ +// +// Created by theom on 14/11/2022. +// + +#ifndef LIVEKIT_WEBRTC_VIDEO_FRAME_BUFFER_H +#define LIVEKIT_WEBRTC_VIDEO_FRAME_BUFFER_H + +#include "api/video/video_frame_buffer.h" +#include "rust_types.h" + +namespace livekit { + +class PlanarYuvBuffer; +class PlanarYuv8Buffer; +class I420Buffer; + +class VideoFrameBuffer { + public: + explicit VideoFrameBuffer(rtc::scoped_refptr buffer) + : buffer_(std::move(buffer)) {} + + VideoFrameBufferType buffer_type() const { + return static_cast(buffer_->type()); + } + + int width() const { return buffer_->width(); } + int height() const { return buffer_->height(); } + + std::shared_ptr to_i420() { + return std::make_shared(buffer_->ToI420()); + } + + protected: + rtc::scoped_refptr buffer_; +}; + +class PlanarYuvBuffer : public VideoFrameBuffer { + public: + explicit PlanarYuvBuffer(rtc::scoped_refptr buffer) + : VideoFrameBuffer(buffer) {} + + int chroma_width() const { return buffer()->ChromaWidth(); } + int chroma_height() const { return buffer()->ChromaHeight(); } + + int stride_y() const { return buffer()->StrideY(); } + int stride_u() const { return buffer()->StrideU(); } + int stride_v() const { return buffer()->StrideV(); } + + private: + webrtc::PlanarYuvBuffer* buffer() const { + return static_cast(buffer_.get()); + } +}; + +class PlanarYuv8Buffer : public PlanarYuvBuffer { + public: + explicit PlanarYuv8Buffer(rtc::scoped_refptr buffer) + : PlanarYuvBuffer(buffer) {} + + const uint8_t* data_y() const { return buffer()->DataY(); } + const uint8_t* data_u() const { return buffer()->DataU(); } + const uint8_t* data_v() const { return buffer()->DataV(); } + + private: + webrtc::PlanarYuv8Buffer* buffer() const { + return static_cast(buffer_.get()); + } +}; + +class I420Buffer : public PlanarYuv8Buffer { + public: + explicit I420Buffer(rtc::scoped_refptr buffer) + : PlanarYuv8Buffer(buffer) {} +}; + +std::shared_ptr to_video_frame_buffer( + std::shared_ptr buffer) { + return buffer; +} + +std::shared_ptr to_yuv_buffer( + std::shared_ptr buffer) { + return buffer; +} + +std::shared_ptr to_yuv8_buffer( + std::shared_ptr buffer) { + return buffer; +} + +} // namespace livekit + +#endif // LIVEKIT_WEBRTC_VIDEO_FRAME_BUFFER_H diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/candidate.cpp b/crates/livekit-webrtc/libwebrtc-sys/src/candidate.cpp index 970575d..09f2190 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/src/candidate.cpp +++ b/crates/livekit-webrtc/libwebrtc-sys/src/candidate.cpp @@ -7,4 +7,4 @@ namespace livekit { Candidate::Candidate(const cricket::Candidate& candidate) : candidate_(candidate) {} -} // namespace livekit \ No newline at end of file +} // namespace livekit diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/data_channel.cpp b/crates/livekit-webrtc/libwebrtc-sys/src/data_channel.cpp index 72f3c65..41986f1 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/src/data_channel.cpp +++ b/crates/livekit-webrtc/libwebrtc-sys/src/data_channel.cpp @@ -87,4 +87,4 @@ std::unique_ptr create_native_data_channel_observer( rust::Box observer) { return std::make_unique(std::move(observer)); } -} // namespace livekit \ No newline at end of file +} // namespace livekit diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/jsep.cpp b/crates/livekit-webrtc/libwebrtc-sys/src/jsep.cpp index 20ba43d..6340a83 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/src/jsep.cpp +++ b/crates/livekit-webrtc/libwebrtc-sys/src/jsep.cpp @@ -158,4 +158,4 @@ create_native_set_remote_sdp_observer( std::move(observer))}); } -} // namespace livekit \ No newline at end of file +} // namespace livekit diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/lib.rs b/crates/livekit-webrtc/libwebrtc-sys/src/lib.rs index ecd1f3e..ab6e52a 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/src/lib.rs +++ b/crates/livekit-webrtc/libwebrtc-sys/src/lib.rs @@ -7,4 +7,10 @@ pub mod peer_connection_factory; pub mod rtc_error; pub mod rtp_receiver; pub mod rtp_transceiver; +pub mod video_frame; +pub mod video_frame_buffer; pub mod webrtc; + +pub const MEDIA_TYPE_VIDEO: &str = "video"; +pub const MEDIA_TYPE_AUDIO: &str = "audio"; +pub const MEDIA_TYPE_DATA: &str = "data"; diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/media_stream.cpp b/crates/livekit-webrtc/libwebrtc-sys/src/media_stream.cpp index 04eb1f9..b4017ee 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/src/media_stream.cpp +++ b/crates/livekit-webrtc/libwebrtc-sys/src/media_stream.cpp @@ -4,12 +4,27 @@ #include "livekit/media_stream.h" +#include "libwebrtc-sys/src/media_stream.rs.h" + namespace livekit { MediaStreamTrack::MediaStreamTrack( rtc::scoped_refptr track) : track_(std::move(track)) {} +std::unique_ptr MediaStreamTrack::from( + rtc::scoped_refptr track) { + if (track->kind() == webrtc::MediaStreamTrackInterface::kVideoKind) { + return std::make_unique( + rtc::scoped_refptr( + static_cast(track.get()))); + } else { + return std::make_unique( + rtc::scoped_refptr( + static_cast(track.get()))); + } +} + rust::String MediaStreamTrack::kind() const { return track_->kind(); } @@ -35,7 +50,60 @@ MediaStream::MediaStream( : media_stream_(std::move(stream)) {} rust::String MediaStream::id() const { - return media_stream_->id(); + return media_stream_->id(); } -} // namespace livekit \ No newline at end of file +VideoTrack::VideoTrack(rtc::scoped_refptr track) + : MediaStreamTrack(std::move(track)) {} + +void VideoTrack::add_sink(NativeVideoFrameSink& sink) { + track()->AddOrUpdateSink(&sink, rtc::VideoSinkWants()); +} + +void VideoTrack::remove_sink(NativeVideoFrameSink& sink) { + track()->RemoveSink(&sink); +} + +void VideoTrack::set_should_receive(bool should_receive) { + track()->set_should_receive(should_receive); +} + +bool VideoTrack::should_receive() const { + return track()->should_receive(); +} + +ContentHint VideoTrack::content_hint() const { + return static_cast(track()->content_hint()); +} + +void VideoTrack::set_content_hint(ContentHint hint) { + track()->set_content_hint( + static_cast(hint)); +} + +NativeVideoFrameSink::NativeVideoFrameSink( + rust::Box observer) + : observer_(std::move(observer)) {} + +void NativeVideoFrameSink::OnFrame(const webrtc::VideoFrame& frame) { + observer_->on_frame(std::make_unique(frame)); +} + +void NativeVideoFrameSink::OnDiscardedFrame() { + observer_->on_discarded_frame(); +} + +void NativeVideoFrameSink::OnConstraintsChanged( + const webrtc::VideoTrackSourceConstraints& constraints) { + VideoTrackSourceConstraints cst; + cst.min_fps = constraints.min_fps.value_or(-1); + cst.max_fps = constraints.max_fps.value_or(-1); + observer_->on_constraints_changed(cst); +} + +std::unique_ptr create_native_video_frame_sink( + rust::Box observer) { + return std::make_unique(std::move(observer)); +} + +} // namespace livekit diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/media_stream.rs b/crates/livekit-webrtc/libwebrtc-sys/src/media_stream.rs index 0f46ebf..86b8215 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/src/media_stream.rs +++ b/crates/livekit-webrtc/libwebrtc-sys/src/media_stream.rs @@ -1,3 +1,7 @@ +use cxx::UniquePtr; + +use crate::video_frame::ffi::VideoFrame; + #[cxx::bridge(namespace = "livekit")] pub mod ffi { @@ -8,11 +12,33 @@ pub mod ffi { Ended, } + #[derive(Debug)] + #[repr(i32)] + pub enum ContentHint { + None, + Fluid, + Detailed, + Text, + } + + // -1 = optional + pub struct VideoTrackSourceConstraints { + pub min_fps: f64, + pub max_fps: f64, + } + unsafe extern "C++" { include!("livekit/media_stream.h"); + include!("livekit/video_frame.h"); + type NativeVideoFrameSink; type MediaStreamTrack; type MediaStream; + type AudioTrack; + type VideoTrack; + type VideoFrame = crate::video_frame::ffi::VideoFrame; + + fn id(self: &MediaStream) -> String; fn kind(self: &MediaStreamTrack) -> String; fn id(self: &MediaStreamTrack) -> String; @@ -20,17 +46,84 @@ pub mod ffi { fn set_enabled(self: Pin<&mut MediaStreamTrack>, enable: bool) -> bool; fn state(self: &MediaStreamTrack) -> TrackState; - fn id(self: &MediaStream) -> String; + unsafe fn add_sink(self: Pin<&mut VideoTrack>, sink: Pin<&mut NativeVideoFrameSink>); + unsafe fn remove_sink(self: Pin<&mut VideoTrack>, sink: Pin<&mut NativeVideoFrameSink>); + + fn set_should_receive(self: Pin<&mut VideoTrack>, should_receive: bool); + fn should_receive(self: &VideoTrack) -> bool; + fn content_hint(self: &VideoTrack) -> ContentHint; + fn set_content_hint(self: Pin<&mut VideoTrack>, hint: ContentHint); + + fn create_native_video_frame_sink( + observer: Box, + ) -> UniquePtr; + + unsafe fn video_to_media(track: *const VideoTrack) -> *const MediaStreamTrack; + unsafe fn audio_to_media(track: *const AudioTrack) -> *const MediaStreamTrack; + unsafe fn media_to_video(track: *const MediaStreamTrack) -> *const VideoTrack; + unsafe fn media_to_audio(track: *const MediaStreamTrack) -> *const AudioTrack; fn _unique_media_stream_track() -> UniquePtr; // Ignore fn _unique_media_stream() -> UniquePtr; // Ignore + fn _unique_audio_track() -> UniquePtr; // Ignore + fn _unique_video_track() -> UniquePtr; // Ignore + } + + extern "Rust" { + type VideoFrameSinkWrapper; + + fn on_frame(self: &VideoFrameSinkWrapper, frame: UniquePtr); + fn on_discarded_frame(self: &VideoFrameSinkWrapper); + fn on_constraints_changed( + self: &VideoFrameSinkWrapper, + constraints: VideoTrackSourceConstraints, + ); } } unsafe impl Sync for ffi::MediaStreamTrack {} - unsafe impl Send for ffi::MediaStreamTrack {} - unsafe impl Sync for ffi::MediaStream {} +unsafe impl Send for ffi::MediaStream {} +unsafe impl Send for ffi::AudioTrack {} +unsafe impl Sync for ffi::AudioTrack {} +unsafe impl Send for ffi::VideoTrack {} +unsafe impl Sync for ffi::VideoTrack {} +unsafe impl Send for ffi::NativeVideoFrameSink {} +unsafe impl Sync for ffi::NativeVideoFrameSink {} -unsafe impl Send for ffi::MediaStream {} \ No newline at end of file +pub trait VideoFrameSink: Send + Sync { + fn on_frame(&self, frame: UniquePtr); + fn on_discarded_frame(&self); + fn on_constraints_changed(&self, constraints: ffi::VideoTrackSourceConstraints); +} + +pub struct VideoFrameSinkWrapper { + observer: *mut dyn VideoFrameSink, +} + +impl VideoFrameSinkWrapper { + /// # Safety + /// VideoFrameSink must lives as long as VideoSinkInterfaceWrapper does + pub unsafe fn new(observer: *mut dyn VideoFrameSink) -> Self { + Self { observer } + } + + fn on_frame(&self, frame: UniquePtr) { + unsafe { + (*self.observer).on_frame(frame); + } + } + + fn on_discarded_frame(&self) { + unsafe { + (*self.observer).on_discarded_frame(); + } + } + + fn on_constraints_changed(&self, constraints: ffi::VideoTrackSourceConstraints) { + unsafe { + (*self.observer).on_constraints_changed(constraints); + } + } +} diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/peer_connection.rs b/crates/livekit-webrtc/libwebrtc-sys/src/peer_connection.rs index f6b914f..688b7d3 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/src/peer_connection.rs +++ b/crates/livekit-webrtc/libwebrtc-sys/src/peer_connection.rs @@ -344,7 +344,7 @@ pub struct PeerConnectionObserverWrapper { } impl PeerConnectionObserverWrapper { - /// SAFETY + /// # Safety /// PeerConnectionObserver must lives as long as PeerConnectionObserverWrapper does pub unsafe fn new(observer: *mut dyn PeerConnectionObserver) -> Self { Self { observer } diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/peer_connection_factory.rs b/crates/livekit-webrtc/libwebrtc-sys/src/peer_connection_factory.rs index 6266242..2850983 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/src/peer_connection_factory.rs +++ b/crates/livekit-webrtc/libwebrtc-sys/src/peer_connection_factory.rs @@ -45,7 +45,7 @@ pub mod ffi { ) -> UniquePtr; fn create_rtc_configuration(conf: RTCConfiguration) -> UniquePtr; - /// SAFETY + /// # Safety /// The observer must live as long as the PeerConnection unsafe fn create_peer_connection( self: &PeerConnectionFactory, diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/rtp_receiver.cpp b/crates/livekit-webrtc/libwebrtc-sys/src/rtp_receiver.cpp index e9095c0..a7b70c1 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/src/rtp_receiver.cpp +++ b/crates/livekit-webrtc/libwebrtc-sys/src/rtp_receiver.cpp @@ -10,7 +10,7 @@ RtpReceiver::RtpReceiver( : receiver_(std::move(receiver)) {} std::unique_ptr RtpReceiver::track() const { - return std::make_unique(receiver_->track()); + return MediaStreamTrack::from(receiver_->track()); } } // namespace livekit \ No newline at end of file diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/rtp_receiver.rs b/crates/livekit-webrtc/libwebrtc-sys/src/rtp_receiver.rs index 34327de..a349fa6 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/src/rtp_receiver.rs +++ b/crates/livekit-webrtc/libwebrtc-sys/src/rtp_receiver.rs @@ -15,4 +15,4 @@ pub mod ffi { unsafe impl Sync for ffi::RtpReceiver {} -unsafe impl Send for ffi::RtpReceiver {} \ No newline at end of file +unsafe impl Send for ffi::RtpReceiver {} diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/video_frame.rs b/crates/livekit-webrtc/libwebrtc-sys/src/video_frame.rs new file mode 100644 index 0000000..79ba2bf --- /dev/null +++ b/crates/livekit-webrtc/libwebrtc-sys/src/video_frame.rs @@ -0,0 +1,32 @@ +#[cxx::bridge(namespace = "livekit")] +pub mod ffi { + #[derive(Debug)] + #[repr(i32)] + pub enum VideoRotation { + VideoRotation0 = 0, + VideoRotation90 = 90, + VideoRotation180 = 180, + VideoRotation270 = 270, + } + + unsafe extern "C++" { + include!("livekit/video_frame.h"); + include!("livekit/video_frame_buffer.h"); + + type VideoFrame; + type VideoFrameBuffer = crate::video_frame_buffer::ffi::VideoFrameBuffer; + + fn width(self: &VideoFrame) -> i32; + fn height(self: &VideoFrame) -> i32; + fn size(self: &VideoFrame) -> u32; + fn id(self: &VideoFrame) -> u16; + fn timestamp_us(self: &VideoFrame) -> i64; + fn ntp_time_ms(self: &VideoFrame) -> i64; + fn transport_frame_id(self: &VideoFrame) -> u32; + fn timestamp(self: &VideoFrame) -> u32; + fn rotation(self: &VideoFrame) -> VideoRotation; + fn video_frame_buffer(self: &VideoFrame) -> SharedPtr; + + fn _unique_video_frame() -> UniquePtr; // Ignore + } +} diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/video_frame_buffer.rs b/crates/livekit-webrtc/libwebrtc-sys/src/video_frame_buffer.rs new file mode 100644 index 0000000..65d4fad --- /dev/null +++ b/crates/livekit-webrtc/libwebrtc-sys/src/video_frame_buffer.rs @@ -0,0 +1,43 @@ +#[cxx::bridge(namespace = "livekit")] +pub mod ffi { + #[derive(Debug)] + #[repr(i32)] + pub enum VideoFrameBufferType { + Native, + I420, + I420A, + I422, + I444, + I010, + NV12, + } + + unsafe extern "C++" { + include!("livekit/video_frame_buffer.h"); + + type VideoFrameBuffer; + type PlanarYuvBuffer; + type PlanarYuv8Buffer; + type I420Buffer; + + fn buffer_type(self: &VideoFrameBuffer) -> VideoFrameBufferType; + fn width(self: &VideoFrameBuffer) -> i32; + fn height(self: &VideoFrameBuffer) -> i32; + fn to_i420(self: Pin<&mut VideoFrameBuffer>) -> SharedPtr; + + fn chroma_width(self: &PlanarYuvBuffer) -> i32; + fn chroma_height(self: &PlanarYuvBuffer) -> i32; + fn stride_y(self: &PlanarYuvBuffer) -> i32; + fn stride_u(self: &PlanarYuvBuffer) -> i32; + fn stride_v(self: &PlanarYuvBuffer) -> i32; + + fn data_y(self: &PlanarYuv8Buffer) -> *const u8; + fn data_u(self: &PlanarYuv8Buffer) -> *const u8; + fn data_v(self: &PlanarYuv8Buffer) -> *const u8; + + fn to_video_frame_buffer(buffer: SharedPtr) + -> SharedPtr; + fn to_yuv_buffer(buffer: SharedPtr) -> SharedPtr; + fn to_yuv8_buffer(buffer: SharedPtr) -> SharedPtr; + } +} diff --git a/crates/livekit-webrtc/src/.media_stream.rs.rustfmt b/crates/livekit-webrtc/src/.media_stream.rs.rustfmt new file mode 100644 index 0000000..c5641bd --- /dev/null +++ b/crates/livekit-webrtc/src/.media_stream.rs.rustfmt @@ -0,0 +1,277 @@ +use cxx::UniquePtr; +use libwebrtc_sys::media_stream as sys_ms; +use libwebrtc_sys::MEDIA_TYPE_VIDEO; +use std::fmt::{Debug, Formatter}; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; + +pub use sys_ms::ffi::ContentHint; +pub use sys_ms::ffi::TrackState; + +use crate::video_frame::VideoFrame; + +pub trait MediaStreamTrackTrait { + fn kind(&self) -> String; + fn id(&self) -> String; + fn enabled(&self) -> bool; + fn set_enabled(&self, enabled: bool) -> bool; + fn state(&self) -> TrackState; +} + +pub enum MediaStreamTrack { + Audio(Arc), + Video(Arc), +} + +macro_rules! shared_getter { + ($x:ident, $ret:ty) => { + fn $x(&self) -> $ret { + match self { + Self::Video(inner) => inner.$x(), + Self::Audio(inner) => inner.$x(), + } + } + }; +} + +impl MediaStreamTrack { + pub(crate) fn new(cxx_handle: UniquePtr) -> Self { + unsafe { + if cxx_handle.kind() == MEDIA_TYPE_VIDEO { + Self::Video(Arc::new(VideoTrack::new(UniquePtr::from_raw( + sys_ms::ffi::media_to_video(cxx_handle.into_raw()) + as *mut sys_ms::ffi::VideoTrack, + )))) + } else { + Self::Audio(Arc::new(AudioTrack::new(UniquePtr::from_raw( + sys_ms::ffi::media_to_audio(cxx_handle.into_raw()) + as *mut sys_ms::ffi::AudioTrack, + )))) + } + } + } +} + +impl Debug for MediaStreamTrack { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + f.debug_struct("MediaStreamTrack") + .field("id", &self.id()) + .field("kind", &self.kind()) + .field("enabled", &self.enabled()) + .field("state", &self.state()) + .finish() + } +} + +impl MediaStreamTrackTrait for MediaStreamTrack { + shared_getter!(kind, String); + shared_getter!(id, String); + shared_getter!(enabled, bool); + shared_getter!(state, TrackState); + + fn set_enabled(&self, enabled: bool) -> bool { + match self { + Self::Video(inner) => inner.set_enabled(enabled), + Self::Audio(inner) => inner.set_enabled(enabled), + } + } +} + +pub struct AudioTrack { + cxx_handle: Mutex>, +} + +impl AudioTrack { + pub(crate) fn new(cxx_handle: UniquePtr) -> Self { + Self { + cxx_handle: Mutex::new(cxx_handle), + } + } +} + +pub struct VideoTrack { + cxx_handle: Mutex>, + observer: Box, + + // Keep alive for c++ + native_observer: UniquePtr, +} + +macro_rules! impl_media_stream_track_trait { + ($x:ty, $cast:ident) => { + impl MediaStreamTrackTrait for $x { + fn kind(&self) -> String { + unsafe { (*sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())).kind() } + } + + fn id(&self) -> String { + unsafe { (*sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())).id() } + } + + fn enabled(&self) -> bool { + unsafe { (*sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())).enabled() } + } + + fn set_enabled(&self, enabled: bool) -> bool { + unsafe { + let media = + sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap()) as *mut sys_ms::ffi::MediaStreamTrack; + + Pin::new_unchecked(&mut *media).set_enabled(enabled) + } + } + + fn state(&self) -> TrackState { + unsafe { (*sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())).state() } + } + } + }; +} + +impl_media_stream_track_trait!(VideoTrack, video_to_media); +impl_media_stream_track_trait!(AudioTrack, audio_to_media); + +pub type OnFrameHandler = Box; +pub type OnDiscardedFrameHandler = Box; +pub type OnConstraintsChanged = Box; + +#[derive(Default)] +struct InternalVideoTrackSink { + on_frame_handler: Mutex>, + on_discarded_frame_handler: Mutex>, + on_constraints_changed_handler: Mutex>, +} + +pub struct VideoTrackSourceConstraints { + pub min_fps: Option, + pub max_fps: Option, +} + +impl From for VideoTrackSourceConstraints { + fn from(cst: sys_ms::ffi::VideoTrackSourceConstraints) -> Self { + Self { + min_fps: (cst.min_fps != 1.0).then_some(cst.min_fps), + max_fps: (cst.max_fps != 1.0).then_some(cst.max_fps), + } + } +} + +impl sys_ms::VideoFrameSink for InternalVideoTrackSink { + fn on_frame(&self, frame: UniquePtr) { + if let Some(cb) = self.on_frame_handler.lock().unwrap().as_mut() { + cb(VideoFrame::new(frame)); + } + } + + fn on_discarded_frame(&self) { + if let Some(cb) = self.on_discarded_frame_handler.lock().unwrap().as_mut() { + cb(); + } + } + + fn on_constraints_changed(&self, constraints: sys_ms::ffi::VideoTrackSourceConstraints) { + if let Some(cb) = self.on_constraints_changed_handler.lock().unwrap().as_mut() { + cb(constraints.into()); + } + } +} + +impl VideoTrack { + fn new(cxx_handle: UniquePtr) -> Self { + let mut observer = Box::new(InternalVideoTrackSink::default()); + + let mut track = unsafe { + Self { + cxx_handle: Mutex::new(cxx_handle), + native_observer: sys_ms::ffi::create_native_video_frame_sink(Box::new( + sys_ms::VideoFrameSinkWrapper::new(&mut *observer), + )), + observer, + } + }; + + unsafe { + track + .cxx_handle + .lock() + .unwrap() + .pin_mut() + .add_sink(track.native_observer.pin_mut()); + } + + track + } + + pub fn set_should_receive(&mut self, should_receive: bool) { + + + + self.cxx_handle + .lock() + .unwrap() + .pin_mut() + .set_should_receive(should_receive) + } + + pub fn set_content_hint(&mut self, hint: ContentHint) { + self.cxx_handle + .lock() + .unwrap() + .pin_mut() + .set_content_hint(hint) + } + + pub fn should_receive(&self) -> bool { + self.cxx_handle.lock().unwrap().should_receive() + } + + pub fn content_hint(&self) -> ContentHint { + self.cxx_handle.lock().unwrap().content_hint() + } + + pub fn on_frame(&mut self, handler: OnFrameHandler) { + *self.observer.on_frame_handler.lock().unwrap() = Some(handler); + } + + pub fn on_discarded_frame(&mut self, handler: OnDiscardedFrameHandler) { + *self.observer.on_discarded_frame_handler.lock().unwrap() = Some(handler); + } + + pub fn on_constraints_changed(&mut self, handler: OnConstraintsChanged) { + *self.observer.on_constraints_changed_handler.lock().unwrap() = Some(handler); + } +} + +impl Drop for VideoTrack { + fn drop(&mut self) { + unsafe { + self.cxx_handle + .lock() + .unwrap() + .pin_mut() + .remove_sink(self.native_observer.pin_mut()); + } + } +} + +pub struct MediaStream { + cxx_handle: UniquePtr, +} + +impl Debug for MediaStream { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + f.debug_struct("MediaStream") + .field("id", &self.id()) + .finish() + } +} + +impl MediaStream { + pub(crate) fn new(cxx_handle: UniquePtr) -> Self { + Self { cxx_handle } + } + + pub fn id(&self) -> String { + self.cxx_handle.id() + } +} diff --git a/crates/livekit-webrtc/src/data_channel.rs b/crates/livekit-webrtc/src/data_channel.rs index a231ee3..284340f 100644 --- a/crates/livekit-webrtc/src/data_channel.rs +++ b/crates/livekit-webrtc/src/data_channel.rs @@ -109,13 +109,13 @@ impl Drop for DataChannel { pub type OnStateChangeHandler = Box; pub type OnMessageHandler = Box; -// data, is_binary pub type OnBufferedAmountChangeHandler = Box; +#[derive(Default)] struct InternalDataChannelObserver { - on_state_change_handler: Arc>>, - on_message_handler: Arc>>, - on_buffered_amount_change_handler: Arc>>, + on_state_change_handler: Mutex>, + on_message_handler: Mutex>, + on_buffered_amount_change_handler: Mutex>, } impl sys_dc::DataChannelObserver for InternalDataChannelObserver { @@ -144,16 +144,6 @@ impl sys_dc::DataChannelObserver for InternalDataChannelObserver { } } -impl Default for InternalDataChannelObserver { - fn default() -> Self { - Self { - on_state_change_handler: Arc::new(Default::default()), - on_message_handler: Arc::new(Default::default()), - on_buffered_amount_change_handler: Arc::new(Default::default()), - } - } -} - #[derive(Debug)] pub struct DataChannelInit { #[deprecated] diff --git a/crates/livekit-webrtc/src/jsep.rs b/crates/livekit-webrtc/src/jsep.rs index 228ba62..4638545 100644 --- a/crates/livekit-webrtc/src/jsep.rs +++ b/crates/livekit-webrtc/src/jsep.rs @@ -1,4 +1,4 @@ -use std::fmt::{Debug, Display, Formatter}; +use std::fmt::{Debug, Formatter}; use cxx::UniquePtr; diff --git a/crates/livekit-webrtc/src/lib.rs b/crates/livekit-webrtc/src/lib.rs index 4bcbcfc..684388d 100644 --- a/crates/livekit-webrtc/src/lib.rs +++ b/crates/livekit-webrtc/src/lib.rs @@ -6,4 +6,6 @@ pub mod peer_connection_factory; pub mod rtc_error; pub mod rtp_receiver; pub mod rtp_transceiver; +pub mod video_frame; +pub mod video_frame_buffer; pub mod webrtc; diff --git a/crates/livekit-webrtc/src/media_stream.rs b/crates/livekit-webrtc/src/media_stream.rs index 38bfed9..bb5981d 100644 --- a/crates/livekit-webrtc/src/media_stream.rs +++ b/crates/livekit-webrtc/src/media_stream.rs @@ -1,14 +1,59 @@ use cxx::UniquePtr; -use std::fmt::{Debug, Formatter}; - use libwebrtc_sys::media_stream as sys_ms; +use libwebrtc_sys::MEDIA_TYPE_VIDEO; +use std::fmt::{Debug, Formatter}; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; + +pub use sys_ms::ffi::ContentHint; pub use sys_ms::ffi::TrackState; -pub struct MediaStreamTrack { - cxx_handle: UniquePtr, +use crate::video_frame::VideoFrame; + +pub trait MediaStreamTrackTrait { + fn kind(&self) -> String; + fn id(&self) -> String; + fn enabled(&self) -> bool; + fn set_enabled(&self, enabled: bool) -> bool; + fn state(&self) -> TrackState; } -impl Debug for MediaStreamTrack { +#[derive(Clone)] +pub enum MediaStreamTrackHandle { + Audio(Arc), + Video(Arc), +} + +macro_rules! shared_getter { + ($x:ident, $ret:ty) => { + fn $x(&self) -> $ret { + match self { + Self::Video(inner) => inner.$x(), + Self::Audio(inner) => inner.$x(), + } + } + }; +} + +impl MediaStreamTrackHandle { + pub(crate) fn new(cxx_handle: UniquePtr) -> Self { + unsafe { + if cxx_handle.kind() == MEDIA_TYPE_VIDEO { + Self::Video(VideoTrack::new(UniquePtr::from_raw( + sys_ms::ffi::media_to_video(cxx_handle.into_raw()) + as *mut sys_ms::ffi::VideoTrack, + ))) + } else { + Self::Audio(AudioTrack::new(UniquePtr::from_raw( + sys_ms::ffi::media_to_audio(cxx_handle.into_raw()) + as *mut sys_ms::ffi::AudioTrack, + ))) + } + } + } +} + +impl Debug for MediaStreamTrackHandle { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { f.debug_struct("MediaStreamTrack") .field("id", &self.id()) @@ -19,29 +64,191 @@ impl Debug for MediaStreamTrack { } } -impl MediaStreamTrack { - pub(crate) fn new(cxx_handle: UniquePtr) -> Self { - Self { cxx_handle } +impl MediaStreamTrackTrait for MediaStreamTrackHandle { + shared_getter!(kind, String); + shared_getter!(id, String); + shared_getter!(enabled, bool); + shared_getter!(state, TrackState); + + fn set_enabled(&self, enabled: bool) -> bool { + match self { + Self::Video(inner) => inner.set_enabled(enabled), + Self::Audio(inner) => inner.set_enabled(enabled), + } + } +} + +pub struct AudioTrack { + cxx_handle: Mutex>, +} + +impl AudioTrack { + pub(crate) fn new(cxx_handle: UniquePtr) -> Arc { + Arc::new(Self { + cxx_handle: Mutex::new(cxx_handle), + }) + } +} + +pub struct VideoTrack { + cxx_handle: Mutex>, + observer: Box, + + // Keep alive for c++ + native_observer: UniquePtr, +} + +macro_rules! impl_media_stream_track_trait { + ($x:ty, $cast:ident) => { + impl MediaStreamTrackTrait for $x { + fn kind(&self) -> String { + unsafe { (*sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())).kind() } + } + + fn id(&self) -> String { + unsafe { (*sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())).id() } + } + + fn enabled(&self) -> bool { + unsafe { (*sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())).enabled() } + } + + fn set_enabled(&self, enabled: bool) -> bool { + unsafe { + let media = + sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap()) as *mut sys_ms::ffi::MediaStreamTrack; + + Pin::new_unchecked(&mut *media).set_enabled(enabled) + } + } + + fn state(&self) -> TrackState { + unsafe { (*sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())).state() } + } + } + }; +} + +impl_media_stream_track_trait!(VideoTrack, video_to_media); +impl_media_stream_track_trait!(AudioTrack, audio_to_media); + +pub type OnFrameHandler = Box; +pub type OnDiscardedFrameHandler = Box; +pub type OnConstraintsChanged = Box; + +#[derive(Default)] +struct InternalVideoTrackSink { + on_frame_handler: Mutex>, + on_discarded_frame_handler: Mutex>, + on_constraints_changed_handler: Mutex>, +} + +pub struct VideoTrackSourceConstraints { + pub min_fps: Option, + pub max_fps: Option, +} + +impl From for VideoTrackSourceConstraints { + fn from(cst: sys_ms::ffi::VideoTrackSourceConstraints) -> Self { + Self { + min_fps: (cst.min_fps != 1.0).then_some(cst.min_fps), + max_fps: (cst.max_fps != 1.0).then_some(cst.max_fps), + } + } +} + +impl sys_ms::VideoFrameSink for InternalVideoTrackSink { + fn on_frame(&self, frame: UniquePtr) { + if let Some(cb) = self.on_frame_handler.lock().unwrap().as_mut() { + cb(VideoFrame::new(frame)); + } } - fn kind(&self) -> String { - self.cxx_handle.kind() + fn on_discarded_frame(&self) { + if let Some(cb) = self.on_discarded_frame_handler.lock().unwrap().as_mut() { + cb(); + } } - fn id(&self) -> String { - self.cxx_handle.id() + fn on_constraints_changed(&self, constraints: sys_ms::ffi::VideoTrackSourceConstraints) { + if let Some(cb) = self.on_constraints_changed_handler.lock().unwrap().as_mut() { + cb(constraints.into()); + } + } +} + +impl VideoTrack { + fn new(cxx_handle: UniquePtr) -> Arc { + let mut observer = Box::new(InternalVideoTrackSink::default()); + + let mut track = unsafe { + Self { + cxx_handle: Mutex::new(cxx_handle), + native_observer: sys_ms::ffi::create_native_video_frame_sink(Box::new( + sys_ms::VideoFrameSinkWrapper::new(&mut *observer), + )), + observer, + } + }; + + unsafe { + track + .cxx_handle + .lock() + .unwrap() + .pin_mut() + .add_sink(track.native_observer.pin_mut()); + } + + Arc::new(track) } - fn enabled(&self) -> bool { - self.cxx_handle.enabled() + pub fn set_should_receive(&mut self, should_receive: bool) { + self.cxx_handle + .lock() + .unwrap() + .pin_mut() + .set_should_receive(should_receive) } - fn set_enabled(&mut self, enable: bool) -> bool { - self.cxx_handle.pin_mut().set_enabled(enable) + pub fn set_content_hint(&mut self, hint: ContentHint) { + self.cxx_handle + .lock() + .unwrap() + .pin_mut() + .set_content_hint(hint) } - fn state(&self) -> TrackState { - self.cxx_handle.state() + pub fn should_receive(&self) -> bool { + self.cxx_handle.lock().unwrap().should_receive() + } + + pub fn content_hint(&self) -> ContentHint { + self.cxx_handle.lock().unwrap().content_hint() + } + + pub fn on_frame(&mut self, handler: OnFrameHandler) { + *self.observer.on_frame_handler.lock().unwrap() = Some(handler); + } + + pub fn on_discarded_frame(&mut self, handler: OnDiscardedFrameHandler) { + *self.observer.on_discarded_frame_handler.lock().unwrap() = Some(handler); + } + + pub fn on_constraints_changed(&mut self, handler: OnConstraintsChanged) { + *self.observer.on_constraints_changed_handler.lock().unwrap() = Some(handler); + } +} + +impl Drop for VideoTrack { + fn drop(&mut self) { + unsafe { + self.cxx_handle + .lock() + .unwrap() + .pin_mut() + .remove_sink(self.native_observer.pin_mut()); + } } } diff --git a/crates/livekit-webrtc/src/peer_connection.rs b/crates/livekit-webrtc/src/peer_connection.rs index b95005a..52fa6c0 100644 --- a/crates/livekit-webrtc/src/peer_connection.rs +++ b/crates/livekit-webrtc/src/peer_connection.rs @@ -18,7 +18,7 @@ pub use libwebrtc_sys::peer_connection::ffi::SignalingState; use crate::data_channel::{DataChannel, DataChannelInit}; use crate::jsep::{IceCandidate, SessionDescription}; -use crate::media_stream::MediaStream; +use crate::media_stream::{MediaStream, VideoTrack, AudioTrack}; use crate::rtc_error::RTCError; use crate::rtp_receiver::RtpReceiver; use crate::rtp_transceiver::RtpTransceiver; diff --git a/crates/livekit-webrtc/src/rtp_receiver.rs b/crates/livekit-webrtc/src/rtp_receiver.rs index 6a4ea5b..c35e1e2 100644 --- a/crates/livekit-webrtc/src/rtp_receiver.rs +++ b/crates/livekit-webrtc/src/rtp_receiver.rs @@ -1,4 +1,4 @@ -use crate::media_stream::MediaStreamTrack; +use crate::media_stream::{MediaStreamTrackHandle, MediaStreamTrackTrait}; use cxx::UniquePtr; use libwebrtc_sys::rtp_receiver as sys_rec; use std::fmt::{Debug, Formatter}; @@ -17,10 +17,12 @@ impl Debug for RtpReceiver { impl RtpReceiver { pub(crate) fn new(cxx_handle: UniquePtr) -> Self { - Self { cxx_handle } + Self { + cxx_handle, + } } - pub fn track(&self) -> MediaStreamTrack { - MediaStreamTrack::new(self.cxx_handle.track()) + pub fn track(&self) -> MediaStreamTrackHandle { + MediaStreamTrackHandle::new(self.cxx_handle.track()) } } diff --git a/crates/livekit-webrtc/src/video_frame.rs b/crates/livekit-webrtc/src/video_frame.rs new file mode 100644 index 0000000..4926525 --- /dev/null +++ b/crates/livekit-webrtc/src/video_frame.rs @@ -0,0 +1,50 @@ +use cxx::UniquePtr; +use libwebrtc_sys::video_frame as vf_sys; + +pub use vf_sys::ffi::VideoRotation; + +pub struct VideoFrame { + cxx_handle: UniquePtr, +} + +impl VideoFrame { + pub(crate) fn new(cxx_handle: UniquePtr) -> Self { + Self { cxx_handle } + } + + pub fn width(&self) -> i32 { + self.cxx_handle.width() + } + + pub fn height(&self) -> i32 { + self.cxx_handle.height() + } + + pub fn size(&self) -> u32 { + self.cxx_handle.size() + } + + pub fn id(&self) -> u16 { + self.cxx_handle.id() + } + + pub fn timestamp_us(&self) -> i64 { + self.cxx_handle.timestamp_us() + } + + pub fn ntp_time_ms(&self) -> i64 { + self.cxx_handle.ntp_time_ms() + } + + pub fn transport_frame_id(&self) -> u32 { + self.cxx_handle.transport_frame_id() + } + + pub fn timestamp(&self) -> u32 { + self.cxx_handle.timestamp() + } + + pub fn rotation(&self) -> VideoRotation { + self.cxx_handle.rotation() + } +} diff --git a/crates/livekit-webrtc/src/video_frame_buffer.rs b/crates/livekit-webrtc/src/video_frame_buffer.rs new file mode 100644 index 0000000..9c60b85 --- /dev/null +++ b/crates/livekit-webrtc/src/video_frame_buffer.rs @@ -0,0 +1,18 @@ +use cxx::UniquePtr; +use libwebrtc_sys::video_frame_buffer as vfb_sys; + +pub use vfb_sys::ffi::VideoFrameBufferType; + +pub struct VideoFrameBuffer { + cxx_handle: UniquePtr, +} + +impl VideoFrameBuffer { + pub fn new(cxx_handle: UniquePtr) -> Self { + Self { cxx_handle } + } + + pub fn buffer_type(&self) -> VideoFrameBufferType { + self.cxx_handle.buffer_type() + } +} diff --git a/examples/simple_room/src/main.rs b/examples/simple_room/src/main.rs index 52b161c..3997612 100644 --- a/examples/simple_room/src/main.rs +++ b/examples/simple_room/src/main.rs @@ -1,6 +1,7 @@ -use livekit::room::Room; +use livekit::room::track::TrackTrait; +use livekit::room::{track::remote_track::RemoteTrackHandle, Room}; use std::sync::{Arc, Mutex}; -use tracing::{info, trace}; +use tracing::{event_enabled, info, trace}; const URL: &str = "ws://localhost:7880"; const TOKEN : &str = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjIzODQ4MDY0NzMsImlzcyI6IkFQSXpLYkFTaUNWYWtnSiIsIm5hbWUiOiJuYXRpdmUiLCJuYmYiOjE2NjQ4MDY0NzMsInN1YiI6Im5hdGl2ZSIsInZpZGVvIjp7InJvb21DcmVhdGUiOnRydWUsInJvb21Kb2luIjp0cnVlfX0.BgVdBnq3XFD3_BQHoe1azqjifYysubgFl6Qlzu9IQGI"; @@ -12,8 +13,19 @@ async fn main() { tracing_subscriber::fmt::init(); let room = Room::new(); - room.on_participant_connected(async |participant| { + room.events() + .on_participant_connected(|event| async move {}); + + room.events().on_track_subscribed(|event| async move { + let track = event.publication.track().unwrap(); + if let RemoteTrackHandle::Video(video_track) = track { + let rtc_track = video_track.rtc_track(); + rtc_track.on_frame(Box::new(|frame| { Box::pin(async move { + + - }) + }) })) + } + }); } From a415748f061efe9527cdb160cd87b5740f233dc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Tue, 22 Nov 2022 18:02:07 +0100 Subject: [PATCH 5/8] Delete .media_stream.rs.rustfmt --- .../src/.media_stream.rs.rustfmt | 277 ------------------ 1 file changed, 277 deletions(-) delete mode 100644 crates/livekit-webrtc/src/.media_stream.rs.rustfmt diff --git a/crates/livekit-webrtc/src/.media_stream.rs.rustfmt b/crates/livekit-webrtc/src/.media_stream.rs.rustfmt deleted file mode 100644 index c5641bd..0000000 --- a/crates/livekit-webrtc/src/.media_stream.rs.rustfmt +++ /dev/null @@ -1,277 +0,0 @@ -use cxx::UniquePtr; -use libwebrtc_sys::media_stream as sys_ms; -use libwebrtc_sys::MEDIA_TYPE_VIDEO; -use std::fmt::{Debug, Formatter}; -use std::pin::Pin; -use std::sync::{Arc, Mutex}; - -pub use sys_ms::ffi::ContentHint; -pub use sys_ms::ffi::TrackState; - -use crate::video_frame::VideoFrame; - -pub trait MediaStreamTrackTrait { - fn kind(&self) -> String; - fn id(&self) -> String; - fn enabled(&self) -> bool; - fn set_enabled(&self, enabled: bool) -> bool; - fn state(&self) -> TrackState; -} - -pub enum MediaStreamTrack { - Audio(Arc), - Video(Arc), -} - -macro_rules! shared_getter { - ($x:ident, $ret:ty) => { - fn $x(&self) -> $ret { - match self { - Self::Video(inner) => inner.$x(), - Self::Audio(inner) => inner.$x(), - } - } - }; -} - -impl MediaStreamTrack { - pub(crate) fn new(cxx_handle: UniquePtr) -> Self { - unsafe { - if cxx_handle.kind() == MEDIA_TYPE_VIDEO { - Self::Video(Arc::new(VideoTrack::new(UniquePtr::from_raw( - sys_ms::ffi::media_to_video(cxx_handle.into_raw()) - as *mut sys_ms::ffi::VideoTrack, - )))) - } else { - Self::Audio(Arc::new(AudioTrack::new(UniquePtr::from_raw( - sys_ms::ffi::media_to_audio(cxx_handle.into_raw()) - as *mut sys_ms::ffi::AudioTrack, - )))) - } - } - } -} - -impl Debug for MediaStreamTrack { - fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - f.debug_struct("MediaStreamTrack") - .field("id", &self.id()) - .field("kind", &self.kind()) - .field("enabled", &self.enabled()) - .field("state", &self.state()) - .finish() - } -} - -impl MediaStreamTrackTrait for MediaStreamTrack { - shared_getter!(kind, String); - shared_getter!(id, String); - shared_getter!(enabled, bool); - shared_getter!(state, TrackState); - - fn set_enabled(&self, enabled: bool) -> bool { - match self { - Self::Video(inner) => inner.set_enabled(enabled), - Self::Audio(inner) => inner.set_enabled(enabled), - } - } -} - -pub struct AudioTrack { - cxx_handle: Mutex>, -} - -impl AudioTrack { - pub(crate) fn new(cxx_handle: UniquePtr) -> Self { - Self { - cxx_handle: Mutex::new(cxx_handle), - } - } -} - -pub struct VideoTrack { - cxx_handle: Mutex>, - observer: Box, - - // Keep alive for c++ - native_observer: UniquePtr, -} - -macro_rules! impl_media_stream_track_trait { - ($x:ty, $cast:ident) => { - impl MediaStreamTrackTrait for $x { - fn kind(&self) -> String { - unsafe { (*sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())).kind() } - } - - fn id(&self) -> String { - unsafe { (*sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())).id() } - } - - fn enabled(&self) -> bool { - unsafe { (*sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())).enabled() } - } - - fn set_enabled(&self, enabled: bool) -> bool { - unsafe { - let media = - sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap()) as *mut sys_ms::ffi::MediaStreamTrack; - - Pin::new_unchecked(&mut *media).set_enabled(enabled) - } - } - - fn state(&self) -> TrackState { - unsafe { (*sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())).state() } - } - } - }; -} - -impl_media_stream_track_trait!(VideoTrack, video_to_media); -impl_media_stream_track_trait!(AudioTrack, audio_to_media); - -pub type OnFrameHandler = Box; -pub type OnDiscardedFrameHandler = Box; -pub type OnConstraintsChanged = Box; - -#[derive(Default)] -struct InternalVideoTrackSink { - on_frame_handler: Mutex>, - on_discarded_frame_handler: Mutex>, - on_constraints_changed_handler: Mutex>, -} - -pub struct VideoTrackSourceConstraints { - pub min_fps: Option, - pub max_fps: Option, -} - -impl From for VideoTrackSourceConstraints { - fn from(cst: sys_ms::ffi::VideoTrackSourceConstraints) -> Self { - Self { - min_fps: (cst.min_fps != 1.0).then_some(cst.min_fps), - max_fps: (cst.max_fps != 1.0).then_some(cst.max_fps), - } - } -} - -impl sys_ms::VideoFrameSink for InternalVideoTrackSink { - fn on_frame(&self, frame: UniquePtr) { - if let Some(cb) = self.on_frame_handler.lock().unwrap().as_mut() { - cb(VideoFrame::new(frame)); - } - } - - fn on_discarded_frame(&self) { - if let Some(cb) = self.on_discarded_frame_handler.lock().unwrap().as_mut() { - cb(); - } - } - - fn on_constraints_changed(&self, constraints: sys_ms::ffi::VideoTrackSourceConstraints) { - if let Some(cb) = self.on_constraints_changed_handler.lock().unwrap().as_mut() { - cb(constraints.into()); - } - } -} - -impl VideoTrack { - fn new(cxx_handle: UniquePtr) -> Self { - let mut observer = Box::new(InternalVideoTrackSink::default()); - - let mut track = unsafe { - Self { - cxx_handle: Mutex::new(cxx_handle), - native_observer: sys_ms::ffi::create_native_video_frame_sink(Box::new( - sys_ms::VideoFrameSinkWrapper::new(&mut *observer), - )), - observer, - } - }; - - unsafe { - track - .cxx_handle - .lock() - .unwrap() - .pin_mut() - .add_sink(track.native_observer.pin_mut()); - } - - track - } - - pub fn set_should_receive(&mut self, should_receive: bool) { - - - - self.cxx_handle - .lock() - .unwrap() - .pin_mut() - .set_should_receive(should_receive) - } - - pub fn set_content_hint(&mut self, hint: ContentHint) { - self.cxx_handle - .lock() - .unwrap() - .pin_mut() - .set_content_hint(hint) - } - - pub fn should_receive(&self) -> bool { - self.cxx_handle.lock().unwrap().should_receive() - } - - pub fn content_hint(&self) -> ContentHint { - self.cxx_handle.lock().unwrap().content_hint() - } - - pub fn on_frame(&mut self, handler: OnFrameHandler) { - *self.observer.on_frame_handler.lock().unwrap() = Some(handler); - } - - pub fn on_discarded_frame(&mut self, handler: OnDiscardedFrameHandler) { - *self.observer.on_discarded_frame_handler.lock().unwrap() = Some(handler); - } - - pub fn on_constraints_changed(&mut self, handler: OnConstraintsChanged) { - *self.observer.on_constraints_changed_handler.lock().unwrap() = Some(handler); - } -} - -impl Drop for VideoTrack { - fn drop(&mut self) { - unsafe { - self.cxx_handle - .lock() - .unwrap() - .pin_mut() - .remove_sink(self.native_observer.pin_mut()); - } - } -} - -pub struct MediaStream { - cxx_handle: UniquePtr, -} - -impl Debug for MediaStream { - fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - f.debug_struct("MediaStream") - .field("id", &self.id()) - .finish() - } -} - -impl MediaStream { - pub(crate) fn new(cxx_handle: UniquePtr) -> Self { - Self { cxx_handle } - } - - pub fn id(&self) -> String { - self.cxx_handle.id() - } -} From 39fa798c7f1d2b1bca245e7235e12834f276c357 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Mon, 28 Nov 2022 15:52:37 +0100 Subject: [PATCH 6/8] trying to receive VideoFrame, progress... --- crates/livekit-core/Cargo.toml | 2 +- crates/livekit-core/src/events.rs | 2 +- crates/livekit-core/src/room/mod.rs | 73 +++-- .../src/room/participant/local_participant.rs | 12 +- .../livekit-core/src/room/participant/mod.rs | 22 +- .../room/participant/remote_participant.rs | 66 ++++- .../livekit-core/src/room/publication/mod.rs | 97 +++++-- crates/livekit-core/src/room/track/mod.rs | 27 +- crates/livekit-core/src/rtc_engine/mod.rs | 17 +- .../include/livekit/media_stream.h | 8 +- .../include/livekit/video_frame.h | 4 +- .../include/livekit/video_frame_buffer.h | 32 ++- .../libwebrtc-sys/src/media_stream.cpp | 4 + .../libwebrtc-sys/src/video_frame.rs | 2 +- .../libwebrtc-sys/src/video_frame_buffer.rs | 15 +- crates/livekit-webrtc/src/media_stream.rs | 21 +- crates/livekit-webrtc/src/peer_connection.rs | 8 +- crates/livekit-webrtc/src/video_frame.rs | 12 +- .../livekit-webrtc/src/video_frame_buffer.rs | 250 +++++++++++++++++- examples/simple_room/src/main.rs | 29 +- 20 files changed, 575 insertions(+), 128 deletions(-) diff --git a/crates/livekit-core/Cargo.toml b/crates/livekit-core/Cargo.toml index a9ce7cd..3cf11f7 100644 --- a/crates/livekit-core/Cargo.toml +++ b/crates/livekit-core/Cargo.toml @@ -21,4 +21,4 @@ lazy_static = "1.4.0" tracing = "0.1" [build-dependencies] -prost-build = { version = "0.11.1" } \ No newline at end of file +prost-build = { version = "0.11.1" } diff --git a/crates/livekit-core/src/events.rs b/crates/livekit-core/src/events.rs index 5dfecd7..c85506f 100644 --- a/crates/livekit-core/src/events.rs +++ b/crates/livekit-core/src/events.rs @@ -8,7 +8,7 @@ macro_rules! event_setter { pub fn $fnc(&self, mut callback: F) where F: FnMut($event) -> Fut + Send + Sync + 'static, - Fut: Future + Send + Sync + 'static, + Fut: Future + Send + 'static, { *self.$fnc.lock() = Some(Box::new(move |event| Box::pin(callback(event)))); } diff --git a/crates/livekit-core/src/room/mod.rs b/crates/livekit-core/src/room/mod.rs index c293d29..f9305ea 100644 --- a/crates/livekit-core/src/room/mod.rs +++ b/crates/livekit-core/src/room/mod.rs @@ -9,11 +9,13 @@ use self::participant::local_participant::LocalParticipant; use self::participant::remote_participant::RemoteParticipant; use self::participant::ParticipantInternalTrait; use self::participant::ParticipantTrait; -use crate::events::room::{ParticipantConnectedEvent, ParticipantDisconnectedEvent, RoomEvents}; +use crate::events::room::{ + ParticipantConnectedEvent, ParticipantDisconnectedEvent, RoomEvents, TrackSubscribedEvent, +}; use crate::proto; use crate::proto::participant_info; use thiserror::Error; -use tracing::error; +use tracing::{debug, error}; use crate::rtc_engine::{EngineError, EngineEvent, EngineEvents, RTCEngine}; use crate::signal_client::SignalOptions; @@ -63,10 +65,6 @@ impl Room { } } - pub fn events(&self) -> Arc { - self.events.clone() - } - pub async fn connect(&mut self, url: &str, token: &str) -> RoomResult<()> { let (rtc_engine, engine_events) = RTCEngine::connect(url, token, SignalOptions::default()).await?; @@ -88,11 +86,21 @@ impl Room { self.inner = Some(inner.clone()); + // Add already connected participants + for pi in join_response.other_participants { + let p = Self::create_participant(inner.clone(), self.events.clone(), pi.clone()); + p.update_info(pi).await; + } + tokio::spawn(Self::room_task(inner, self.events.clone(), engine_events)); Ok(()) } + pub fn events(&self) -> Arc { + self.events.clone() + } + pub fn get_handle(&self) -> Option { self.inner.as_ref().map(|inner| RoomHandle { inner: inner.clone(), @@ -121,6 +129,7 @@ impl Room { match event { EngineEvent::ParticipantUpdate(update) => { Self::handle_participant_update(room_inner.clone(), room_events.clone(), update) + .await } EngineEvent::AddTrack { rtp_receiver, @@ -164,7 +173,7 @@ impl Room { Ok(()) } - fn handle_participant_update( + async fn handle_participant_update( room_inner: Arc, room_events: Arc, update: proto::ParticipantUpdate, @@ -173,7 +182,7 @@ impl Room { if pi.sid == room_inner.local_participant.sid() || pi.identity == room_inner.local_participant.identity() { - room_inner.local_participant.update_info(pi); + room_inner.local_participant.clone().update_info(pi).await; continue; } @@ -190,12 +199,12 @@ impl Room { ) } else { // Participant is already connected, update the informations - remote_participant.update_info(pi); + remote_participant.update_info(pi).await; } } else { // Create a new participant and call OnConnect event let remote_participant = - Self::get_or_create_participant(room_inner.clone(), room_events.clone(), pi); + Self::create_participant(room_inner.clone(), room_events.clone(), pi); let mut handler = room_events.on_participant_connected.lock(); if let Some(cb) = handler.as_mut() { cb(ParticipantConnectedEvent { @@ -235,29 +244,41 @@ impl Room { room_inner.participants.read().get(sid).cloned() } - fn get_or_create_participant( + fn create_participant( room_inner: Arc, room_events: Arc, pi: proto::ParticipantInfo, ) -> Arc { - let participants = room_inner.participants.upgradable_read(); - let sid = pi.sid.clone().into(); - if let Some(p) = participants.get(&sid) { - p.update_info(pi); - p.clone() - } else { - let mut participants = RwLockUpgradableReadGuard::upgrade(participants); - let p = Arc::new(RemoteParticipant::new(pi)); + let p = Arc::new(RemoteParticipant::new(pi.clone())); - // Forward participantevents to room events - p.internal_events().on_track_published({ + // Forward participantevents to room events + p.internal_events().on_track_subscribed({ + let room_events = room_events.clone(); + let room_inner = room_inner.clone(); + + move |event| { let room_events = room_events.clone(); - |event| async move {} - }); + let room_inner = room_inner.clone(); - participants.insert(sid, p.clone()); - p - } + async move { + if let Some(cb) = room_events.clone().on_track_subscribed.lock().as_mut() { + cb(TrackSubscribedEvent { + room_handle: RoomHandle::from(room_inner.clone()), + track: event.track, + participant: event.participant, + publication: event.publication, + }) + .await; + } + } + } + }); + + room_inner + .participants + .write() + .insert(pi.sid.into(), p.clone()); + p } } diff --git a/crates/livekit-core/src/room/participant/local_participant.rs b/crates/livekit-core/src/room/participant/local_participant.rs index ebc877f..7927e21 100644 --- a/crates/livekit-core/src/room/participant/local_participant.rs +++ b/crates/livekit-core/src/room/participant/local_participant.rs @@ -1,5 +1,5 @@ use crate::proto::{data_packet, DataPacket, UserPacket}; -use crate::room::participant::{impl_participant_trait, ParticipantShared}; +use crate::room::participant::{impl_participant_trait, ParticipantShared, ParticipantInternalTrait}; use crate::room::RoomError; use crate::rtc_engine::RTCEngine; @@ -40,6 +40,16 @@ impl LocalParticipant { .await .map_err(Into::into) } + + pub(crate) async fn update_info(self: Arc, info: ParticipantInfo) { + self.shared.update_info(info); + } +} + +impl ParticipantInternalTrait for LocalParticipant { + fn internal_events(&self) -> Arc { + self.shared.internal_events.clone() + } } impl_participant_trait!(LocalParticipant); diff --git a/crates/livekit-core/src/room/participant/mod.rs b/crates/livekit-core/src/room/participant/mod.rs index 9158452..56c7da2 100644 --- a/crates/livekit-core/src/room/participant/mod.rs +++ b/crates/livekit-core/src/room/participant/mod.rs @@ -65,7 +65,6 @@ pub trait ParticipantTrait { fn identity(&self) -> ParticipantIdentity; fn name(&self) -> String; fn metadata(&self) -> String; - fn update_info(&self, info: ParticipantInfo); } #[derive(Clone)] @@ -74,6 +73,16 @@ pub enum ParticipantHandle { Remote(Arc), } +impl ParticipantHandle { + // TODO(theomonnom): Add async support to wrap_variants ... + pub(crate) async fn update_info(&self, info: ParticipantInfo) { + match self { + Self::Local(inner) => inner.clone().update_info(info).await, + Self::Remote(inner) => inner.clone().update_info(info).await, + } + } +} + impl ParticipantInternalTrait for ParticipantHandle { wrap_variants!( [Local, Remote] @@ -89,7 +98,6 @@ impl ParticipantTrait for ParticipantHandle { fnc!(identity, ParticipantIdentity, []); fnc!(name, String, []); fnc!(metadata, String, []); - fnc!(update_info, (), [info: ParticipantInfo]); ); } @@ -100,12 +108,6 @@ macro_rules! impl_participant_trait { use crate::room::id::{ParticipantIdentity, ParticipantSid}; use std::sync::Arc; - impl crate::room::participant::ParticipantInternalTrait for $x { - fn internal_events(&self) -> Arc { - self.shared.internal_events.clone() - } - } - impl crate::room::participant::ParticipantTrait for $x { fn events(&self) -> Arc { self.shared.events.clone() @@ -126,10 +128,6 @@ macro_rules! impl_participant_trait { fn metadata(&self) -> String { self.shared.metadata.lock().clone() } - - fn update_info(&self, info: ParticipantInfo) { - self.shared.update_info(info); - } } }; } diff --git a/crates/livekit-core/src/room/participant/remote_participant.rs b/crates/livekit-core/src/room/participant/remote_participant.rs index 1266504..12efbae 100644 --- a/crates/livekit-core/src/room/participant/remote_participant.rs +++ b/crates/livekit-core/src/room/participant/remote_participant.rs @@ -1,16 +1,25 @@ -use crate::events::participant::{TrackSubscribedEvent, TrackSubscriptionFailedEvent}; +use crate::events::participant::{ + TrackPublishedEvent, TrackSubscribedEvent, TrackSubscriptionFailedEvent, +}; use crate::events::TrackError; use crate::room::id::TrackSid; -use crate::room::participant::{impl_participant_trait, ParticipantShared}; -use crate::room::publication::{RemoteTrackPublication, TrackPublication, TrackPublicationTrait}; +use crate::room::participant::{ + impl_participant_trait, ParticipantInternalTrait, ParticipantShared, +}; +use crate::room::publication::{ + RemoteTrackPublication, TrackPublication, TrackPublicationInternalTrait, TrackPublicationTrait, +}; use crate::room::track::remote_audio_track::RemoteAudioTrack; use crate::room::track::remote_track::RemoteTrackHandle; use crate::room::track::remote_video_track::RemoteVideoTrack; -use crate::room::track::{TrackTrait, TrackKind}; +use crate::room::track::{TrackKind, TrackTrait, TrackHandle}; use livekit_webrtc::media_stream::MediaStreamTrackHandle; +use std::collections::HashSet; use std::time::Duration; use tokio::time::{sleep, timeout}; -use tracing::error; +use tracing::{info, error}; + +use super::ParticipantTrait; const ADD_TRACK_TIMEOUT: Duration = Duration::from_secs(5); @@ -80,6 +89,9 @@ impl RemoteParticipant { _ => unreachable!(), }; + info!("starting track: {:?}", sid); + + remote_publication.update_track(Some(track.clone().into())); self.shared .add_track_publication(TrackPublication::Remote(remote_publication.clone())); track.start(); @@ -144,6 +156,50 @@ impl RemoteParticipant { } }) } + + pub(crate) async fn update_info(self: Arc, info: ParticipantInfo) { + self.shared.update_info(info.clone()); + + let mut valid_tracks = HashSet::::new(); + + for track in info.tracks { + if let Some(publication) = self.get_track_publication(&track.sid.clone().into()) { + publication.update_info(track.clone()); + } else { + let publication = RemoteTrackPublication::new(track.clone(), self.sid(), None); + self.shared + .add_track_publication(TrackPublication::Remote(publication.clone())); + + // This is a new track, fire publish events + let event = TrackPublishedEvent { + participant: self.clone(), + publication: publication.clone(), + }; + + if let Some(cb) = self + .shared + .internal_events + .on_track_published + .lock() + .as_mut() + { + cb(event.clone()).await; + } + + if let Some(cb) = self.shared.events.on_track_published.lock().as_mut() { + cb(event).await; + } + } + + valid_tracks.insert(track.sid.into()); + } + } +} + +impl ParticipantInternalTrait for RemoteParticipant { + fn internal_events(&self) -> Arc { + self.shared.internal_events.clone() + } } impl_participant_trait!(RemoteParticipant); diff --git a/crates/livekit-core/src/room/publication/mod.rs b/crates/livekit-core/src/room/publication/mod.rs index 5a36bbb..1820e88 100644 --- a/crates/livekit-core/src/room/publication/mod.rs +++ b/crates/livekit-core/src/room/publication/mod.rs @@ -1,11 +1,22 @@ +use crate::proto::TrackType; +use crate::proto::{TrackInfo, TrackSource as ProtoTrackSource}; +use crate::room::id::ParticipantSid; use crate::room::id::TrackSid; use crate::room::track::local_track::LocalTrackHandle; use crate::room::track::remote_track::RemoteTrackHandle; use crate::room::track::{TrackHandle, TrackKind, TrackSource}; +use crate::utils::wrap_variants; use parking_lot::Mutex; use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; use std::sync::Arc; +use super::track::TrackDimension; + +pub(crate) trait TrackPublicationInternalTrait { + fn update_track(&self, track: Option); + fn update_info(&self, info: TrackInfo); +} + pub trait TrackPublicationTrait { fn name(&self) -> String; fn sid(&self) -> TrackSid; @@ -21,6 +32,43 @@ pub(super) struct TrackPublicationShared { pub(super) kind: AtomicU8, // Casted to TrackKind pub(super) source: AtomicU8, // Casted to TrackSource pub(super) simulcasted: AtomicBool, + pub(super) dimension: Mutex, + pub(super) mime_type: Mutex, + pub(super) participant: ParticipantSid, // TODO(theomonnom) Use WeakParticipant instead +} + +impl TrackPublicationShared { + pub fn new(info: TrackInfo, participant: ParticipantSid, track: Option) -> Arc { + Arc::new(Self { + track: Mutex::new(track), + name: Mutex::new(info.name), + sid: Mutex::new(info.sid.into()), + kind: AtomicU8::new(TrackKind::from(TrackType::from_i32(info.r#type).unwrap()) as u8), + source: AtomicU8::new(TrackSource::from( + ProtoTrackSource::from_i32(info.source).unwrap(), + ) as u8), + simulcasted: AtomicBool::new(info.simulcast), + dimension: Mutex::new(TrackDimension(info.width, info.height)), + mime_type: Mutex::new(info.mime_type), + participant, + }) + } + + pub fn update_info(&self, info: TrackInfo) { + *self.name.lock() = info.name; + *self.sid.lock() = info.sid.into(); + self.kind.store( + TrackKind::from(TrackType::from_i32(info.r#type).unwrap()) as u8, + Ordering::SeqCst, + ); + self.source.store( + TrackSource::from(ProtoTrackSource::from_i32(info.source).unwrap()) as u8, + Ordering::SeqCst, + ); + self.simulcasted.store(info.simulcast, Ordering::SeqCst); + *self.dimension.lock() = TrackDimension(info.width, info.height); + *self.mime_type.lock() = info.mime_type; + } } #[derive(Clone)] @@ -29,19 +77,9 @@ pub enum TrackPublication { Remote(RemoteTrackPublication), } -macro_rules! shared_getter { - ($x:ident, $ret:ty) => { - fn $x(&self) -> $ret { - match self { - TrackPublication::Local(p) => p.$x(), - TrackPublication::Remote(p) => p.$x(), - } - } - }; -} - impl TrackPublication { pub fn track(&self) -> Option { + // Not calling Local/Remote function here, we don't need "cast" match self { TrackPublication::Local(p) => p.shared.track.lock().clone(), TrackPublication::Remote(p) => p.shared.track.lock().clone(), @@ -49,16 +87,37 @@ impl TrackPublication { } } +impl TrackPublicationInternalTrait for TrackPublication { + wrap_variants!( + [Local, Remote] + fnc!(update_track, (), [track: Option]); + fnc!(update_info, (), [info: TrackInfo]); + ); +} + impl TrackPublicationTrait for TrackPublication { - shared_getter!(name, String); - shared_getter!(sid, TrackSid); - shared_getter!(kind, TrackKind); - shared_getter!(source, TrackSource); - shared_getter!(simulcasted, bool); + wrap_variants!( + [Local, Remote] + fnc!(sid, TrackSid, []); + fnc!(name, String, []); + fnc!(kind, TrackKind, []); + fnc!(source, TrackSource, []); + fnc!(simulcasted, bool, []); + ); } macro_rules! impl_publication_trait { ($x:ident) => { + impl TrackPublicationInternalTrait for $x { + fn update_track(&self, track: Option) { + *self.shared.track.lock() = track; + } + + fn update_info(&self, info: TrackInfo) { + self.shared.update_info(info); + } + } + impl TrackPublicationTrait for $x { fn name(&self) -> String { self.shared.name.lock().clone() @@ -104,6 +163,12 @@ pub struct RemoteTrackPublication { } impl RemoteTrackPublication { + pub fn new(info: TrackInfo, participant: ParticipantSid, track: Option) -> Self { + Self { + shared: TrackPublicationShared::new(info, participant, track), + } + } + pub fn track(&self) -> Option { self.shared .track diff --git a/crates/livekit-core/src/room/track/mod.rs b/crates/livekit-core/src/room/track/mod.rs index ff53f4d..327084f 100644 --- a/crates/livekit-core/src/room/track/mod.rs +++ b/crates/livekit-core/src/room/track/mod.rs @@ -1,3 +1,4 @@ +use crate::proto::{TrackSource as ProtoTrackSource, TrackType}; use crate::room::id::TrackSid; use crate::room::track::local_audio_track::LocalAudioTrack; use crate::room::track::local_video_track::LocalVideoTrack; @@ -36,6 +37,16 @@ impl From for TrackKind { } } +impl From for TrackKind { + fn from(r#type: TrackType) -> Self { + match r#type { + TrackType::Audio => Self::Audio, + TrackType::Video => Self::Video, + _ => Self::Unknown, + } + } +} + #[derive(Debug)] pub enum StreamState { Unknown, @@ -74,6 +85,20 @@ impl From for TrackSource { } } +impl From for TrackSource { + fn from(source: ProtoTrackSource) -> Self { + match source { + ProtoTrackSource::Camera => Self::Camera, + ProtoTrackSource::Microphone => Self::Microphone, + ProtoTrackSource::ScreenShare => Self::Screenshare, + ProtoTrackSource::ScreenShareAudio => Self::ScreenshareAudio, + ProtoTrackSource::Unknown => Self::Unknown, + } + } +} + +pub struct TrackDimension(pub u32, pub u32); + pub trait TrackTrait { fn sid(&self) -> TrackSid; fn name(&self) -> String; @@ -103,7 +128,7 @@ impl TrackShared { name: Mutex::new(name), kind: AtomicU8::new(kind as u8), stream_state: AtomicU8::new(StreamState::Active as u8), - rtc_track: rtc_track, + rtc_track, } } diff --git a/crates/livekit-core/src/rtc_engine/mod.rs b/crates/livekit-core/src/rtc_engine/mod.rs index 1410b59..4fbb08a 100644 --- a/crates/livekit-core/src/rtc_engine/mod.rs +++ b/crates/livekit-core/src/rtc_engine/mod.rs @@ -10,7 +10,7 @@ use prost::Message; use serde::{Deserialize, Serialize}; use thiserror::Error; use tokio::time::sleep; -use tracing::{debug, error, trace}; +use tracing::{debug, error, info, trace}; use crate::{proto, signal_client}; use livekit_webrtc::data_channel::{DataChannel, DataChannelInit, DataSendError, DataState}; @@ -280,6 +280,7 @@ impl RTCEngine { } RTCEvent::ConnectionChange { state, target } => { // Reconnect if we've been disconnected unexpectedly + trace!("Connection change, {:?} {:?}", state, target); let subscriber_primary = engine_inner.join_response.lock().subscriber_primary; let is_primary = subscriber_primary && target == SignalTarget::Subscriber; @@ -328,10 +329,12 @@ impl RTCEngine { target, } => { if target == SignalTarget::Subscriber { - let _ = emitter.send(EngineEvent::AddTrack { - rtp_receiver, - streams, - }); + let _ = emitter + .send(EngineEvent::AddTrack { + rtp_receiver, + streams, + }) + .await; } } RTCEvent::Data { data, binary } => { @@ -377,7 +380,7 @@ impl RTCEngine { signal_response::Message::Offer(offer) => { // Handle the subscriber offer & send an answer to livekit-server // We always get an offer from the server when connecting - trace!("received offer from the publisher: {:?}", offer); + trace!("received offer for the subscriber: {:?}", offer); let sdp = SessionDescription::from(offer.r#type.parse().unwrap(), &offer.sdp)?; engine_inner @@ -438,7 +441,7 @@ impl RTCEngine { } } signal_response::Message::Update(update) => { - let _ = emitter.send(EngineEvent::ParticipantUpdate(update)); + let _ = emitter.send(EngineEvent::ParticipantUpdate(update)).await; } _ => {} } diff --git a/crates/livekit-webrtc/libwebrtc-sys/include/livekit/media_stream.h b/crates/livekit-webrtc/libwebrtc-sys/include/livekit/media_stream.h index bf5d652..603940a 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/include/livekit/media_stream.h +++ b/crates/livekit-webrtc/libwebrtc-sys/include/livekit/media_stream.h @@ -102,19 +102,19 @@ class NativeVideoFrameSink std::unique_ptr create_native_video_frame_sink( rust::Box observer); -const MediaStreamTrack* video_to_media(const VideoTrack* track) { +static const MediaStreamTrack* video_to_media(const VideoTrack* track) { return track; } -const MediaStreamTrack* audio_to_media(const AudioTrack* track) { +static const MediaStreamTrack* audio_to_media(const AudioTrack* track) { return track; } -const VideoTrack* media_to_video(const MediaStreamTrack* track) { +static const VideoTrack* media_to_video(const MediaStreamTrack* track) { return static_cast(track); } -const AudioTrack* media_to_audio(const MediaStreamTrack* track) { +static const AudioTrack* media_to_audio(const MediaStreamTrack* track) { return static_cast(track); } diff --git a/crates/livekit-webrtc/libwebrtc-sys/include/livekit/video_frame.h b/crates/livekit-webrtc/libwebrtc-sys/include/livekit/video_frame.h index 9c719f4..6a247a5 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/include/livekit/video_frame.h +++ b/crates/livekit-webrtc/libwebrtc-sys/include/livekit/video_frame.h @@ -30,8 +30,8 @@ class VideoFrame { } // TODO(theomonnom) This shouldn't create a new shared_ptr at each call - std::shared_ptr video_frame_buffer() const { - return std::make_shared(frame_.video_frame_buffer()); + std::unique_ptr video_frame_buffer() const { + return std::make_unique(frame_.video_frame_buffer()); } private: diff --git a/crates/livekit-webrtc/libwebrtc-sys/include/livekit/video_frame_buffer.h b/crates/livekit-webrtc/libwebrtc-sys/include/livekit/video_frame_buffer.h index 3b4909c..1e3cfe2 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/include/livekit/video_frame_buffer.h +++ b/crates/livekit-webrtc/libwebrtc-sys/include/livekit/video_frame_buffer.h @@ -5,6 +5,8 @@ #ifndef LIVEKIT_WEBRTC_VIDEO_FRAME_BUFFER_H #define LIVEKIT_WEBRTC_VIDEO_FRAME_BUFFER_H +#include + #include "api/video/video_frame_buffer.h" #include "rust_types.h" @@ -26,8 +28,15 @@ class VideoFrameBuffer { int width() const { return buffer_->width(); } int height() const { return buffer_->height(); } - std::shared_ptr to_i420() { - return std::make_shared(buffer_->ToI420()); + std::unique_ptr to_i420() { + return std::make_unique(buffer_->ToI420()); + } + + std::unique_ptr get_i420() { + // const_cast is valid here because we take the ownership on the rust side + return std::make_unique( + rtc::scoped_refptr( + const_cast(buffer_->GetI420()))); } protected: @@ -73,19 +82,20 @@ class I420Buffer : public PlanarYuv8Buffer { : PlanarYuv8Buffer(buffer) {} }; -std::shared_ptr to_video_frame_buffer( - std::shared_ptr buffer) { - return buffer; +static const VideoFrameBuffer* yuv_to_vfb(const PlanarYuvBuffer* yuv) { + return yuv; } -std::shared_ptr to_yuv_buffer( - std::shared_ptr buffer) { - return buffer; +static const PlanarYuvBuffer* yuv8_to_yuv(const PlanarYuv8Buffer* yuv8) { + return yuv8; } -std::shared_ptr to_yuv8_buffer( - std::shared_ptr buffer) { - return buffer; +static const PlanarYuv8Buffer* i420_to_yuv8(const I420Buffer* i420) { + return i420; +} + +static std::unique_ptr _unique_video_frame_buffer() { + return nullptr; // Ignore } } // namespace livekit diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/media_stream.cpp b/crates/livekit-webrtc/libwebrtc-sys/src/media_stream.cpp index b4017ee..b2758a3 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/src/media_stream.cpp +++ b/crates/livekit-webrtc/libwebrtc-sys/src/media_stream.cpp @@ -4,6 +4,7 @@ #include "livekit/media_stream.h" +#include "api/media_stream_interface.h" #include "libwebrtc-sys/src/media_stream.rs.h" namespace livekit { @@ -53,6 +54,9 @@ rust::String MediaStream::id() const { return media_stream_->id(); } +AudioTrack::AudioTrack(rtc::scoped_refptr track) + : MediaStreamTrack(std::move(track)) {} + VideoTrack::VideoTrack(rtc::scoped_refptr track) : MediaStreamTrack(std::move(track)) {} diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/video_frame.rs b/crates/livekit-webrtc/libwebrtc-sys/src/video_frame.rs index 79ba2bf..def1933 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/src/video_frame.rs +++ b/crates/livekit-webrtc/libwebrtc-sys/src/video_frame.rs @@ -25,7 +25,7 @@ pub mod ffi { fn transport_frame_id(self: &VideoFrame) -> u32; fn timestamp(self: &VideoFrame) -> u32; fn rotation(self: &VideoFrame) -> VideoRotation; - fn video_frame_buffer(self: &VideoFrame) -> SharedPtr; + fn video_frame_buffer(self: &VideoFrame) -> UniquePtr; fn _unique_video_frame() -> UniquePtr; // Ignore } diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/video_frame_buffer.rs b/crates/livekit-webrtc/libwebrtc-sys/src/video_frame_buffer.rs index 65d4fad..ecfa4d2 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/src/video_frame_buffer.rs +++ b/crates/livekit-webrtc/libwebrtc-sys/src/video_frame_buffer.rs @@ -23,7 +23,11 @@ pub mod ffi { fn buffer_type(self: &VideoFrameBuffer) -> VideoFrameBufferType; fn width(self: &VideoFrameBuffer) -> i32; fn height(self: &VideoFrameBuffer) -> i32; - fn to_i420(self: Pin<&mut VideoFrameBuffer>) -> SharedPtr; + + // Require ownership + unsafe fn to_i420(self: Pin<&mut VideoFrameBuffer>) -> UniquePtr; + unsafe fn get_i420(self: Pin<&mut VideoFrameBuffer>) -> UniquePtr; + // TODO(theomonnom): Bridge other get_* fn chroma_width(self: &PlanarYuvBuffer) -> i32; fn chroma_height(self: &PlanarYuvBuffer) -> i32; @@ -35,9 +39,10 @@ pub mod ffi { fn data_u(self: &PlanarYuv8Buffer) -> *const u8; fn data_v(self: &PlanarYuv8Buffer) -> *const u8; - fn to_video_frame_buffer(buffer: SharedPtr) - -> SharedPtr; - fn to_yuv_buffer(buffer: SharedPtr) -> SharedPtr; - fn to_yuv8_buffer(buffer: SharedPtr) -> SharedPtr; + unsafe fn yuv_to_vfb(yuv: *const PlanarYuvBuffer) -> *const VideoFrameBuffer; + unsafe fn yuv8_to_yuv(yuv8: *const PlanarYuv8Buffer) -> *const PlanarYuvBuffer; + unsafe fn i420_to_yuv8(i420: *const I420Buffer) -> *const PlanarYuv8Buffer; + + fn _unique_video_frame_buffer() -> UniquePtr; } } diff --git a/crates/livekit-webrtc/src/media_stream.rs b/crates/livekit-webrtc/src/media_stream.rs index bb5981d..565c92b 100644 --- a/crates/livekit-webrtc/src/media_stream.rs +++ b/crates/livekit-webrtc/src/media_stream.rs @@ -9,6 +9,7 @@ pub use sys_ms::ffi::ContentHint; pub use sys_ms::ffi::TrackState; use crate::video_frame::VideoFrame; +use crate::video_frame_buffer::VideoFrameBuffer; pub trait MediaStreamTrackTrait { fn kind(&self) -> String; @@ -115,8 +116,8 @@ macro_rules! impl_media_stream_track_trait { fn set_enabled(&self, enabled: bool) -> bool { unsafe { - let media = - sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap()) as *mut sys_ms::ffi::MediaStreamTrack; + let media = sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap()) + as *mut sys_ms::ffi::MediaStreamTrack; Pin::new_unchecked(&mut *media).set_enabled(enabled) } @@ -132,7 +133,7 @@ macro_rules! impl_media_stream_track_trait { impl_media_stream_track_trait!(VideoTrack, video_to_media); impl_media_stream_track_trait!(AudioTrack, audio_to_media); -pub type OnFrameHandler = Box; +pub type OnFrameHandler = Box; pub type OnDiscardedFrameHandler = Box; pub type OnConstraintsChanged = Box; @@ -160,7 +161,9 @@ impl From for VideoTrackSourceConstrai impl sys_ms::VideoFrameSink for InternalVideoTrackSink { fn on_frame(&self, frame: UniquePtr) { if let Some(cb) = self.on_frame_handler.lock().unwrap().as_mut() { - cb(VideoFrame::new(frame)); + let frame = VideoFrame::new(frame); + let video_frame_buffer = unsafe { frame.video_frame_buffer() }; + cb(frame, video_frame_buffer); } } @@ -203,7 +206,7 @@ impl VideoTrack { Arc::new(track) } - pub fn set_should_receive(&mut self, should_receive: bool) { + pub fn set_should_receive(&self, should_receive: bool) { self.cxx_handle .lock() .unwrap() @@ -211,7 +214,7 @@ impl VideoTrack { .set_should_receive(should_receive) } - pub fn set_content_hint(&mut self, hint: ContentHint) { + pub fn set_content_hint(&self, hint: ContentHint) { self.cxx_handle .lock() .unwrap() @@ -227,15 +230,15 @@ impl VideoTrack { self.cxx_handle.lock().unwrap().content_hint() } - pub fn on_frame(&mut self, handler: OnFrameHandler) { + pub fn on_frame(&self, handler: OnFrameHandler) { *self.observer.on_frame_handler.lock().unwrap() = Some(handler); } - pub fn on_discarded_frame(&mut self, handler: OnDiscardedFrameHandler) { + pub fn on_discarded_frame(&self, handler: OnDiscardedFrameHandler) { *self.observer.on_discarded_frame_handler.lock().unwrap() = Some(handler); } - pub fn on_constraints_changed(&mut self, handler: OnConstraintsChanged) { + pub fn on_constraints_changed(&self, handler: OnConstraintsChanged) { *self.observer.on_constraints_changed_handler.lock().unwrap() = Some(handler); } } diff --git a/crates/livekit-webrtc/src/peer_connection.rs b/crates/livekit-webrtc/src/peer_connection.rs index 52fa6c0..7cec14e 100644 --- a/crates/livekit-webrtc/src/peer_connection.rs +++ b/crates/livekit-webrtc/src/peer_connection.rs @@ -459,7 +459,7 @@ impl sys_pc::PeerConnectionObserver for InternalObserver { } fn on_ice_connection_change(&self, new_state: IceConnectionState) { - trace!("on_ice_connection_change"); + trace!("on_ice_connection_change (new_state: {:?})", new_state); let mut handler = self.on_ice_connection_change_handler.lock().unwrap(); if let Some(f) = handler.as_mut() { f(new_state); @@ -467,7 +467,7 @@ impl sys_pc::PeerConnectionObserver for InternalObserver { } fn on_standardized_ice_connection_change(&self, new_state: IceConnectionState) { - trace!("on_standardized_ice_connection_change"); + trace!("on_standardized_ice_connection_change (new_state: {:?}", new_state); let mut handler = self .on_standardized_ice_connection_change_handler .lock() @@ -478,7 +478,7 @@ impl sys_pc::PeerConnectionObserver for InternalObserver { } fn on_connection_change(&self, new_state: PeerConnectionState) { - trace!("on_connection_change"); + trace!("on_connection_change (new_state: {:?})", new_state); let mut handler = self.on_connection_change_handler.lock().unwrap(); if let Some(f) = handler.as_mut() { f(new_state); @@ -486,7 +486,7 @@ impl sys_pc::PeerConnectionObserver for InternalObserver { } fn on_ice_gathering_change(&self, new_state: IceGatheringState) { - trace!("on_ice_gathering_change"); + trace!("on_ice_gathering_change (new_state: {:?}", new_state); let mut handler = self.on_ice_gathering_change_handler.lock().unwrap(); if let Some(f) = handler.as_mut() { f(new_state); diff --git a/crates/livekit-webrtc/src/video_frame.rs b/crates/livekit-webrtc/src/video_frame.rs index 4926525..c36bf79 100644 --- a/crates/livekit-webrtc/src/video_frame.rs +++ b/crates/livekit-webrtc/src/video_frame.rs @@ -3,6 +3,8 @@ use libwebrtc_sys::video_frame as vf_sys; pub use vf_sys::ffi::VideoRotation; +use crate::video_frame_buffer::VideoFrameBuffer; + pub struct VideoFrame { cxx_handle: UniquePtr, } @@ -31,7 +33,7 @@ impl VideoFrame { pub fn timestamp_us(&self) -> i64 { self.cxx_handle.timestamp_us() } - + pub fn ntp_time_ms(&self) -> i64 { self.cxx_handle.ntp_time_ms() } @@ -47,4 +49,12 @@ impl VideoFrame { pub fn rotation(&self) -> VideoRotation { self.cxx_handle.rotation() } + + /// # Safety + /// Must be called only once, this function create the safe Rust + /// wrapper around a VideoFrameBuffer. + /// Only one wrapper musts exist at a time. + pub(crate) unsafe fn video_frame_buffer(&self) -> VideoFrameBuffer { + VideoFrameBuffer::new(self.cxx_handle.video_frame_buffer()) + } } diff --git a/crates/livekit-webrtc/src/video_frame_buffer.rs b/crates/livekit-webrtc/src/video_frame_buffer.rs index 9c60b85..64cf28d 100644 --- a/crates/livekit-webrtc/src/video_frame_buffer.rs +++ b/crates/livekit-webrtc/src/video_frame_buffer.rs @@ -1,18 +1,250 @@ use cxx::UniquePtr; use libwebrtc_sys::video_frame_buffer as vfb_sys; +use std::pin::Pin; +use std::slice; +use vfb_sys::ffi::VideoFrameBufferType; -pub use vfb_sys::ffi::VideoFrameBufferType; +pub trait VideoFrameBufferTrait { + fn width(&self) -> i32; + fn height(&self) -> i32; + fn to_i420(self) -> I420Buffer; +} -pub struct VideoFrameBuffer { - cxx_handle: UniquePtr, +pub trait PlanarYuvBuffer: VideoFrameBufferTrait { + fn chroma_width(&self) -> i32; + fn chroma_height(&self) -> i32; + fn stride_y(&self) -> i32; + fn stride_u(&self) -> i32; + fn stride_v(&self) -> i32; +} + +pub trait PlanarYuv8Buffer: PlanarYuvBuffer { + fn data_y(&self) -> &[u8]; + fn data_u(&self) -> &[u8]; + fn data_v(&self) -> &[u8]; +} + +pub enum VideoFrameBuffer { + Native(NativeBuffer), + I420(I420Buffer), + I420A(I420ABuffer), + I422(I422Buffer), + I444(I444Buffer), + I010(I010Buffer), + NV12(NV12Buffer), } impl VideoFrameBuffer { - pub fn new(cxx_handle: UniquePtr) -> Self { - Self { cxx_handle } - } - - pub fn buffer_type(&self) -> VideoFrameBufferType { - self.cxx_handle.buffer_type() + pub(crate) fn new(mut cxx_handle: UniquePtr) -> Self { + unsafe { + match cxx_handle.buffer_type() { + VideoFrameBufferType::Native => Self::Native(NativeBuffer::new(cxx_handle)), + VideoFrameBufferType::I420 => { + Self::I420(I420Buffer::new(cxx_handle.pin_mut().get_i420())) + } + VideoFrameBufferType::I420A => Self::I420A(I420ABuffer::new(cxx_handle)), + VideoFrameBufferType::I422 => Self::I422(I422Buffer::new(cxx_handle)), + VideoFrameBufferType::I444 => Self::I444(I444Buffer::new(cxx_handle)), + VideoFrameBufferType::I010 => Self::I010(I010Buffer::new(cxx_handle)), + VideoFrameBufferType::NV12 => Self::NV12(NV12Buffer::new(cxx_handle)), + _ => unreachable!(), // VideoFrameBufferType is represented as i32 + } + } + } +} + +macro_rules! recursive_cast { + ($ptr:expr $(, $fnc:ident)*) => { + { + let ptr = $ptr; + $( + let ptr = unsafe { vfb_sys::ffi::$fnc(ptr) }; + )* + ptr + } + }; +} + +macro_rules! impl_video_frame_buffer { + ($x:ty $(, $cast:ident)*) => { + + // Allow unused_unsafe when we don't do any cast ( e.g. NativeBuffer ) + #[allow(unused_unsafe)] + impl VideoFrameBufferTrait for $x { + fn width(&self) -> i32 { + let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); + unsafe { + (*ptr).width() + } + } + + fn height(&self) -> i32 { + let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); + unsafe { + (*ptr).height() + } + } + + // Require ownership because libwebrtc uses the same pointers + fn to_i420(self) -> I420Buffer { + let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*) + as *const vfb_sys::ffi::VideoFrameBuffer + as *mut vfb_sys::ffi::VideoFrameBuffer; + + unsafe { + I420Buffer::new(Pin::new_unchecked(&mut *ptr).to_i420()) + } + } + } + }; +} + +macro_rules! impl_yuv_buffer { + ($x:ty $(, $cast:ident)*) => { + impl PlanarYuvBuffer for $x { + fn chroma_width(&self) -> i32 { + let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); + unsafe { + (*ptr).chroma_width() + } + } + + fn chroma_height(&self) -> i32 { + let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); + unsafe { + (*ptr).chroma_height() + } + } + + fn stride_y(&self) -> i32 { + let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); + unsafe { + (*ptr).stride_y() + } + } + + fn stride_u(&self) -> i32 { + let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); + unsafe { + (*ptr).stride_u() + } + } + + fn stride_v(&self) -> i32 { + let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); + unsafe { + (*ptr).stride_v() + } + } + } + }; +} + +macro_rules! impl_yuv8_buffer { + ($x:ty $(, $cast:ident)*) => { + impl PlanarYuv8Buffer for $x { + fn data_y(&self) -> &[u8] { + let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); + unsafe { + slice::from_raw_parts((*ptr).data_y(), self.stride_y().try_into().unwrap()) + } + } + + fn data_u(&self) -> &[u8] { + let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); + unsafe { + slice::from_raw_parts((*ptr).data_u(), self.stride_u().try_into().unwrap()) + } + } + + fn data_v(&self) -> &[u8] { + let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); + unsafe { + slice::from_raw_parts((*ptr).data_v(), self.stride_v().try_into().unwrap()) + } + } + } + }; +} + +pub struct NativeBuffer { + cxx_handle: UniquePtr, +} + +pub struct I420Buffer { + cxx_handle: UniquePtr, +} + +pub struct I420ABuffer { + cxx_handle: UniquePtr, +} + +pub struct I422Buffer { + cxx_handle: UniquePtr, +} + +pub struct I444Buffer { + cxx_handle: UniquePtr, +} + +pub struct I010Buffer { + cxx_handle: UniquePtr, +} + +pub struct NV12Buffer { + cxx_handle: UniquePtr, +} + +impl_video_frame_buffer!(NativeBuffer); +impl_video_frame_buffer!(I420Buffer, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb); +impl_video_frame_buffer!(I420ABuffer); +impl_video_frame_buffer!(I422Buffer); +impl_video_frame_buffer!(I444Buffer); +impl_video_frame_buffer!(I010Buffer); +impl_video_frame_buffer!(NV12Buffer); + +impl_yuv_buffer!(I420Buffer, i420_to_yuv8, yuv8_to_yuv); + +impl_yuv8_buffer!(I420Buffer, i420_to_yuv8); + +impl NativeBuffer { + fn new(cxx_handle: UniquePtr) -> Self { + Self { cxx_handle } + } +} + +impl I420Buffer { + fn new(cxx_handle: UniquePtr) -> Self { + Self { cxx_handle } + } +} + +impl I420ABuffer { + fn new(cxx_handle: UniquePtr) -> Self { + Self { cxx_handle } + } +} + +impl I422Buffer { + fn new(cxx_handle: UniquePtr) -> Self { + Self { cxx_handle } + } +} + +impl I444Buffer { + fn new(cxx_handle: UniquePtr) -> Self { + Self { cxx_handle } + } +} + +impl I010Buffer { + fn new(cxx_handle: UniquePtr) -> Self { + Self { cxx_handle } + } +} + +impl NV12Buffer { + fn new(cxx_handle: UniquePtr) -> Self { + Self { cxx_handle } } } diff --git a/examples/simple_room/src/main.rs b/examples/simple_room/src/main.rs index 3997612..1b49771 100644 --- a/examples/simple_room/src/main.rs +++ b/examples/simple_room/src/main.rs @@ -1,7 +1,8 @@ -use livekit::room::track::TrackTrait; +use std::time::Duration; + +use livekit::room::RoomError; use livekit::room::{track::remote_track::RemoteTrackHandle, Room}; -use std::sync::{Arc, Mutex}; -use tracing::{event_enabled, info, trace}; +use tokio::time::sleep; const URL: &str = "ws://localhost:7880"; const TOKEN : &str = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjIzODQ4MDY0NzMsImlzcyI6IkFQSXpLYkFTaUNWYWtnSiIsIm5hbWUiOiJuYXRpdmUiLCJuYmYiOjE2NjQ4MDY0NzMsInN1YiI6Im5hdGl2ZSIsInZpZGVvIjp7InJvb21DcmVhdGUiOnRydWUsInJvb21Kb2luIjp0cnVlfX0.BgVdBnq3XFD3_BQHoe1azqjifYysubgFl6Qlzu9IQGI"; @@ -9,23 +10,27 @@ const TOKEN : &str = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjIzODQ4MDY0N // eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjIzODQ4MDY3MzAsImlzcyI6IkFQSXpLYkFTaUNWYWtnSiIsIm5hbWUiOiJ3ZWIiLCJuYmYiOjE2NjQ4MDY3MzAsInN1YiI6IndlYiIsInZpZGVvIjp7InJvb21DcmVhdGUiOnRydWUsInJvb21Kb2luIjp0cnVlfX0.VbDoULjX1CVGZu2sPy3SvWYlVZUBXxQVPmdB9BnmlN4 #[tokio::main] -async fn main() { +async fn main() -> Result<(), RoomError> { tracing_subscriber::fmt::init(); - let room = Room::new(); + let mut room = Room::new(); room.events() - .on_participant_connected(|event| async move {}); + .on_participant_connected(|_event| async move {}); room.events().on_track_subscribed(|event| async move { let track = event.publication.track().unwrap(); if let RemoteTrackHandle::Video(video_track) = track { let rtc_track = video_track.rtc_track(); - rtc_track.on_frame(Box::new(|frame| { Box::pin(async move { - - - - - }) })) + rtc_track.set_should_receive(true); + rtc_track.on_frame(Box::new(|_frame, _buffer| { + // called on libwebrtc worker_thread + println!("Received frame"); + })); } }); + + room.connect(URL, TOKEN).await?; + + sleep(Duration::from_secs(200)).await; + Ok(()) } From 59e7ccbffe12d0267d156292f6437c7edb169254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Mon, 28 Nov 2022 17:30:21 +0100 Subject: [PATCH 7/8] Successfully received video frames --- Cargo.toml | 2 +- crates/livekit-core/src/signal_client/mod.rs | 2 +- .../livekit-webrtc/libwebrtc-sys/include/livekit/webrtc.h | 2 ++ .../libwebrtc-sys/src/peer_connection_factory.cpp | 6 +++++- examples/simple_room/Cargo.toml | 2 +- 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a705033..5e5d1fa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,4 +13,4 @@ members = [ ] [dependencies] -livekit-core = { path = "crates/livekit-core" } \ No newline at end of file +livekit-core = { path = "crates/livekit-core" } diff --git a/crates/livekit-core/src/signal_client/mod.rs b/crates/livekit-core/src/signal_client/mod.rs index 9e9d7c4..2958de4 100644 --- a/crates/livekit-core/src/signal_client/mod.rs +++ b/crates/livekit-core/src/signal_client/mod.rs @@ -53,7 +53,7 @@ impl Default for SignalOptions { reconnect: false, auto_subscribe: true, sid: "".to_string(), - adaptive_stream: true, + adaptive_stream: false, } } } diff --git a/crates/livekit-webrtc/libwebrtc-sys/include/livekit/webrtc.h b/crates/livekit-webrtc/libwebrtc-sys/include/livekit/webrtc.h index 11b4a6f..50fe659 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/include/livekit/webrtc.h +++ b/crates/livekit-webrtc/libwebrtc-sys/include/livekit/webrtc.h @@ -32,6 +32,8 @@ class RTCRuntime { std::unique_ptr signaling_thread_; #ifdef WEBRTC_WIN rtc::WinsockInitializer winsock_; + rtc::PhysicalSocketServer ss_; + rtc::AutoSocketServerThread main_thread_ {&ss_}; #endif }; diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/peer_connection_factory.cpp b/crates/livekit-webrtc/libwebrtc-sys/src/peer_connection_factory.cpp index 41ecf56..15c00b6 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/src/peer_connection_factory.cpp +++ b/crates/livekit-webrtc/libwebrtc-sys/src/peer_connection_factory.cpp @@ -27,10 +27,12 @@ PeerConnectionFactory::PeerConnectionFactory( dependencies.network_thread = rtc_runtime_->network_thread(); dependencies.worker_thread = rtc_runtime_->worker_thread(); dependencies.signaling_thread = rtc_runtime_->signaling_thread(); + dependencies.socket_factory = rtc_runtime_->network_thread()->socketserver(); dependencies.task_queue_factory = webrtc::CreateDefaultTaskQueueFactory(); dependencies.event_log_factory = std::make_unique( dependencies.task_queue_factory.get()); dependencies.call_factory = webrtc::CreateCallFactory(); + dependencies.trials = std::make_unique(); cricket::MediaEngineDependencies media_deps; media_deps.task_queue_factory = dependencies.task_queue_factory.get(); @@ -38,6 +40,8 @@ PeerConnectionFactory::PeerConnectionFactory( media_deps.video_decoder_factory = webrtc::CreateBuiltinVideoDecoderFactory(); media_deps.audio_encoder_factory = webrtc::CreateBuiltinAudioEncoderFactory(); media_deps.audio_decoder_factory = webrtc::CreateBuiltinAudioDecoderFactory(); + media_deps.audio_processing = webrtc::AudioProcessingBuilder().Create(); + media_deps.trials = dependencies.trials.get(); dependencies.media_engine = cricket::CreateMediaEngine(std::move(media_deps)); @@ -96,4 +100,4 @@ std::unique_ptr create_rtc_configuration( return rtc; } -} // namespace livekit \ No newline at end of file +} // namespace livekit diff --git a/examples/simple_room/Cargo.toml b/examples/simple_room/Cargo.toml index 1f13b23..aab5f93 100644 --- a/examples/simple_room/Cargo.toml +++ b/examples/simple_room/Cargo.toml @@ -8,4 +8,4 @@ tokio = { version = "1", features = ["full"] } tracing = "0.1" tracing-subscriber = "0.3" livekit = { path = "../.." } -futures = "0.3" \ No newline at end of file +futures = "0.3" From 686f5db969844f11c5371125be0467aa3e432402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Fri, 2 Dec 2022 16:39:42 +0100 Subject: [PATCH 8/8] livekit-utils crate + VideoRenderer proto --- Cargo.lock | 7 + Cargo.toml | 2 + crates/livekit-core/Cargo.toml | 3 +- crates/livekit-core/src/lib.rs | 1 - .../livekit-core/src/room/participant/mod.rs | 20 +- .../livekit-core/src/room/publication/mod.rs | 26 +- crates/livekit-core/src/room/track/mod.rs | 16 +- .../src/room/track/remote_track.rs | 16 +- crates/livekit-utils/Cargo.toml | 8 + .../src/enum_dispatch.rs} | 15 +- crates/livekit-utils/src/lib.rs | 1 + crates/livekit-webrtc/Cargo.toml | 3 +- .../livekit-webrtc/libwebrtc-sys/.gitignore | 1 - crates/livekit-webrtc/libwebrtc-sys/build.rs | 21 +- .../libwebrtc-sys/compile_flags.txt | 6 +- .../include/livekit/yuv_helper.h | 30 + .../libwebrtc-sys/libwebrtc/.gclient | 11 + .../libwebrtc-sys/libwebrtc/.gitignore | 3 + .../libwebrtc-sys/libwebrtc/compile.py | 19 + .../livekit-webrtc/libwebrtc-sys/src/lib.rs | 1 + .../libwebrtc-sys/src/yuv_helper.rs | 19 + crates/livekit-webrtc/src/lib.rs | 1 + .../livekit-webrtc/src/video_frame_buffer.rs | 18 +- crates/livekit-webrtc/src/yuv_helper.rs | 43 + examples/Cargo.lock | 1618 ++++++++++++++++- examples/Cargo.toml | 3 +- examples/simple_room/Cargo.toml | 7 + examples/simple_room/src/main.rs | 206 ++- examples/simple_room/src/video_renderer.rs | 172 ++ src/lib.rs | 4 + 30 files changed, 2211 insertions(+), 90 deletions(-) create mode 100644 crates/livekit-utils/Cargo.toml rename crates/{livekit-core/src/utils.rs => livekit-utils/src/enum_dispatch.rs} (51%) create mode 100644 crates/livekit-utils/src/lib.rs create mode 100644 crates/livekit-webrtc/libwebrtc-sys/include/livekit/yuv_helper.h create mode 100644 crates/livekit-webrtc/libwebrtc-sys/libwebrtc/.gclient create mode 100644 crates/livekit-webrtc/libwebrtc-sys/libwebrtc/.gitignore create mode 100644 crates/livekit-webrtc/libwebrtc-sys/libwebrtc/compile.py create mode 100644 crates/livekit-webrtc/libwebrtc-sys/src/yuv_helper.rs create mode 100644 crates/livekit-webrtc/src/yuv_helper.rs create mode 100644 examples/simple_room/src/video_renderer.rs diff --git a/Cargo.lock b/Cargo.lock index fe5be1c..0dd70ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -484,6 +484,7 @@ name = "livekit" version = "0.1.0" dependencies = [ "livekit-core", + "livekit-webrtc", ] [[package]] @@ -493,6 +494,7 @@ dependencies = [ "futures", "futures-util", "lazy_static", + "livekit-utils", "livekit-webrtc", "parking_lot", "prost", @@ -507,6 +509,10 @@ dependencies = [ "url", ] +[[package]] +name = "livekit-utils" +version = "0.1.0" + [[package]] name = "livekit-webrtc" version = "0.1.0" @@ -514,6 +520,7 @@ dependencies = [ "cxx", "env_logger", "libwebrtc-sys", + "livekit-utils", "log", "thiserror", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 5e5d1fa..36613eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,8 +9,10 @@ exclude = ["libwebrtc"] members = [ "crates/livekit-core", "crates/livekit-webrtc", + "crates/livekit-utils", "crates/livekit-webrtc/libwebrtc-sys" ] [dependencies] livekit-core = { path = "crates/livekit-core" } +livekit-webrtc = { path = "crates/livekit-webrtc" } diff --git a/crates/livekit-core/Cargo.toml b/crates/livekit-core/Cargo.toml index 3cf11f7..09f9c3a 100644 --- a/crates/livekit-core/Cargo.toml +++ b/crates/livekit-core/Cargo.toml @@ -5,6 +5,8 @@ edition = "2021" license = "Apache-2.0" [dependencies] +livekit-webrtc = { path = "../livekit-webrtc" } +livekit-utils = { path = "../livekit-utils" } serde = { version = "1", features = ["derive"] } serde_json = "1.0" tokio-tungstenite = { version = "0.17.2", features = ["native-tls"] } @@ -16,7 +18,6 @@ futures-util = "0.3.23" thiserror = "1.0" prost = "0.11.0" prost-types = "0.11.1" -livekit-webrtc = { path = "../livekit-webrtc" } lazy_static = "1.4.0" tracing = "0.1" diff --git a/crates/livekit-core/src/lib.rs b/crates/livekit-core/src/lib.rs index a9770a7..cbdb92a 100644 --- a/crates/livekit-core/src/lib.rs +++ b/crates/livekit-core/src/lib.rs @@ -7,6 +7,5 @@ pub mod proto { mod events; mod rtc_engine; mod signal_client; -mod utils; pub mod room; diff --git a/crates/livekit-core/src/room/participant/mod.rs b/crates/livekit-core/src/room/participant/mod.rs index 56c7da2..49b6f0d 100644 --- a/crates/livekit-core/src/room/participant/mod.rs +++ b/crates/livekit-core/src/room/participant/mod.rs @@ -4,8 +4,8 @@ use crate::room::id::{ParticipantIdentity, ParticipantSid, TrackSid}; use crate::room::participant::local_participant::LocalParticipant; use crate::room::participant::remote_participant::RemoteParticipant; use crate::room::publication::{TrackPublication, TrackPublicationTrait}; -use crate::utils::wrap_variants; use futures_util::future::BoxFuture; +use livekit_utils::enum_dispatch; use parking_lot::{Mutex, RwLock}; use std::collections::HashMap; use std::sync::Arc; @@ -74,7 +74,7 @@ pub enum ParticipantHandle { } impl ParticipantHandle { - // TODO(theomonnom): Add async support to wrap_variants ... + // TODO(theomonnom): Add async support to wrap_variants ... pub(crate) async fn update_info(&self, info: ParticipantInfo) { match self { Self::Local(inner) => inner.clone().update_info(info).await, @@ -84,20 +84,20 @@ impl ParticipantHandle { } impl ParticipantInternalTrait for ParticipantHandle { - wrap_variants!( + enum_dispatch!( [Local, Remote] - fnc!(internal_events, Arc, []); + fnc!(internal_events, &Self, [], Arc); ); } impl ParticipantTrait for ParticipantHandle { - wrap_variants!( + enum_dispatch!( [Local, Remote] - fnc!(events, Arc, []); - fnc!(sid, ParticipantSid, []); - fnc!(identity, ParticipantIdentity, []); - fnc!(name, String, []); - fnc!(metadata, String, []); + fnc!(events, &Self, [], Arc); + fnc!(sid, &Self, [], ParticipantSid); + fnc!(identity, &Self, [], ParticipantIdentity); + fnc!(name, &Self, [], String); + fnc!(metadata, &Self, [], String); ); } diff --git a/crates/livekit-core/src/room/publication/mod.rs b/crates/livekit-core/src/room/publication/mod.rs index 1820e88..547e825 100644 --- a/crates/livekit-core/src/room/publication/mod.rs +++ b/crates/livekit-core/src/room/publication/mod.rs @@ -5,7 +5,7 @@ use crate::room::id::TrackSid; use crate::room::track::local_track::LocalTrackHandle; use crate::room::track::remote_track::RemoteTrackHandle; use crate::room::track::{TrackHandle, TrackKind, TrackSource}; -use crate::utils::wrap_variants; +use livekit_utils::enum_dispatch; use parking_lot::Mutex; use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; use std::sync::Arc; @@ -38,7 +38,11 @@ pub(super) struct TrackPublicationShared { } impl TrackPublicationShared { - pub fn new(info: TrackInfo, participant: ParticipantSid, track: Option) -> Arc { + pub fn new( + info: TrackInfo, + participant: ParticipantSid, + track: Option, + ) -> Arc { Arc::new(Self { track: Mutex::new(track), name: Mutex::new(info.name), @@ -88,21 +92,21 @@ impl TrackPublication { } impl TrackPublicationInternalTrait for TrackPublication { - wrap_variants!( + enum_dispatch!( [Local, Remote] - fnc!(update_track, (), [track: Option]); - fnc!(update_info, (), [info: TrackInfo]); + fnc!(update_track, &Self, [track: Option], ()); + fnc!(update_info, &Self, [info: TrackInfo], ()); ); } impl TrackPublicationTrait for TrackPublication { - wrap_variants!( + enum_dispatch!( [Local, Remote] - fnc!(sid, TrackSid, []); - fnc!(name, String, []); - fnc!(kind, TrackKind, []); - fnc!(source, TrackSource, []); - fnc!(simulcasted, bool, []); + fnc!(sid, &Self, [], TrackSid); + fnc!(name, &Self, [], String); + fnc!(kind, &Self, [], TrackKind); + fnc!(source, &Self, [], TrackSource); + fnc!(simulcasted, &Self, [], bool); ); } diff --git a/crates/livekit-core/src/room/track/mod.rs b/crates/livekit-core/src/room/track/mod.rs index 327084f..ad9e218 100644 --- a/crates/livekit-core/src/room/track/mod.rs +++ b/crates/livekit-core/src/room/track/mod.rs @@ -4,7 +4,7 @@ use crate::room::track::local_audio_track::LocalAudioTrack; use crate::room::track::local_video_track::LocalVideoTrack; use crate::room::track::remote_audio_track::RemoteAudioTrack; use crate::room::track::remote_video_track::RemoteVideoTrack; -use crate::utils::wrap_variants; +use livekit_utils::enum_dispatch; use livekit_webrtc::media_stream::{MediaStreamTrackHandle, MediaStreamTrackTrait}; use parking_lot::Mutex; use std::sync::atomic::AtomicU8; @@ -150,14 +150,14 @@ pub enum TrackHandle { } impl TrackTrait for TrackHandle { - wrap_variants!( + enum_dispatch!( [LocalVideo, LocalAudio, RemoteVideo, RemoteAudio] - fnc!(sid, TrackSid, []); - fnc!(name, String, []); - fnc!(kind, TrackKind, []); - fnc!(stream_state, StreamState, []); - fnc!(start, (), []); - fnc!(stop, (), []); + fnc!(sid, &Self, [], TrackSid); + fnc!(name, &Self, [], String); + fnc!(kind, &Self, [], TrackKind); + fnc!(stream_state, &Self, [], StreamState); + fnc!(start, &Self, [], ()); + fnc!(stop, &Self, [], ()); ); } diff --git a/crates/livekit-core/src/room/track/remote_track.rs b/crates/livekit-core/src/room/track/remote_track.rs index 591b443..81e53ea 100644 --- a/crates/livekit-core/src/room/track/remote_track.rs +++ b/crates/livekit-core/src/room/track/remote_track.rs @@ -5,7 +5,7 @@ use crate::room::id::TrackSid; use crate::room::track::remote_audio_track::RemoteAudioTrack; use crate::room::track::remote_video_track::RemoteVideoTrack; use crate::room::track::TrackHandle; -use crate::utils::wrap_variants; +use livekit_utils::enum_dispatch; use super::TrackTrait; @@ -16,14 +16,14 @@ pub enum RemoteTrackHandle { } impl TrackTrait for RemoteTrackHandle { - wrap_variants!( + enum_dispatch!( [Audio, Video] - fnc!(sid, TrackSid, []); - fnc!(name, String, []); - fnc!(kind, TrackKind, []); - fnc!(stream_state, StreamState, []); - fnc!(start, (), []); - fnc!(stop, (), []); + fnc!(sid, &Self, [], TrackSid); + fnc!(name, &Self, [], String); + fnc!(kind, &Self, [], TrackKind); + fnc!(stream_state, &Self, [], StreamState); + fnc!(start, &Self, [], ()); + fnc!(stop, &Self, [], ()); ); } diff --git a/crates/livekit-utils/Cargo.toml b/crates/livekit-utils/Cargo.toml new file mode 100644 index 0000000..27b9ca7 --- /dev/null +++ b/crates/livekit-utils/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "livekit-utils" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/crates/livekit-core/src/utils.rs b/crates/livekit-utils/src/enum_dispatch.rs similarity index 51% rename from crates/livekit-core/src/utils.rs rename to crates/livekit-utils/src/enum_dispatch.rs index c250092..c7f59df 100644 --- a/crates/livekit-core/src/utils.rs +++ b/crates/livekit-utils/src/enum_dispatch.rs @@ -1,4 +1,5 @@ -macro_rules! wrap_variants { +#[macro_export] +macro_rules! enum_dispatch { // This arm is used to avoid nested loops with the arguments // The arguments are transformed to $combined_args TokenTree (@match $self:ident $fnc:ident $combined_args:tt [$($variant:ident),+]) => { @@ -9,18 +10,16 @@ macro_rules! wrap_variants { } }; - ($fnc:ident, $ret:ty, [$($arg:ident: $t:ty),*], [$($variant:ident),+]) => { - fn $fnc(&self, $($arg: $t),*) -> $ret { - wrap_variants!(@match self $fnc ($($arg,)*) [$($variant),+]) + ($fnc:ident, $self:ty, [$($arg:ident: $t:ty),*], $ret:ty, [$($variant:ident),+]) => { + fn $fnc(self: $self, $($arg: $t),*) -> $ret { + enum_dispatch!(@match self $fnc ($($arg,)*) [$($variant),+]) } }; - ($variants:tt $(fnc!($fnc:ident, $ret:ty, $args:tt);)+) => { + ($variants:tt $(fnc!($fnc:ident, $self:ty, $args:tt, $ret:ty);)+) => { $( - wrap_variants!($fnc, $ret, $args, $variants); + enum_dispatch!($fnc, $self, $args, $ret, $variants); )* }; } -pub(crate) use wrap_variants; - diff --git a/crates/livekit-utils/src/lib.rs b/crates/livekit-utils/src/lib.rs new file mode 100644 index 0000000..f328b57 --- /dev/null +++ b/crates/livekit-utils/src/lib.rs @@ -0,0 +1 @@ +pub mod enum_dispatch; diff --git a/crates/livekit-webrtc/Cargo.toml b/crates/livekit-webrtc/Cargo.toml index c9264ff..0de0d93 100644 --- a/crates/livekit-webrtc/Cargo.toml +++ b/crates/livekit-webrtc/Cargo.toml @@ -6,10 +6,11 @@ homepage = "https://livekit.io" [dependencies] libwebrtc-sys = { path = "./libwebrtc-sys" } +livekit-utils = { path = "../livekit-utils" } tokio = { version = "1", features = ["full"] } cxx = "1.0" log = "0.4" thiserror = "1.0" [dev-dependencies] -env_logger = "0.9" \ No newline at end of file +env_logger = "0.9" diff --git a/crates/livekit-webrtc/libwebrtc-sys/.gitignore b/crates/livekit-webrtc/libwebrtc-sys/.gitignore index 1a2f000..3aa1f00 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/.gitignore +++ b/crates/livekit-webrtc/libwebrtc-sys/.gitignore @@ -1,2 +1 @@ -/libwebrtc /cmake-build-debug \ No newline at end of file diff --git a/crates/livekit-webrtc/libwebrtc-sys/build.rs b/crates/livekit-webrtc/libwebrtc-sys/build.rs index b1fdd96..377958a 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/build.rs +++ b/crates/livekit-webrtc/libwebrtc-sys/build.rs @@ -55,17 +55,17 @@ fn main() { let target_os = "windows"; //let target_arch = "arm64"; - let libwebrtc_dir = path::PathBuf::from("libwebrtc"); + let libwebrtc_dir = path::PathBuf::from("libwebrtc/src"); // Just required for the bridge build to succeed. let includes = &[ path::PathBuf::from("./include"), - libwebrtc_dir.join("include/"), - libwebrtc_dir.join("include/third_party/abseil-cpp/"), - libwebrtc_dir.join("include/third_party/libc++/"), + libwebrtc_dir.clone(), + libwebrtc_dir.join("third_party/abseil-cpp/"), + libwebrtc_dir.join("third_party/libc++/"), // For mac & ios - libwebrtc_dir.join("include/sdk/objc"), - libwebrtc_dir.join("include/sdk/objc/base"), + libwebrtc_dir.join("sdk/objc"), + libwebrtc_dir.join("sdk/objc/base"), ]; let mut builder = cxx_build::bridges(&[ @@ -81,6 +81,7 @@ fn main() { "src/webrtc.rs", "src/video_frame.rs", "src/video_frame_buffer.rs", + "src/yuv_helper.rs", ]); builder.file("src/peer_connection.cpp"); @@ -100,7 +101,12 @@ fn main() { println!( "cargo:rustc-link-search=native={}", - libwebrtc_dir.canonicalize().unwrap().to_str().unwrap() + libwebrtc_dir + .join("out/Default/obj") + .canonicalize() + .unwrap() + .to_str() + .unwrap() ); match target_os { @@ -124,6 +130,7 @@ fn main() { .flag("/std:c++17") .flag("/EHsc") .define("WEBRTC_WIN", None) + //.define("WEBRTC_ENABLE_SYMBOL_EXPORT", None) Not necessary when using WebRTC as a static library .define("NOMINMAX", None); } "macos" => { diff --git a/crates/livekit-webrtc/libwebrtc-sys/compile_flags.txt b/crates/livekit-webrtc/libwebrtc-sys/compile_flags.txt index e0fb4f0..1a76a73 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/compile_flags.txt +++ b/crates/livekit-webrtc/libwebrtc-sys/compile_flags.txt @@ -1,8 +1,8 @@ -xc++ -std=c++17 -Iinclude --Ilibwebrtc/include --Ilibwebrtc/include/third_party/abseil-cpp --Ilibwebrtc/include/third_party/libc++ +-Ilibwebrtc/src +-Ilibwebrtc/src/third_party/abseil-cpp +-Ilibwebrtc/src/third_party/libc++ -I../../../target/cxxbridge -DWEBRTC_WIN diff --git a/crates/livekit-webrtc/libwebrtc-sys/include/livekit/yuv_helper.h b/crates/livekit-webrtc/libwebrtc-sys/include/livekit/yuv_helper.h new file mode 100644 index 0000000..1378cdd --- /dev/null +++ b/crates/livekit-webrtc/libwebrtc-sys/include/livekit/yuv_helper.h @@ -0,0 +1,30 @@ +// +// Created by Théo Monnom on 01/12/2022. +// + +#ifndef CLIENT_SDK_NATIVE_YUV_HELPER_H +#define CLIENT_SDK_NATIVE_YUV_HELPER_H + +#include + +#include "api/video/yuv_helper.h" + +namespace livekit { + +static void i420_to_abgr(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_rgba, + int dst_stride_abgr, + int width, + int height) { + webrtc::I420ToABGR(src_y, src_stride_y, src_u, src_stride_u, src_v, + src_stride_v, dst_rgba, dst_stride_abgr, width, height); +} + +} // namespace livekit + +#endif // CLIENT_SDK_NATIVE_YUV_HELPER_H diff --git a/crates/livekit-webrtc/libwebrtc-sys/libwebrtc/.gclient b/crates/livekit-webrtc/libwebrtc-sys/libwebrtc/.gclient new file mode 100644 index 0000000..b58c202 --- /dev/null +++ b/crates/livekit-webrtc/libwebrtc-sys/libwebrtc/.gclient @@ -0,0 +1,11 @@ +solutions = [ + { + "name" : 'src', + "url" : 'https://github.com/webrtc-sdk/webrtc.git', + "deps_file" : 'DEPS', + "managed" : False, + "custom_deps" : { + }, + "custom_vars": {}, + }, +] diff --git a/crates/livekit-webrtc/libwebrtc-sys/libwebrtc/.gitignore b/crates/livekit-webrtc/libwebrtc-sys/libwebrtc/.gitignore new file mode 100644 index 0000000..2ec6974 --- /dev/null +++ b/crates/livekit-webrtc/libwebrtc-sys/libwebrtc/.gitignore @@ -0,0 +1,3 @@ +.cipd +src +.gclient_* diff --git a/crates/livekit-webrtc/libwebrtc-sys/libwebrtc/compile.py b/crates/livekit-webrtc/libwebrtc-sys/libwebrtc/compile.py new file mode 100644 index 0000000..7119707 --- /dev/null +++ b/crates/livekit-webrtc/libwebrtc-sys/libwebrtc/compile.py @@ -0,0 +1,19 @@ +import subprocess + +GN_ARGS = [ + "is_debug=false", + "treat_warnings_as_errors=false", + 'target_os="win"', + 'target_cpu="x64"', + "rtc_include_tests=false", + "rtc_use_h264=false", + "is_component_build=false", + "rtc_build_examples=false", + "use_rtti=true", + "rtc_build_tools=false", + "use_custom_libcxx=false", + "strip_debug_info=true", + "symbol_level=0" +] + +subprocess.call(["gn", "gen", "out/Default", "--args=" + ' '.join(GN_ARGS)], shell=True) diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/lib.rs b/crates/livekit-webrtc/libwebrtc-sys/src/lib.rs index ab6e52a..a2abebd 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/src/lib.rs +++ b/crates/livekit-webrtc/libwebrtc-sys/src/lib.rs @@ -10,6 +10,7 @@ pub mod rtp_transceiver; pub mod video_frame; pub mod video_frame_buffer; pub mod webrtc; +pub mod yuv_helper; pub const MEDIA_TYPE_VIDEO: &str = "video"; pub const MEDIA_TYPE_AUDIO: &str = "audio"; diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/yuv_helper.rs b/crates/livekit-webrtc/libwebrtc-sys/src/yuv_helper.rs new file mode 100644 index 0000000..69da044 --- /dev/null +++ b/crates/livekit-webrtc/libwebrtc-sys/src/yuv_helper.rs @@ -0,0 +1,19 @@ +#[cxx::bridge(namespace = "livekit")] +pub mod ffi { + unsafe extern "C++" { + include!("livekit/yuv_helper.h"); + + unsafe fn i420_to_abgr( + src_y: *const u8, + src_stride_y: i32, + src_u: *const u8, + src_stride_u: i32, + src_v: *const u8, + src_stride_v: i32, + dst_abgr: *mut u8, + dst_stride_abgr: i32, + width: i32, + height: i32, + ); + } +} diff --git a/crates/livekit-webrtc/src/lib.rs b/crates/livekit-webrtc/src/lib.rs index 684388d..951b6ac 100644 --- a/crates/livekit-webrtc/src/lib.rs +++ b/crates/livekit-webrtc/src/lib.rs @@ -9,3 +9,4 @@ pub mod rtp_transceiver; pub mod video_frame; pub mod video_frame_buffer; pub mod webrtc; +pub mod yuv_helper; diff --git a/crates/livekit-webrtc/src/video_frame_buffer.rs b/crates/livekit-webrtc/src/video_frame_buffer.rs index 64cf28d..c85a123 100644 --- a/crates/livekit-webrtc/src/video_frame_buffer.rs +++ b/crates/livekit-webrtc/src/video_frame_buffer.rs @@ -1,5 +1,6 @@ use cxx::UniquePtr; use libwebrtc_sys::video_frame_buffer as vfb_sys; +use livekit_utils::enum_dispatch; use std::pin::Pin; use std::slice; use vfb_sys::ffi::VideoFrameBufferType; @@ -53,6 +54,15 @@ impl VideoFrameBuffer { } } +impl VideoFrameBufferTrait for VideoFrameBuffer { + enum_dispatch!( + [Native, I420, I420A, I422, I444, I010, NV12] + fnc!(width, &Self, [], i32); + fnc!(height, &Self, [], i32); + fnc!(to_i420, Self, [], I420Buffer); + ); +} + macro_rules! recursive_cast { ($ptr:expr $(, $fnc:ident)*) => { { @@ -146,21 +156,23 @@ macro_rules! impl_yuv8_buffer { fn data_y(&self) -> &[u8] { let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); unsafe { - slice::from_raw_parts((*ptr).data_y(), self.stride_y().try_into().unwrap()) + slice::from_raw_parts((*ptr).data_y(), (self.width() * self.height()) as usize) } } fn data_u(&self) -> &[u8] { let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); unsafe { - slice::from_raw_parts((*ptr).data_u(), self.stride_u().try_into().unwrap()) + let chroma_height = (self.height() + 1) / 2; + slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * chroma_height) as usize) } } fn data_v(&self) -> &[u8] { let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); unsafe { - slice::from_raw_parts((*ptr).data_v(), self.stride_v().try_into().unwrap()) + let chroma_height = (self.height() + 1) / 2; + slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * chroma_height) as usize) } } } diff --git a/crates/livekit-webrtc/src/yuv_helper.rs b/crates/livekit-webrtc/src/yuv_helper.rs new file mode 100644 index 0000000..91a3f70 --- /dev/null +++ b/crates/livekit-webrtc/src/yuv_helper.rs @@ -0,0 +1,43 @@ +use std::convert::TryInto; + +use libwebrtc_sys::yuv_helper as yuv_sys; + +pub fn i420_to_abgr( + src_y: &[u8], + src_stride_y: i32, + src_u: &[u8], + src_stride_u: i32, + src_v: &[u8], + src_stride_v: i32, + dst_abgr: &mut [u8], + dst_stride_abgr: i32, + width: i32, + height: i32, +) { + // Assert minimum capacity for safety + let chroma_height = (height + 1) / 2; // the buffer should be padded? + let min_y: usize = (src_stride_y * height).try_into().unwrap(); + let min_u: usize = (src_stride_u * chroma_height).try_into().unwrap(); + let min_v: usize = (src_stride_v * chroma_height).try_into().unwrap(); + let min_abgr: usize = (dst_stride_abgr * height).try_into().unwrap(); + + assert!(src_y.len() >= min_y); + assert!(src_u.len() >= min_u); + assert!(src_v.len() >= min_v); + assert!(dst_abgr.len() >= min_abgr); + + unsafe { + yuv_sys::ffi::i420_to_abgr( + src_y.as_ptr(), + src_stride_y, + src_u.as_ptr(), + src_stride_u, + src_v.as_ptr(), + src_stride_v, + dst_abgr.as_mut_ptr(), + dst_stride_abgr, + width, + height, + ); + } +} diff --git a/examples/Cargo.lock b/examples/Cargo.lock index c82ce3c..35e7fa4 100644 --- a/examples/Cargo.lock +++ b/examples/Cargo.lock @@ -2,6 +2,50 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "ab_glyph" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcdbc68024b653943864d436fe8a24b028095bc1cf91a8926f8241e4aaffe59" +dependencies = [ + "ab_glyph_rasterizer", + "owned_ttf_parser", +] + +[[package]] +name = "ab_glyph_rasterizer" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330223a1aecc308757b9926e9391c9b47f8ef2dbd8aea9df88312aea18c5e8d6" + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "ahash" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +dependencies = [ + "getrandom", + "once_cell", + "version_check", +] + +[[package]] +name = "ahash" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf6ccdb167abbf410dcb915cabd428929d7f6a04980b54a11f26a39f1c7f7107" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "0.7.19" @@ -11,6 +55,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "ansi_term" version = "0.12.1" @@ -26,6 +79,57 @@ version = "1.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98161a4e3e2184da77bb14f02184cdd111e83bbbcc9979dfee3c44b9a85f5602" +[[package]] +name = "arboard" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6041616acea41d67c4a984709ddab1587fd0b10efe5cc563fee954d2f011854" +dependencies = [ + "clipboard-win", + "log", + "objc", + "objc-foundation", + "objc_id", + "once_cell", + "parking_lot", + "thiserror", + "winapi", + "x11rb", +] + +[[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" + +[[package]] +name = "arrayvec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" + +[[package]] +name = "arrayvec" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" + +[[package]] +name = "ash" +version = "0.37.1+1.3.235" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "911015c962d56e2e4052f40182ca5462ba60a3d2ff04e827c365a0ab3d65726d" +dependencies = [ + "libloading", +] + +[[package]] +name = "atomic_refcell" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b5e5f48b927f04e952dedc932f31995a65a0bf65ec971c74436e51bf6e970d" + [[package]] name = "autocfg" version = "1.1.0" @@ -38,12 +142,33 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bitflags" version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "block" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a" + [[package]] name = "block-buffer" version = "0.10.3" @@ -53,6 +178,32 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bumpalo" +version = "3.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" + +[[package]] +name = "bytemuck" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaa3a8d9a1ca92e282c96a32d6511b695d7d994d1d102ba85d279f9b2756947f" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fe233b960f12f8007e3db2d136e3cb1c291bfd7396e384ee76025fc1a3932b4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "byteorder" version = "1.4.3" @@ -65,18 +216,94 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" +[[package]] +name = "calloop" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bcf530afb40e45e14440701e5e996d7fd139e84a912a4d83a8d6a0fb3e58663" +dependencies = [ + "log", + "nix 0.25.0", + "slotmap", + "thiserror", + "vec_map", +] + [[package]] name = "cc" version = "1.0.73" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + +[[package]] +name = "clipboard-win" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4ab1b92798304eedc095b53942963240037c0516452cb11aeba709d420b2219" +dependencies = [ + "error-code", + "str-buf", + "winapi", +] + +[[package]] +name = "cmake" +version = "0.1.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db34956e100b30725f2eb215f90d4871051239535632f84fea3bc92722c66b7c" +dependencies = [ + "cc", +] + +[[package]] +name = "cocoa" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f425db7937052c684daec3bd6375c8abe2d146dca4b8b143d6db777c39138f3a" +dependencies = [ + "bitflags", + "block", + "cocoa-foundation", + "core-foundation", + "core-graphics", + "foreign-types 0.3.2", + "libc", + "objc", +] + +[[package]] +name = "cocoa-foundation" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ade49b65d560ca58c403a479bb396592b155c0185eada742ee323d1d68d6318" +dependencies = [ + "bitflags", + "block", + "core-foundation", + "core-graphics-types", + "foreign-types 0.3.2", + "libc", + "objc", +] + [[package]] name = "codespan-reporting" version = "0.11.1" @@ -87,6 +314,16 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "combine" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "core-foundation" version = "0.9.3" @@ -103,6 +340,43 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +[[package]] +name = "core-graphics" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2581bbab3b8ffc6fcbd550bf46c355135d16e9ff2a6ea032ad6b9bf1d7efe4fb" +dependencies = [ + "bitflags", + "core-foundation", + "core-graphics-types", + "foreign-types 0.3.2", + "libc", +] + +[[package]] +name = "core-graphics-types" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a68b68b3446082644c91ac778bf50cd4104bfb002b5a6a7c44cca5a2c70788b" +dependencies = [ + "bitflags", + "core-foundation", + "foreign-types 0.3.2", + "libc", +] + +[[package]] +name = "core-text" +version = "19.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d74ada66e07c1cefa18f8abfba765b486f250de2e4a999e5727fc0dd4b4a25" +dependencies = [ + "core-foundation", + "core-graphics", + "foreign-types 0.3.2", + "libc", +] + [[package]] name = "cpufeatures" version = "0.2.5" @@ -112,6 +386,38 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossfont" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21fd3add36ea31aba1520aa5288714dd63be506106753226d0eb387a93bc9c45" +dependencies = [ + "cocoa", + "core-foundation", + "core-foundation-sys", + "core-graphics", + "core-text", + "dwrote", + "foreign-types 0.5.0", + "freetype-rs", + "libc", + "log", + "objc", + "once_cell", + "pkg-config", + "servo-fontconfig", + "winapi", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -122,6 +428,12 @@ dependencies = [ "typenum", ] +[[package]] +name = "cty" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35" + [[package]] name = "cxx" version = "1.0.78" @@ -166,6 +478,52 @@ dependencies = [ "syn", ] +[[package]] +name = "d3d12" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "827914e1f53b1e0e025ecd3d967a7836b7bcb54520f90e21ef8df7b4d88a2759" +dependencies = [ + "bitflags", + "libloading", + "winapi", +] + +[[package]] +name = "darling" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" +dependencies = [ + "darling_core", + "quote", + "syn", +] + [[package]] name = "digest" version = "0.10.5" @@ -176,12 +534,169 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "dispatch" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd0c93bb4b0c6d9b77f4435b0ae98c24d17f1c45b2ff844c6151a07256ca923b" + +[[package]] +name = "dlib" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac1b7517328c04c2aa68422fc60a41b92208182142ed04a25879c26c8f878794" +dependencies = [ + "libloading", +] + +[[package]] +name = "downcast-rs" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650" + +[[package]] +name = "dwrote" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439a1c2ba5611ad3ed731280541d36d2e9c4ac5e7fb818a27b604bdc5a6aa65b" +dependencies = [ + "lazy_static", + "libc", + "serde", + "serde_derive", + "winapi", + "wio", +] + +[[package]] +name = "egui" +version = "0.19.0" +source = "git+https://github.com/emilk/egui#0336816faf9f361e59ca77f99c19fcfa2bf7c993" +dependencies = [ + "ahash 0.8.2", + "epaint", + "nohash-hasher", + "tracing", +] + +[[package]] +name = "egui-wgpu" +version = "0.19.0" +source = "git+https://github.com/emilk/egui#0336816faf9f361e59ca77f99c19fcfa2bf7c993" +dependencies = [ + "bytemuck", + "egui", + "pollster", + "tracing", + "type-map", + "wgpu", + "winit", +] + +[[package]] +name = "egui-winit" +version = "0.19.0" +source = "git+https://github.com/emilk/egui#0336816faf9f361e59ca77f99c19fcfa2bf7c993" +dependencies = [ + "arboard", + "egui", + "instant", + "smithay-clipboard", + "tracing", + "webbrowser", + "winit", +] + +[[package]] +name = "egui_demo_lib" +version = "0.19.0" +source = "git+https://github.com/emilk/egui#0336816faf9f361e59ca77f99c19fcfa2bf7c993" +dependencies = [ + "egui", + "egui_extras", + "enum-map", + "tracing", + "unicode_names2", +] + +[[package]] +name = "egui_extras" +version = "0.19.0" +source = "git+https://github.com/emilk/egui#0336816faf9f361e59ca77f99c19fcfa2bf7c993" +dependencies = [ + "egui", +] + [[package]] name = "either" version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" +[[package]] +name = "emath" +version = "0.19.0" +source = "git+https://github.com/emilk/egui#0336816faf9f361e59ca77f99c19fcfa2bf7c993" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "enum-map" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5a56d54c8dd9b3ad34752ed197a4eb2a6601bc010808eb097a04a58ae4c43e1" +dependencies = [ + "enum-map-derive", + "serde", +] + +[[package]] +name = "enum-map-derive" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9045e2676cd5af83c3b167d917b0a5c90a4d8e266e2683d6631b235c457fc27" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "epaint" +version = "0.19.0" +source = "git+https://github.com/emilk/egui#0336816faf9f361e59ca77f99c19fcfa2bf7c993" +dependencies = [ + "ab_glyph", + "ahash 0.8.2", + "atomic_refcell", + "bytemuck", + "emath", + "nohash-hasher", + "parking_lot", +] + +[[package]] +name = "error-code" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64f18991e7bf11e7ffee451b5318b5c1a73c52d0d0ada6e5a3017c8c1ced6a21" +dependencies = [ + "libc", + "str-buf", +] + +[[package]] +name = "expat-sys" +version = "2.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "658f19728920138342f68408b7cf7644d90d4784353d8ebc32e7e8663dbe45fa" +dependencies = [ + "cmake", + "pkg-config", +] + [[package]] name = "fastrand" version = "1.8.0" @@ -197,6 +712,16 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "flate2" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -209,7 +734,28 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" dependencies = [ - "foreign-types-shared", + "foreign-types-shared 0.1.1", +] + +[[package]] +name = "foreign-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" +dependencies = [ + "foreign-types-macros", + "foreign-types-shared 0.3.1", +] + +[[package]] +name = "foreign-types-macros" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8469d0d40519bc608ec6863f1cc88f3f1deee15913f2f3b3e573d81ed38cccc" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -218,6 +764,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" +[[package]] +name = "foreign-types-shared" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" + [[package]] name = "form_urlencoded" version = "1.1.0" @@ -227,6 +779,28 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "freetype-rs" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74eadec9d0a5c28c54bb9882e54787275152a4e36ce206b45d7451384e5bf5fb" +dependencies = [ + "bitflags", + "freetype-sys", + "libc", +] + +[[package]] +name = "freetype-sys" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a37d4011c0cc628dfa766fcc195454f4b068d7afdc2adfd28861191d866e731a" +dependencies = [ + "cmake", + "libc", + "pkg-config", +] + [[package]] name = "futures" version = "0.3.24" @@ -316,6 +890,15 @@ dependencies = [ "slab", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "generic-array" version = "0.14.6" @@ -326,6 +909,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "gethostname" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ebd34e35c46e00bb73e81363248d627782724609fe1b6396f553f68fe3862e" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "getrandom" version = "0.2.7" @@ -343,11 +936,65 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +[[package]] +name = "glow" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8bd5877156a19b8ac83a29b2306fe20537429d318f3ff0a1a2119f8d9c61919" +dependencies = [ + "js-sys", + "slotmap", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "gpu-alloc" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fc59e5f710e310e76e6707f86c561dd646f69a8876da9131703b2f717de818d" +dependencies = [ + "bitflags", + "gpu-alloc-types", +] + +[[package]] +name = "gpu-alloc-types" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54804d0d6bc9d7f26db4eaec1ad10def69b599315f487d32c334a80d1efe67a5" +dependencies = [ + "bitflags", +] + +[[package]] +name = "gpu-descriptor" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b0c02e1ba0bdb14e965058ca34e09c020f8e507a760df1121728e0aef68d57a" +dependencies = [ + "bitflags", + "gpu-descriptor-types", + "hashbrown", +] + +[[package]] +name = "gpu-descriptor-types" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "363e3677e55ad168fef68cf9de3a4a310b53124c5e784c53a1d70e92d23f2126" +dependencies = [ + "bitflags", +] + [[package]] name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash 0.7.6", +] [[package]] name = "heck" @@ -364,6 +1011,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hexf-parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" + [[package]] name = "http" version = "0.2.8" @@ -381,6 +1034,12 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "0.3.0" @@ -408,6 +1067,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", ] [[package]] @@ -425,6 +1087,46 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754" +[[package]] +name = "jni" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "039022cdf4d7b1cf548d31f60ae783138e5fd42013f6271049d7df7afadef96c" +dependencies = [ + "cesu8", + "combine", + "jni-sys", + "log", + "thiserror", + "walkdir", +] + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" + +[[package]] +name = "js-sys" +version = "0.3.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "khronos-egl" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c2352bd1d0bceb871cb9d40f24360c8133c11d7486b68b5381c1dd1a32015e3" +dependencies = [ + "libc", + "libloading", + "pkg-config", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -437,6 +1139,16 @@ version = "0.2.134" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "329c933548736bc49fd575ee68c89e8be4d260064184389a5b77517cddd99ffb" +[[package]] +name = "libloading" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +dependencies = [ + "cfg-if", + "winapi", +] + [[package]] name = "libwebrtc-sys" version = "0.1.0" @@ -462,6 +1174,7 @@ name = "livekit" version = "0.1.0" dependencies = [ "livekit-core", + "livekit-webrtc", ] [[package]] @@ -471,6 +1184,7 @@ dependencies = [ "futures", "futures-util", "lazy_static", + "livekit-utils", "livekit-webrtc", "parking_lot", "prost", @@ -485,12 +1199,17 @@ dependencies = [ "url", ] +[[package]] +name = "livekit-utils" +version = "0.1.0" + [[package]] name = "livekit-webrtc" version = "0.1.0" dependencies = [ "cxx", "libwebrtc-sys", + "livekit-utils", "log", "thiserror", "tokio", @@ -515,12 +1234,68 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "malloc_buf" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62bb907fe88d54d8d9ce32a3cceab4218ed2f6b7d35617cafe9adf84e43919cb" +dependencies = [ + "libc", +] + [[package]] name = "memchr" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "memmap2" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b182332558b18d807c4ce1ca8ca983b34c3ee32765e47b3f0f69b90355cc1dc" +dependencies = [ + "libc", +] + +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + +[[package]] +name = "metal" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de11355d1f6781482d027a3b4d4de7825dcedb197bf573e0596d00008402d060" +dependencies = [ + "bitflags", + "block", + "core-graphics-types", + "foreign-types 0.3.2", + "log", + "objc", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +dependencies = [ + "adler", +] + [[package]] name = "mio" version = "0.8.4" @@ -539,6 +1314,26 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +[[package]] +name = "naga" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "262d2840e72dbe250e8cf2f522d080988dfca624c4112c096238a4845f591707" +dependencies = [ + "bit-set", + "bitflags", + "codespan-reporting", + "hexf-parse", + "indexmap", + "log", + "num-traits", + "rustc-hash", + "spirv", + "termcolor", + "thiserror", + "unicode-xid", +] + [[package]] name = "native-tls" version = "0.2.10" @@ -557,6 +1352,114 @@ dependencies = [ "tempfile", ] +[[package]] +name = "ndk" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "451422b7e4718271c8b5b3aadf5adedba43dc76312454b387e98fae0fc951aa0" +dependencies = [ + "bitflags", + "jni-sys", + "ndk-sys", + "num_enum", + "raw-window-handle 0.5.0", + "thiserror", +] + +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + +[[package]] +name = "ndk-glue" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0434fabdd2c15e0aab768ca31d5b7b333717f03cf02037d5a0a3ff3c278ed67f" +dependencies = [ + "libc", + "log", + "ndk", + "ndk-context", + "ndk-macro", + "ndk-sys", + "once_cell", + "parking_lot", +] + +[[package]] +name = "ndk-macro" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0df7ac00c4672f9d5aece54ee3347520b7e20f158656c7db2e6de01902eb7a6c" +dependencies = [ + "darling", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "ndk-sys" +version = "0.4.1+23.1.7779620" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cf2aae958bd232cac5069850591667ad422d263686d75b52a065f9badeee5a3" +dependencies = [ + "jni-sys", +] + +[[package]] +name = "nix" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "195cdbc1741b8134346d515b3a56a1c94b0912758009cfd53f99ea0f57b065fc" +dependencies = [ + "bitflags", + "cfg-if", + "libc", + "memoffset", +] + +[[package]] +name = "nix" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e322c04a9e3440c327fca7b6c8a63e6890a32fa2ad689db972425f07e0d22abb" +dependencies = [ + "autocfg", + "bitflags", + "cfg-if", + "libc", + "memoffset", +] + +[[package]] +name = "nohash-hasher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" + +[[package]] +name = "nom" +version = "7.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.13.1" @@ -567,6 +1470,66 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf5395665662ef45796a4ff5486c5d41d29e0c09640af4c5f17fd94ee2c119c9" +dependencies = [ + "num_enum_derive", +] + +[[package]] +name = "num_enum_derive" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0498641e53dd6ac1a4f22547548caa6864cc4933784319cd1775271c5a46ce" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "objc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1" +dependencies = [ + "malloc_buf", + "objc_exception", +] + +[[package]] +name = "objc-foundation" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1add1b659e36c9607c7aab864a76c7a4c2760cd0cd2e120f3fb8b952c7e22bf9" +dependencies = [ + "block", + "objc", + "objc_id", +] + +[[package]] +name = "objc_exception" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad970fb455818ad6cba4c122ad012fae53ae8b4795f86378bce65e4f6bab2ca4" +dependencies = [ + "cc", +] + +[[package]] +name = "objc_id" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92d4ddb4bd7b50d730c215ff871754d0da6b2178849f8a2a2ab69712d0c073b" +dependencies = [ + "objc", +] + [[package]] name = "once_cell" version = "1.15.0" @@ -581,7 +1544,7 @@ checksum = "12fc0523e3bd51a692c8850d075d74dc062ccf251c0110668cbd921917118a13" dependencies = [ "bitflags", "cfg-if", - "foreign-types", + "foreign-types 0.3.2", "libc", "once_cell", "openssl-macros", @@ -618,6 +1581,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "owned_ttf_parser" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18904d3c65493a9f0d7542293d1a7f69bfdc309a6b9ef4f46dc3e58b0577edc5" +dependencies = [ + "ttf-parser", +] + [[package]] name = "parking_lot" version = "0.12.1" @@ -675,12 +1647,41 @@ version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" +[[package]] +name = "png" +version = "0.17.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d708eaf860a19b19ce538740d2b4bdeeb8337fa53f7738455e706623ad5c638" +dependencies = [ + "bitflags", + "crc32fast", + "flate2", + "miniz_oxide", +] + +[[package]] +name = "pollster" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5da3b0203fd7ee5720aa0b5e790b591aa5d3f41c3ed2c34a3a393382198af2f7" + [[package]] name = "ppv-lite86" version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" +[[package]] +name = "proc-macro-crate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eda0fc3b0fb7c975631757e14d9049da17374063edb6ebbcbc54d880d4fe94e9" +dependencies = [ + "once_cell", + "thiserror", + "toml", +] + [[package]] name = "proc-macro2" version = "1.0.46" @@ -690,6 +1691,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "profiling" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74605f360ce573babfe43964cbe520294dcb081afbf8c108fc6e23036b4da2df" + [[package]] name = "prost" version = "0.11.0" @@ -782,6 +1789,30 @@ dependencies = [ "getrandom", ] +[[package]] +name = "range-alloc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e935c45e09cc6dcf00d2f0b2d630a58f4095320223d47fc68918722f0538b6" + +[[package]] +name = "raw-window-handle" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b800beb9b6e7d2df1fe337c9e3d04e3af22a124460fb4c30fcc22c9117cefb41" +dependencies = [ + "cty", +] + +[[package]] +name = "raw-window-handle" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed7e3d950b66e19e0c372f3fa3fbbcf85b1746b571f74e0c2af6042a5c93420a" +dependencies = [ + "cty", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -817,12 +1848,42 @@ dependencies = [ "winapi", ] +[[package]] +name = "renderdoc-sys" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1382d1f0a252c4bf97dc20d979a2fdd05b024acd7c2ed0f7595d7817666a157" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "ryu" version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" +[[package]] +name = "safe_arch" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ff3d6d9696af502cc3110dacce942840fb06ff4514cad92236ecc455f2ce05" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.20" @@ -833,6 +1894,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.1.0" @@ -845,6 +1912,18 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" +[[package]] +name = "sctk-adwaita" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61270629cc6b4d77ec1907db1033d5c2e1a404c412743621981a871dc9c12339" +dependencies = [ + "crossfont", + "log", + "smithay-client-toolkit", + "tiny-skia", +] + [[package]] name = "security-framework" version = "2.7.0" @@ -899,6 +1978,27 @@ dependencies = [ "serde", ] +[[package]] +name = "servo-fontconfig" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7e3e22fe5fd73d04ebf0daa049d3efe3eae55369ce38ab16d07ddd9ac5c217c" +dependencies = [ + "libc", + "servo-fontconfig-sys", +] + +[[package]] +name = "servo-fontconfig-sys" +version = "5.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e36b879db9892dfa40f95da1c38a835d41634b825fbd8c4c418093d53c24b388" +dependencies = [ + "expat-sys", + "freetype-sys", + "pkg-config", +] + [[package]] name = "sha-1" version = "0.10.0" @@ -932,11 +2032,17 @@ dependencies = [ name = "simple_room" version = "0.1.0" dependencies = [ + "egui", + "egui-wgpu", + "egui-winit", + "egui_demo_lib", "futures", "livekit", "tokio", "tracing", "tracing-subscriber", + "wgpu", + "winit", ] [[package]] @@ -948,12 +2054,50 @@ dependencies = [ "autocfg", ] +[[package]] +name = "slotmap" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e08e261d0e8f5c43123b7adf3e4ca1690d655377ac93a03b2c9d3e98de1342" +dependencies = [ + "version_check", +] + [[package]] name = "smallvec" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1" +[[package]] +name = "smithay-client-toolkit" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f307c47d32d2715eb2e0ece5589057820e0e5e70d07c247d1063e844e107f454" +dependencies = [ + "bitflags", + "calloop", + "dlib", + "lazy_static", + "log", + "memmap2", + "nix 0.24.2", + "pkg-config", + "wayland-client", + "wayland-cursor", + "wayland-protocols", +] + +[[package]] +name = "smithay-clipboard" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a345c870a1fae0b1b779085e81b51e614767c239e93503588e54c5b17f4b0e8" +dependencies = [ + "smithay-client-toolkit", + "wayland-client", +] + [[package]] name = "socket2" version = "0.4.7" @@ -964,6 +2108,34 @@ dependencies = [ "winapi", ] +[[package]] +name = "spirv" +version = "0.2.0+1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "246bfa38fe3db3f1dfc8ca5a2cdeb7348c78be2112740cc0ec8ef18b6d94f830" +dependencies = [ + "bitflags", + "num-traits", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "str-buf" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e08d8363704e6c71fc928674353e6b7c23dcea9d82d7012c8faf2a3a025f8d0" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "syn" version = "1.0.101" @@ -1027,6 +2199,31 @@ dependencies = [ "once_cell", ] +[[package]] +name = "tiny-skia" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "642680569bb895b16e4b9d181c60be1ed136fa0c9c7f11d004daf053ba89bf82" +dependencies = [ + "arrayref", + "arrayvec 0.5.2", + "bytemuck", + "cfg-if", + "png", + "safe_arch", + "tiny-skia-path", +] + +[[package]] +name = "tiny-skia-path" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c114d32f0c2ee43d585367cb013dfaba967ab9f62b90d9af0d696e955e70fa6c" +dependencies = [ + "arrayref", + "bytemuck", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -1097,6 +2294,15 @@ dependencies = [ "tungstenite", ] +[[package]] +name = "toml" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" +dependencies = [ + "serde", +] + [[package]] name = "tracing" version = "0.1.36" @@ -1155,6 +2361,12 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "ttf-parser" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "375812fa44dab6df41c195cd2f7fecb488f6c09fbaafb62807488cefab642bff" + [[package]] name = "tungstenite" version = "0.17.3" @@ -1175,6 +2387,15 @@ dependencies = [ "utf-8", ] +[[package]] +name = "type-map" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deb68604048ff8fa93347f02441e4487594adc20bb8a084f9e564d2b827a0a9f" +dependencies = [ + "rustc-hash", +] + [[package]] name = "typenum" version = "1.15.0" @@ -1208,6 +2429,18 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +[[package]] +name = "unicode-xid" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" + +[[package]] +name = "unicode_names2" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "029df4cc8238cefc911704ff8fa210853a0f3bce2694d8f51181dd41ee0f3301" + [[package]] name = "url" version = "2.3.1" @@ -1237,18 +2470,294 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + [[package]] name = "version_check" version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "walkdir" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" + +[[package]] +name = "wayland-client" +version = "0.29.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3b068c05a039c9f755f881dc50f01732214f5685e379829759088967c46715" +dependencies = [ + "bitflags", + "downcast-rs", + "libc", + "nix 0.24.2", + "scoped-tls", + "wayland-commons", + "wayland-scanner", + "wayland-sys", +] + +[[package]] +name = "wayland-commons" +version = "0.29.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8691f134d584a33a6606d9d717b95c4fa20065605f798a3f350d78dced02a902" +dependencies = [ + "nix 0.24.2", + "once_cell", + "smallvec", + "wayland-sys", +] + +[[package]] +name = "wayland-cursor" +version = "0.29.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6865c6b66f13d6257bef1cd40cbfe8ef2f150fb8ebbdb1e8e873455931377661" +dependencies = [ + "nix 0.24.2", + "wayland-client", + "xcursor", +] + +[[package]] +name = "wayland-protocols" +version = "0.29.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b950621f9354b322ee817a23474e479b34be96c2e909c14f7bc0100e9a970bc6" +dependencies = [ + "bitflags", + "wayland-client", + "wayland-commons", + "wayland-scanner", +] + +[[package]] +name = "wayland-scanner" +version = "0.29.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f4303d8fa22ab852f789e75a967f0a2cdc430a607751c0499bada3e451cbd53" +dependencies = [ + "proc-macro2", + "quote", + "xml-rs", +] + +[[package]] +name = "wayland-sys" +version = "0.29.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be12ce1a3c39ec7dba25594b97b42cb3195d54953ddb9d3d95a7c3902bc6e9d4" +dependencies = [ + "dlib", + "lazy_static", + "pkg-config", +] + +[[package]] +name = "web-sys" +version = "0.3.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webbrowser" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0cc7962b5aaa0dfcebaeef0161eec6edf5f4606c12e6777fd7d392f52033a5" +dependencies = [ + "jni", + "ndk-context", + "objc", + "raw-window-handle 0.5.0", + "url", + "web-sys", + "widestring", + "winapi", +] + +[[package]] +name = "wgpu" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2272b17bffc8a0c7d53897435da7c1db587c87d3a14e8dae9cdb8d1d210fc0f" +dependencies = [ + "arrayvec 0.7.2", + "js-sys", + "log", + "naga", + "parking_lot", + "raw-window-handle 0.5.0", + "smallvec", + "static_assertions", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "wgpu-core", + "wgpu-hal", + "wgpu-types", +] + +[[package]] +name = "wgpu-core" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73d14cad393054caf992ee02b7da6a372245d39a484f7461c1f44f6f6359bd28" +dependencies = [ + "arrayvec 0.7.2", + "bit-vec", + "bitflags", + "cfg_aliases", + "codespan-reporting", + "fxhash", + "log", + "naga", + "parking_lot", + "profiling", + "raw-window-handle 0.5.0", + "smallvec", + "thiserror", + "web-sys", + "wgpu-hal", + "wgpu-types", +] + +[[package]] +name = "wgpu-hal" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cc320a61acb26be4f549c9b1b53405c10a223fbfea363ec39474c32c348d12f" +dependencies = [ + "android_system_properties", + "arrayvec 0.7.2", + "ash", + "bit-set", + "bitflags", + "block", + "core-graphics-types", + "d3d12", + "foreign-types 0.3.2", + "fxhash", + "glow", + "gpu-alloc", + "gpu-descriptor", + "js-sys", + "khronos-egl", + "libloading", + "log", + "metal", + "naga", + "objc", + "parking_lot", + "profiling", + "range-alloc", + "raw-window-handle 0.5.0", + "renderdoc-sys", + "smallvec", + "thiserror", + "wasm-bindgen", + "web-sys", + "wgpu-types", + "winapi", +] + +[[package]] +name = "wgpu-types" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb6b28ef22cac17b9109b25b3bf8c9a103eeb293d7c5f78653979b09140375f6" +dependencies = [ + "bitflags", +] + [[package]] name = "which" version = "4.3.0" @@ -1260,6 +2769,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "widestring" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "653f141f39ec16bba3c5abe400a0c60da7468261cc2cbf36805022876bc721a8" + [[package]] name = "winapi" version = "0.3.9" @@ -1285,6 +2800,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "winapi-wsapoll" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c17110f57155602a80dca10be03852116403c9ff3cd25b079d666f2aa3df6e" +dependencies = [ + "winapi", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -1333,3 +2857,93 @@ name = "windows_x86_64_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" + +[[package]] +name = "winit" +version = "0.27.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb796d6fbd86b2fd896c9471e6f04d39d750076ebe5680a3958f00f5ab97657c" +dependencies = [ + "bitflags", + "cocoa", + "core-foundation", + "core-graphics", + "dispatch", + "instant", + "libc", + "log", + "mio", + "ndk", + "ndk-glue", + "objc", + "once_cell", + "parking_lot", + "percent-encoding", + "raw-window-handle 0.4.3", + "raw-window-handle 0.5.0", + "sctk-adwaita", + "smithay-client-toolkit", + "wasm-bindgen", + "wayland-client", + "wayland-protocols", + "web-sys", + "windows-sys", + "x11-dl", +] + +[[package]] +name = "wio" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d129932f4644ac2396cb456385cbf9e63b5b30c6e8dc4820bdca4eb082037a5" +dependencies = [ + "winapi", +] + +[[package]] +name = "x11-dl" +version = "2.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1536d6965a5d4e573c7ef73a2c15ebcd0b2de3347bdf526c34c297c00ac40f0" +dependencies = [ + "lazy_static", + "libc", + "pkg-config", +] + +[[package]] +name = "x11rb" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "592b4883219f345e712b3209c62654ebda0bb50887f330cbd018d0f654bfd507" +dependencies = [ + "gethostname", + "nix 0.24.2", + "winapi", + "winapi-wsapoll", + "x11rb-protocol", +] + +[[package]] +name = "x11rb-protocol" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56b245751c0ac9db0e006dc812031482784e434630205a93c73cfefcaabeac67" +dependencies = [ + "nix 0.24.2", +] + +[[package]] +name = "xcursor" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "463705a63313cd4301184381c5e8042f0a7e9b4bb63653f216311d4ae74690b7" +dependencies = [ + "nom", +] + +[[package]] +name = "xml-rs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3" diff --git a/examples/Cargo.toml b/examples/Cargo.toml index 5f00359..6696429 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -1,3 +1,4 @@ [workspace] members = ["*"] -exclude = ["target"] \ No newline at end of file +exclude = ["target"] +resolver = "2" diff --git a/examples/simple_room/Cargo.toml b/examples/simple_room/Cargo.toml index aab5f93..d9e266a 100644 --- a/examples/simple_room/Cargo.toml +++ b/examples/simple_room/Cargo.toml @@ -9,3 +9,10 @@ tracing = "0.1" tracing-subscriber = "0.3" livekit = { path = "../.." } futures = "0.3" +wgpu = "0.14.0" +winit = "0.27.5" + +egui = { git = "https://github.com/emilk/egui" } +egui-wgpu = { git = "https://github.com/emilk/egui", features = ["winit"] } +egui-winit = { git = "https://github.com/emilk/egui" } +egui_demo_lib = { git = "https://github.com/emilk/egui" } diff --git a/examples/simple_room/src/main.rs b/examples/simple_room/src/main.rs index 1b49771..a35bbbf 100644 --- a/examples/simple_room/src/main.rs +++ b/examples/simple_room/src/main.rs @@ -1,36 +1,192 @@ -use std::time::Duration; +use std::convert::TryInto; +use std::ops::DerefMut; +use std::{num::NonZeroU32, time::Duration}; + +use egui_wgpu::WgpuConfiguration; +use livekit::webrtc::media_stream::VideoTrack; +use livekit::webrtc::video_frame_buffer::{ + PlanarYuv8Buffer, PlanarYuvBuffer, VideoFrameBufferTrait, +}; +use livekit::webrtc::yuv_helper; +use std::sync::{Arc, Mutex}; +use video_renderer::VideoRenderer; +use wgpu::{Device, Queue}; -use livekit::room::RoomError; -use livekit::room::{track::remote_track::RemoteTrackHandle, Room}; use tokio::time::sleep; +use livekit::room::track::remote_track::RemoteTrackHandle; +use livekit::room::{Room, RoomError}; + const URL: &str = "ws://localhost:7880"; const TOKEN : &str = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjIzODQ4MDY0NzMsImlzcyI6IkFQSXpLYkFTaUNWYWtnSiIsIm5hbWUiOiJuYXRpdmUiLCJuYmYiOjE2NjQ4MDY0NzMsInN1YiI6Im5hdGl2ZSIsInZpZGVvIjp7InJvb21DcmVhdGUiOnRydWUsInJvb21Kb2luIjp0cnVlfX0.BgVdBnq3XFD3_BQHoe1azqjifYysubgFl6Qlzu9IQGI"; // eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjIzODQ4MDY3MzAsImlzcyI6IkFQSXpLYkFTaUNWYWtnSiIsIm5hbWUiOiJ3ZWIiLCJuYmYiOjE2NjQ4MDY3MzAsInN1YiI6IndlYiIsInZpZGVvIjp7InJvb21DcmVhdGUiOnRydWUsInJvb21Kb2luIjp0cnVlfX0.VbDoULjX1CVGZu2sPy3SvWYlVZUBXxQVPmdB9BnmlN4 -#[tokio::main] -async fn main() -> Result<(), RoomError> { - tracing_subscriber::fmt::init(); +mod video_renderer; - let mut room = Room::new(); - room.events() - .on_participant_connected(|_event| async move {}); +use winit::{ + event::*, + event_loop::{ControlFlow, EventLoop}, + window::{Window, WindowBuilder, WindowId}, +}; - room.events().on_track_subscribed(|event| async move { - let track = event.publication.track().unwrap(); - if let RemoteTrackHandle::Video(video_track) = track { - let rtc_track = video_track.rtc_track(); - rtc_track.set_should_receive(true); - rtc_track.on_frame(Box::new(|_frame, _buffer| { - // called on libwebrtc worker_thread - println!("Received frame"); - })); - } - }); - - room.connect(URL, TOKEN).await?; - - sleep(Duration::from_secs(200)).await; - Ok(()) +struct AppState { + room: Room, + demo: egui_demo_lib::DemoWindows, + egui_context: egui::Context, + egui_state: egui_winit::State, + egui_painter: egui_wgpu::winit::Painter, + window: winit::window::Window, +} + +impl AppState { + fn on_event(&mut self, event: Event<'_, T>, control_flow: &mut ControlFlow) { + match event { + Event::WindowEvent { window_id, event } => { + if let Some(flow) = self.on_window_event(window_id, event) { + *control_flow = flow; + } + } + Event::RedrawRequested(window_id) if window_id == self.window.id() => { + self.render(); + } + Event::RedrawEventsCleared => { + self.window.request_redraw(); + } + _ => {} + }; + } + + fn on_window_event( + &mut self, + _window_id: WindowId, + event: WindowEvent<'_>, + ) -> Option { + if self + .egui_state + .on_event(&self.egui_context, &event) + .consumed + { + return None; + } + + match event { + WindowEvent::CloseRequested => Some(ControlFlow::Exit), + WindowEvent::Resized(inner_size) => { + self.egui_painter + .on_window_resized(inner_size.width, inner_size.height); + None + } + WindowEvent::ScaleFactorChanged { new_inner_size, .. } => { + self.egui_painter + .on_window_resized(new_inner_size.width, new_inner_size.height); + None + } + _ => None, + } + } + + fn render(&mut self) { + let raw_inputs = self.egui_state.take_egui_input(&self.window); + let full_output = self.egui_context.run(raw_inputs, |ctx| { + //self.ui(ctx); + }); + let clipped_primitives = self.egui_context.tessellate(full_output.shapes); + + self.egui_painter.paint_and_update_textures( + egui_winit::native_pixels_per_point(&self.window), + egui::Rgba::BLACK, + &clipped_primitives, + &full_output.textures_delta, + ); + + self.egui_state.handle_platform_output( + &self.window, + &self.egui_context, + full_output.platform_output, + ); + } +} + +struct App { + rt: tokio::runtime::Runtime, +} + +impl App { + pub fn new(rt: tokio::runtime::Runtime) -> Self { + Self { rt } + } + + pub fn run(&mut self) { + self.rt.block_on(async { + let event_loop = EventLoop::new(); + let window = WindowBuilder::new().build(&event_loop).unwrap(); + + let egui_context = egui::Context::default(); + let egui_state = egui_winit::State::new(&event_loop); + let mut egui_painter = + egui_wgpu::winit::Painter::new(WgpuConfiguration::default(), 1, 32); + unsafe { + egui_painter.set_window(Some(&window)); + } + + let mut inner = AppState { + room: Room::new(), + demo: egui_demo_lib::DemoWindows::default(), + egui_context, + egui_state, + egui_painter, + window, + }; + + inner + .room + .events() + .on_participant_connected(|_event| async move {}); + + inner.room.events().on_track_subscribed({ + let test = Arc::new(Mutex::new(None)); + + let egui_render = inner.egui_painter.render_state().clone().unwrap(); + + move |event| { + let test = test.clone(); + let egui_render = egui_render.clone(); + + async move { + let track = event.publication.track().unwrap(); + if let RemoteTrackHandle::Video(video_track) = track { + *test.lock().unwrap() = + Some(VideoRenderer::new(egui_render, video_track.rtc_track())) + } + } + } + }); + + inner.room.connect(URL, TOKEN).await.unwrap(); + + tokio::spawn(async { + loop { + println!("Test"); + tokio::time::sleep(Duration::from_secs(5)).await; + } + }); + + tokio::task::block_in_place(move || loop { + event_loop.run(move |event, _, control_flow| { + inner.on_event(event, control_flow); + }); + }); + }); + } +} + +fn main() { + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap(); + + let mut app = App::new(rt); + app.run(); } diff --git a/examples/simple_room/src/video_renderer.rs b/examples/simple_room/src/video_renderer.rs new file mode 100644 index 0000000..124322b --- /dev/null +++ b/examples/simple_room/src/video_renderer.rs @@ -0,0 +1,172 @@ +use livekit::webrtc::media_stream::VideoTrack; +use livekit::webrtc::video_frame_buffer::PlanarYuv8Buffer; +use livekit::webrtc::video_frame_buffer::PlanarYuvBuffer; +use livekit::webrtc::video_frame_buffer::VideoFrameBufferTrait; +use livekit::webrtc::yuv_helper; +use std::convert::TryInto; +use std::num::NonZeroU32; +use std::{ + ops::DerefMut, + sync::{Arc, Mutex}, +}; + +pub struct VideoRenderer { + internal: Arc>, + rtc_track: Arc, +} + +struct RendererInternal { + render_state: egui_wgpu::RenderState, + width: u32, + height: u32, + rgba_data: Vec, + texture: Option, + texture_view: Option, + egui_texture: Option, +} + +impl RendererInternal { + fn ensure_texture_size(&mut self, width: u32, height: u32) { + if self.width == width && self.height == height { + return; + } + + self.width = width; + self.height = height; + self.rgba_data.resize((width * height * 4) as usize, 0); + + self.texture = Some( + self.render_state + .device + .create_texture(&wgpu::TextureDescriptor { + label: Some("lk-videotexture"), + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + dimension: wgpu::TextureDimension::D2, + size: wgpu::Extent3d { + width, + height, + ..Default::default() + }, + sample_count: 1, + mip_level_count: 1, + format: wgpu::TextureFormat::Rgba8UnormSrgb, + }), + ); + + self.texture_view = Some(self.texture.as_mut().unwrap().create_view( + &wgpu::TextureViewDescriptor { + label: Some("lk-videotexture-view"), + format: Some(wgpu::TextureFormat::Rgba8UnormSrgb), + dimension: Some(wgpu::TextureViewDimension::D2), + mip_level_count: NonZeroU32::new(1), + array_layer_count: NonZeroU32::new(1), + ..Default::default() + }, + )); + + if let Some(texture_id) = self.egui_texture { + // Update the existing texture + self.render_state + .renderer + .write() + .update_egui_texture_from_wgpu_texture( + &*self.render_state.device, + self.texture_view.as_ref().unwrap(), + wgpu::FilterMode::Linear, + texture_id, + ); + } else { + self.egui_texture = Some(self.render_state.renderer.write().register_native_texture( + &*self.render_state.device, + self.texture_view.as_ref().unwrap(), + wgpu::FilterMode::Linear, + )); + } + } +} + +impl VideoRenderer { + pub fn new(render_state: egui_wgpu::RenderState, rtc_track: Arc) -> Self { + let internal = Arc::new(Mutex::new(RendererInternal { + render_state, + width: 0, + height: 0, + rgba_data: Vec::default(), + texture: None, + texture_view: None, + egui_texture: None, + })); + + rtc_track.on_frame({ + let internal = internal.clone(); + + Box::new(move |_frame, buffer| { + let mut internal = internal.lock().unwrap(); + let buffer = buffer.to_i420(); + + let width: u32 = buffer.width().try_into().unwrap(); + let height: u32 = buffer.height().try_into().unwrap(); + + internal.ensure_texture_size(width, height); + + let rgba_ptr = internal.rgba_data.deref_mut(); + let rgba_stride = buffer.width() * 4; + + yuv_helper::i420_to_abgr( + buffer.data_y(), + buffer.stride_y(), + buffer.data_u(), + buffer.stride_u(), + buffer.data_v(), + buffer.stride_v(), + rgba_ptr, + rgba_stride, + buffer.width(), + buffer.height(), + ); + + let copy_desc = wgpu::ImageCopyTexture { + texture: internal.texture.as_ref().unwrap(), + mip_level: 0, + origin: wgpu::Origin3d::default(), + aspect: wgpu::TextureAspect::default(), + }; + + let copy_layout = wgpu::ImageDataLayout { + bytes_per_row: Some(NonZeroU32::new(width * 4).unwrap()), + ..Default::default() + }; + + let copy_size = wgpu::Extent3d { + width, + height, + ..Default::default() + }; + + internal.render_state.queue.write_texture( + copy_desc, + &internal.rgba_data, + copy_layout, + copy_size, + ); + + println!("wrote"); + }) + }); + + Self { + rtc_track, + internal, + } + } + + pub fn texture_id(&self) -> Option { + self.internal.lock().unwrap().egui_texture.clone() + } +} + +impl Drop for VideoRenderer { + fn drop(&mut self) { + self.rtc_track.on_frame(Box::new(|_, _| {})); + } +} diff --git a/src/lib.rs b/src/lib.rs index ef89eee..f18044b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,2 +1,6 @@ // export everything inside livekit-core pub use livekit_core::*; + +pub mod webrtc { + pub use livekit_webrtc::*; +}