diff --git a/.cargo/config b/.cargo/config deleted file mode 100644 index 0c17df0..0000000 --- a/.cargo/config +++ /dev/null @@ -1,2 +0,0 @@ -[target.x86_64-pc-windows-msvc] -rustflags = ["-C", "target-feature=+crt-static"] \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 0dd70ea..dc34d0d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -512,6 +512,10 @@ dependencies = [ [[package]] name = "livekit-utils" version = "0.1.0" +dependencies = [ + "parking_lot", + "tokio", +] [[package]] name = "livekit-webrtc" diff --git a/crates/livekit-core/src/events.rs b/crates/livekit-core/src/events.rs deleted file mode 100644 index c85506f..0000000 --- a/crates/livekit-core/src/events.rs +++ /dev/null @@ -1,140 +0,0 @@ -use futures_util::future::BoxFuture; -use thiserror::Error; - -type EventHandler = Box BoxFuture<'static, ()> + Send + Sync>; - -macro_rules! event_setter { - ($fnc:ident, $event:ty) => { - pub fn $fnc(&self, mut callback: F) - where - F: FnMut($event) -> Fut + Send + Sync + 'static, - Fut: Future + Send + 'static, - { - *self.$fnc.lock() = Some(Box::new(move |event| Box::pin(callback(event)))); - } - }; -} - -#[derive(Error, Debug, Clone)] -pub enum TrackError { - #[error("could not find published track with sid: {0}")] - TrackNotFound(String), -} - -pub mod room { - use super::{EventHandler, TrackError}; - use crate::room::id::TrackSid; - use crate::room::participant::remote_participant::RemoteParticipant; - use crate::room::publication::RemoteTrackPublication; - use crate::room::track::remote_track::RemoteTrackHandle; - use crate::room::RoomHandle; - use futures::future::Future; - use parking_lot::Mutex; - use std::sync::Arc; - - #[derive(Clone)] - pub struct ParticipantConnectedEvent { - pub room_handle: RoomHandle, - pub participant: Arc, - } - - #[derive(Clone)] - pub struct ParticipantDisconnectedEvent { - pub room_handle: RoomHandle, - pub participant: Arc, - } - - #[derive(Clone)] - pub struct TrackSubscribedEvent { - pub room_handle: RoomHandle, - pub track: RemoteTrackHandle, - pub publication: RemoteTrackPublication, - pub participant: Arc, - } - - #[derive(Clone)] - pub struct TrackPublishedEvent { - pub room_handle: RoomHandle, - pub publication: RemoteTrackPublication, - pub participant: Arc, - } - - #[derive(Clone)] - pub struct TrackSubscriptionFailedEvent { - pub room_handle: RoomHandle, - pub error: TrackError, - pub sid: TrackSid, - pub participant: Arc, - } - - pub(crate) type OnParticipantConnectedHandler = EventHandler; - pub(crate) type OnParticipantDisconnectedHandler = EventHandler; - pub(crate) type OnTrackSubscribedEventHandler = EventHandler; - pub(crate) type OnTrackPublishedEventHandler = EventHandler; - pub(crate) type OnTrackSubscriptionFailedHandler = EventHandler; - - #[derive(Default)] - pub struct RoomEvents { - pub(crate) on_participant_connected: Mutex>, - pub(crate) on_participant_disconnected: Mutex>, - pub(crate) on_track_subscribed: Mutex>, - pub(crate) on_track_published: Mutex>, - pub(crate) on_track_subscription_failed: Mutex>, - } - - impl RoomEvents { - event_setter!(on_participant_connected, ParticipantConnectedEvent); - event_setter!(on_participant_disconnected, ParticipantDisconnectedEvent); - event_setter!(on_track_subscribed, TrackSubscribedEvent); - event_setter!(on_track_published, TrackPublishedEvent); - event_setter!(on_track_subscription_failed, TrackSubscriptionFailedEvent); - } -} - -pub mod participant { - use super::{EventHandler, TrackError}; - use crate::room::id::TrackSid; - use crate::room::participant::remote_participant::RemoteParticipant; - use crate::room::publication::RemoteTrackPublication; - use crate::room::track::remote_track::RemoteTrackHandle; - use futures::future::Future; - use parking_lot::Mutex; - use std::sync::Arc; - - #[derive(Clone)] - pub struct TrackPublishedEvent { - pub publication: RemoteTrackPublication, - pub participant: Arc, - } - - #[derive(Clone)] - pub struct TrackSubscribedEvent { - pub track: RemoteTrackHandle, - pub publication: RemoteTrackPublication, - pub participant: Arc, - } - - #[derive(Clone)] - pub struct TrackSubscriptionFailedEvent { - pub sid: TrackSid, - pub error: TrackError, - pub participant: Arc, - } - - pub(crate) type TrackPublishedHandler = EventHandler; - pub(crate) type TrackSubscribedHandler = EventHandler; - pub(crate) type TrackSubscriptionFailedHandler = EventHandler; - - #[derive(Default)] - pub struct ParticipantEvents { - pub(crate) on_track_published: Mutex>, - pub(crate) on_track_subscribed: Mutex>, - pub(crate) on_track_subscription_failed: Mutex>, - } - - impl ParticipantEvents { - event_setter!(on_track_published, TrackPublishedEvent); - event_setter!(on_track_subscribed, TrackSubscribedEvent); - event_setter!(on_track_subscription_failed, TrackSubscriptionFailedEvent); - } -} diff --git a/crates/livekit-core/src/lib.rs b/crates/livekit-core/src/lib.rs index cbdb92a..cb1ca08 100644 --- a/crates/livekit-core/src/lib.rs +++ b/crates/livekit-core/src/lib.rs @@ -4,7 +4,6 @@ pub mod proto { include!(concat!(env!("OUT_DIR"), "/livekit.rs")); } -mod events; mod rtc_engine; mod signal_client; diff --git a/crates/livekit-core/src/room/mod.rs b/crates/livekit-core/src/room/mod.rs index f9305ea..e4b4a8d 100644 --- a/crates/livekit-core/src/room/mod.rs +++ b/crates/livekit-core/src/room/mod.rs @@ -1,317 +1,115 @@ -use parking_lot::lock_api::RwLockUpgradableReadGuard; -use parking_lot::{Mutex, RwLock}; -use std::collections::HashMap; -use std::sync::atomic::AtomicU8; +use self::participant::ConnectionQuality; +use self::room_session::{ConnectionState, RoomSession, SessionHandle}; +use crate::proto::data_packet; +use crate::room::id::TrackSid; +use crate::room::participant::remote_participant::RemoteParticipant; +use crate::room::participant::Participant; +use crate::room::publication::RemoteTrackPublication; +use crate::room::publication::TrackPublication; +use crate::room::track::remote_track::RemoteTrackHandle; +use crate::rtc_engine::EngineError; +use std::fmt::Debug; use std::sync::Arc; - -use self::id::ParticipantSid; -use self::participant::local_participant::LocalParticipant; -use self::participant::remote_participant::RemoteParticipant; -use self::participant::ParticipantInternalTrait; -use self::participant::ParticipantTrait; -use crate::events::room::{ - ParticipantConnectedEvent, ParticipantDisconnectedEvent, RoomEvents, TrackSubscribedEvent, -}; -use crate::proto; -use crate::proto::participant_info; use thiserror::Error; -use tracing::{debug, error}; +use tokio::sync::mpsc; -use crate::rtc_engine::{EngineError, EngineEvent, EngineEvents, RTCEngine}; -use crate::signal_client::SignalOptions; +pub use crate::rtc_engine::SimulateScenario; pub mod id; pub mod participant; pub mod publication; +pub mod room_session; pub mod track; +pub type RoomEvents = mpsc::UnboundedReceiver; +pub type RoomEmitter = mpsc::UnboundedSender; +pub type RoomResult = Result; + #[derive(Error, Debug)] pub enum RoomError { - #[error("internal RTCEngine failure")] + #[error("engine : {0}")] Engine(#[from] EngineError), - #[error("internal Room failure")] + #[error("room failure: {0}")] Internal(String), } -type RoomResult = Result; +#[derive(Error, Debug, Clone)] +pub enum TrackError { + #[error("could not find published track with sid: {0}")] + TrackNotFound(String), +} + +#[derive(Clone, Debug)] +pub enum RoomEvent { + ParticipantConnected(Arc), + ParticipantDisconnected(Arc), + TrackSubscribed { + track: RemoteTrackHandle, + publication: RemoteTrackPublication, + participant: Arc, + }, + TrackPublished { + publication: RemoteTrackPublication, + participant: Arc, + }, + TrackUnpublished { + publication: RemoteTrackPublication, + participant: Arc, + }, + TrackUnsubscribed { + track: RemoteTrackHandle, + publication: RemoteTrackPublication, + participant: Arc, + }, + TrackSubscriptionFailed { + error: TrackError, + sid: TrackSid, + participant: Arc, + }, + TrackMuted { + publication: TrackPublication, + participant: Participant, + }, + TrackUnmuted { + publication: TrackPublication, + participant: Participant, + }, + ActiveSpeakersChanged { + speakers: Vec, + }, + ConnectionQualityChanged { + quality: ConnectionQuality, + participant: Participant, + }, + DataReceived { + payload: Arc>, + kind: data_packet::Kind, + participant: Arc, + }, + ConnectionStateChanged(ConnectionState), + Connected, + Disconnected, + Reconnecting, + Reconnected, +} #[derive(Debug)] -pub enum ConnectionState { - Disconnected, - Connecting, - Connected, - Reconnecting, -} - -struct RoomInner { - state: AtomicU8, // ConnectionState - sid: Mutex, - name: Mutex, - participants: RwLock>>, - rtc_engine: Arc, - local_participant: Arc, -} - pub struct Room { - inner: Option>, - events: Arc, + handle: SessionHandle, } impl Room { - pub fn new() -> Room { - Self { - inner: None, - events: Default::default(), - } + pub async fn connect(url: &str, token: &str) -> RoomResult<(Self, RoomEvents)> { + let (emitter, events) = mpsc::unbounded_channel(); + let handle = SessionHandle::connect(emitter, url, token).await?; + Ok((Self { handle }, events)) } - pub async fn connect(&mut self, url: &str, token: &str) -> RoomResult<()> { - let (rtc_engine, engine_events) = - RTCEngine::connect(url, token, SignalOptions::default()).await?; - let rtc_engine = Arc::new(rtc_engine); - let join_response = rtc_engine.join_response(); - let local_participant = Arc::new(LocalParticipant::new( - rtc_engine.clone(), - join_response.participant.unwrap().clone(), - )); - let room_info = join_response.room.unwrap(); - let inner = Arc::new(RoomInner { - state: AtomicU8::new(ConnectionState::Connecting as u8), - sid: Mutex::new(room_info.sid), - name: Mutex::new(room_info.name), - participants: Default::default(), - rtc_engine, - local_participant, - }); - - self.inner = Some(inner.clone()); - - // Add already connected participants - for pi in join_response.other_participants { - let p = Self::create_participant(inner.clone(), self.events.clone(), pi.clone()); - p.update_info(pi).await; - } - - tokio::spawn(Self::room_task(inner, self.events.clone(), engine_events)); - - Ok(()) + pub async fn close(self) { + self.handle.close().await; } - pub fn events(&self) -> Arc { - self.events.clone() - } - - pub fn get_handle(&self) -> Option { - self.inner.as_ref().map(|inner| RoomHandle { - inner: inner.clone(), - }) - } - - async fn room_task( - room_inner: Arc, - room_events: Arc, - mut engine_events: EngineEvents, - ) { - while let Some(event) = engine_events.recv().await { - if let Err(err) = - Self::handle_event(room_inner.clone(), room_events.clone(), event).await - { - error!("failed to handle engine event: {:?}", err); - } - } - } - - async fn handle_event( - room_inner: Arc, - room_events: Arc, - event: EngineEvent, - ) -> RoomResult<()> { - match event { - EngineEvent::ParticipantUpdate(update) => { - Self::handle_participant_update(room_inner.clone(), room_events.clone(), update) - .await - } - EngineEvent::AddTrack { - rtp_receiver, - streams, - } => { - if streams.is_empty() { - Err(RoomError::Internal( - "AddTrack event with empty streams".to_string(), - ))?; - } - - let first_stream_id = streams.first().unwrap().id(); - let stream_id = unpack_stream_id(&first_stream_id); - if stream_id.is_none() { - Err(RoomError::Internal(format!( - "AddTrack event with invalid track_id: {:?}", - first_stream_id - )))?; - } - - let (participant_sid, track_sid) = stream_id.unwrap(); - let remote_participant = - Self::get_participant(room_inner.clone(), &participant_sid.to_string().into()); - - if let Some(remote_participant) = remote_participant { - remote_participant.add_subscribed_media_track( - track_sid.to_string().into(), - rtp_receiver.track(), - ); - } else { - // The server should send participant updates before sending a new offer - // So this should not happen. - Err(RoomError::Internal(format!( - "AddTrack event with invalid participant_sid: {:?}", - participant_sid - )))?; - } - } - } - - Ok(()) - } - - async fn handle_participant_update( - room_inner: Arc, - room_events: Arc, - update: proto::ParticipantUpdate, - ) { - for pi in update.participants { - if pi.sid == room_inner.local_participant.sid() - || pi.identity == room_inner.local_participant.identity() - { - room_inner.local_participant.clone().update_info(pi).await; - continue; - } - - let remote_participant = - Self::get_participant(room_inner.clone(), &pi.sid.clone().into()); - - if let Some(remote_participant) = remote_participant { - if pi.state == participant_info::State::Disconnected as i32 { - // Participant disconencted - Self::handle_participant_disconnect( - room_inner.clone(), - room_events.clone(), - remote_participant, - ) - } else { - // Participant is already connected, update the informations - remote_participant.update_info(pi).await; - } - } else { - // Create a new participant and call OnConnect event - let remote_participant = - Self::create_participant(room_inner.clone(), room_events.clone(), pi); - let mut handler = room_events.on_participant_connected.lock(); - if let Some(cb) = handler.as_mut() { - cb(ParticipantConnectedEvent { - room_handle: RoomHandle::from(room_inner.clone()), - participant: remote_participant.clone(), - }); - } - } - } - } - - fn handle_participant_disconnect( - room_inner: Arc, - room_events: Arc, - remote_participant: Arc, - ) { - room_inner - .participants - .write() - .remove(&remote_participant.sid()); - - // TODO(theomonnom): Unpublish all tracks - - let mut handler = room_events.on_participant_disconnected.lock(); - if let Some(cb) = handler.as_mut() { - cb(ParticipantDisconnectedEvent { - room_handle: RoomHandle::from(room_inner.clone()), - participant: remote_participant.clone(), - }); - } - } - - fn get_participant( - room_inner: Arc, - sid: &ParticipantSid, - ) -> Option> { - room_inner.participants.read().get(sid).cloned() - } - - fn create_participant( - room_inner: Arc, - room_events: Arc, - pi: proto::ParticipantInfo, - ) -> Arc { - let p = Arc::new(RemoteParticipant::new(pi.clone())); - - // Forward participantevents to room events - p.internal_events().on_track_subscribed({ - let room_events = room_events.clone(); - let room_inner = room_inner.clone(); - - move |event| { - let room_events = room_events.clone(); - let room_inner = room_inner.clone(); - - async move { - if let Some(cb) = room_events.clone().on_track_subscribed.lock().as_mut() { - cb(TrackSubscribedEvent { - room_handle: RoomHandle::from(room_inner.clone()), - track: event.track, - participant: event.participant, - publication: event.publication, - }) - .await; - } - } - } - }); - - room_inner - .participants - .write() - .insert(pi.sid.into(), p.clone()); - p - } -} - -#[derive(Clone)] -pub struct RoomHandle { - inner: Arc, -} - -impl RoomHandle { - fn from(room_inner: Arc) -> Self { - Self { inner: room_inner } - } - - pub fn sid(&self) -> String { - self.inner.sid.lock().clone() - } - - pub fn name(&self) -> String { - self.inner.name.lock().clone() - } - - pub fn local_participant(&self) -> Arc { - self.inner.local_participant.clone() - } -} - -fn unpack_stream_id(stream_id: &str) -> Option<(&str, &str)> { - let split: Vec<&str> = stream_id.split('|').collect(); - if split.len() == 2 { - let participant_sid = split.get(0).unwrap(); - let track_sid = split.get(1).unwrap(); - Some((participant_sid, track_sid)) - } else { - None + pub fn session(&self) -> RoomSession { + self.handle.session() } } diff --git a/crates/livekit-core/src/room/participant/local_participant.rs b/crates/livekit-core/src/room/participant/local_participant.rs index 7927e21..e2f2d5d 100644 --- a/crates/livekit-core/src/room/participant/local_participant.rs +++ b/crates/livekit-core/src/room/participant/local_participant.rs @@ -1,22 +1,35 @@ -use crate::proto::{data_packet, DataPacket, UserPacket}; -use crate::room::participant::{impl_participant_trait, ParticipantShared, ParticipantInternalTrait}; +use super::ConnectionQuality; +use crate::proto::{data_packet, DataPacket, ParticipantInfo, UserPacket}; +use crate::room::id::{ParticipantIdentity, ParticipantSid, TrackSid}; +use crate::room::participant::{ + impl_participant_trait, ParticipantEvent, ParticipantInternalTrait, ParticipantShared, + ParticipantTrait, +}; +use crate::room::publication::TrackPublication; use crate::room::RoomError; use crate::rtc_engine::RTCEngine; +use parking_lot::RwLockReadGuard; +use std::collections::HashMap; +use std::sync::atomic::Ordering; +use std::sync::Arc; +use tokio::sync::mpsc; +#[derive(Debug)] pub struct LocalParticipant { shared: ParticipantShared, rtc_engine: Arc, } impl LocalParticipant { - pub(crate) fn new(rtc_engine: Arc, info: ParticipantInfo) -> Self { + pub(crate) fn new( + rtc_engine: Arc, + sid: ParticipantSid, + identity: ParticipantIdentity, + name: String, + metadata: String, + ) -> Self { Self { - shared: ParticipantShared::new( - info.sid.into(), - info.identity.into(), - info.name, - info.metadata, - ), + shared: ParticipantShared::new(sid, identity, name, metadata), rtc_engine, } } @@ -29,7 +42,7 @@ impl LocalParticipant { let data = DataPacket { kind: kind as i32, value: Some(data_packet::Value::User(UserPacket { - participant_sid: "".to_string(), /*self.sid().to_owned()*/ + participant_sid: self.sid().to_string(), payload: data.to_vec(), destination_sids: vec![], })), @@ -40,15 +53,23 @@ impl LocalParticipant { .await .map_err(Into::into) } - - pub(crate) async fn update_info(self: Arc, info: ParticipantInfo) { - self.shared.update_info(info); - } } impl ParticipantInternalTrait for LocalParticipant { - fn internal_events(&self) -> Arc { - self.shared.internal_events.clone() + fn update_info(self: &Arc, info: ParticipantInfo, _emit_events: bool) { + self.shared.update_info(info); + } + + fn set_speaking(&self, speaking: bool) { + self.shared.set_speaking(speaking); + } + + fn set_audio_level(&self, level: f32) { + self.shared.set_audio_level(level); + } + + fn set_connection_quality(&self, quality: ConnectionQuality) { + self.shared.set_connection_quality(quality); } } diff --git a/crates/livekit-core/src/room/participant/mod.rs b/crates/livekit-core/src/room/participant/mod.rs index 49b6f0d..2c2b71f 100644 --- a/crates/livekit-core/src/room/participant/mod.rs +++ b/crates/livekit-core/src/room/participant/mod.rs @@ -1,28 +1,103 @@ -use crate::events::participant::ParticipantEvents; +use super::publication::RemoteTrackPublication; +use super::TrackError; +use crate::proto; use crate::proto::ParticipantInfo; use crate::room::id::{ParticipantIdentity, ParticipantSid, TrackSid}; use crate::room::participant::local_participant::LocalParticipant; use crate::room::participant::remote_participant::RemoteParticipant; use crate::room::publication::{TrackPublication, TrackPublicationTrait}; -use futures_util::future::BoxFuture; +use crate::room::track::remote_track::RemoteTrackHandle; use livekit_utils::enum_dispatch; -use parking_lot::{Mutex, RwLock}; +use livekit_utils::observer::Dispatcher; +use parking_lot::{Mutex, RwLock, RwLockReadGuard}; +use proto::data_packet; use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU8, Ordering}; use std::sync::Arc; +use tokio::sync::mpsc; pub mod local_participant; pub mod remote_participant; -type OnTrackSubscribed = Box BoxFuture<'static, ()> + Send + Sync>; +#[derive(Debug, Clone)] +pub enum ParticipantEvent { + TrackPublished { + publication: RemoteTrackPublication, + }, + TrackUnpublished { + publication: RemoteTrackPublication, + }, + TrackSubscribed { + track: RemoteTrackHandle, + publication: RemoteTrackPublication, + }, + TrackUnsubscribed { + track: RemoteTrackHandle, + publication: RemoteTrackPublication, + }, + TrackSubscriptionFailed { + error: TrackError, + sid: TrackSid, + }, + DataReceived { + payload: Arc>, + kind: data_packet::Kind, + }, + SpeakingChanged { + speaking: bool, + }, + TrackMuted { + publication: TrackPublication, + }, + TrackUnmuted { + publication: TrackPublication, + }, + ConnectionQualityChanged { + quality: ConnectionQuality, + }, +} +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +#[repr(u8)] +pub enum ConnectionQuality { + Unknown, + Excellent, + Good, + Poor, +} + +impl From for ConnectionQuality { + fn from(value: u8) -> Self { + match value { + 1 => Self::Excellent, + 2 => Self::Good, + 3 => Self::Poor, + _ => Self::Unknown, + } + } +} + +impl From for ConnectionQuality { + fn from(value: proto::ConnectionQuality) -> Self { + match value { + proto::ConnectionQuality::Excellent => Self::Excellent, + proto::ConnectionQuality::Good => Self::Good, + proto::ConnectionQuality::Poor => Self::Poor, + } + } +} + +#[derive(Debug)] pub(super) struct ParticipantShared { - pub(super) events: Arc, - pub(super) internal_events: Arc, pub(super) sid: Mutex, pub(super) identity: Mutex, pub(super) name: Mutex, pub(super) metadata: Mutex, pub(super) tracks: RwLock>, + pub(super) speaking: AtomicBool, + pub(super) audio_level: AtomicU32, + pub(super) connection_quality: AtomicU8, + pub(super) dispatcher: Mutex>, } impl ParticipantShared { @@ -33,13 +108,15 @@ impl ParticipantShared { metadata: String, ) -> Self { Self { - events: Default::default(), - internal_events: Default::default(), sid: Mutex::new(sid), identity: Mutex::new(identity), name: Mutex::new(name), metadata: Mutex::new(metadata), tracks: Default::default(), + speaking: Default::default(), + audio_level: Default::default(), + connection_quality: AtomicU8::new(ConnectionQuality::Unknown as u8), + dispatcher: Default::default(), } } @@ -50,69 +127,84 @@ impl ParticipantShared { *self.metadata.lock() = info.metadata; // TODO(theomonnom): callback MetadataChanged } + pub(crate) fn set_speaking(&self, speaking: bool) { + self.speaking.store(speaking, Ordering::SeqCst); + } + + pub(crate) fn set_audio_level(&self, audio_level: f32) { + self.audio_level + .store(audio_level.to_bits(), Ordering::SeqCst) + } + + pub(crate) fn register_observer(&self) -> mpsc::UnboundedReceiver { + self.dispatcher.lock().register() + } + + pub(crate) fn set_connection_quality(&self, quality: ConnectionQuality) { + self.connection_quality + .store(quality as u8, Ordering::SeqCst); + } + pub(crate) fn add_track_publication(&self, publication: TrackPublication) { self.tracks.write().insert(publication.sid(), publication); } } pub(crate) trait ParticipantInternalTrait { - fn internal_events(&self) -> Arc; + fn set_speaking(&self, speaking: bool); + fn set_audio_level(&self, level: f32); + fn set_connection_quality(&self, quality: ConnectionQuality); + fn update_info(self: &Arc, info: ParticipantInfo, emit_events: bool); } pub trait ParticipantTrait { - fn events(&self) -> Arc; fn sid(&self) -> ParticipantSid; fn identity(&self) -> ParticipantIdentity; fn name(&self) -> String; fn metadata(&self) -> String; + fn is_speaking(&self) -> bool; + fn audio_level(&self) -> f32; + fn connection_quality(&self) -> ConnectionQuality; + fn tracks(&self) -> RwLockReadGuard>; + fn register_observer(&self) -> mpsc::UnboundedReceiver; } -#[derive(Clone)] -pub enum ParticipantHandle { +#[derive(Debug, Clone)] +pub enum Participant { Local(Arc), Remote(Arc), } -impl ParticipantHandle { - // TODO(theomonnom): Add async support to wrap_variants ... - pub(crate) async fn update_info(&self, info: ParticipantInfo) { - match self { - Self::Local(inner) => inner.clone().update_info(info).await, - Self::Remote(inner) => inner.clone().update_info(info).await, - } - } -} +// TODO(theomonnom): Should I provide a WeakParticipant here ? -impl ParticipantInternalTrait for ParticipantHandle { +impl Participant { enum_dispatch!( [Local, Remote] - fnc!(internal_events, &Self, [], Arc); + fnc!(pub(crate), update_info, &Self, [info: ParticipantInfo, emit_events: bool], ()); + fnc!(pub(crate), set_speaking, &Self, [speaking: bool], ()); + fnc!(pub(crate), set_audio_level, &Self, [audio_level: f32], ()); + fnc!(pub(crate), set_connection_quality, &Self, [quality: ConnectionQuality], ()); ); } -impl ParticipantTrait for ParticipantHandle { +impl ParticipantTrait for Participant { enum_dispatch!( [Local, Remote] - fnc!(events, &Self, [], Arc); fnc!(sid, &Self, [], ParticipantSid); fnc!(identity, &Self, [], ParticipantIdentity); fnc!(name, &Self, [], String); fnc!(metadata, &Self, [], String); + fnc!(is_speaking, &Self, [], bool); + fnc!(audio_level, &Self, [], f32); + fnc!(connection_quality, &Self, [], ConnectionQuality); + fnc!(tracks, &Self, [], RwLockReadGuard>); + fnc!(register_observer, &Self, [], mpsc::UnboundedReceiver); ); } macro_rules! impl_participant_trait { ($x:ty) => { - use crate::events::participant::ParticipantEvents; - use crate::proto::ParticipantInfo; - use crate::room::id::{ParticipantIdentity, ParticipantSid}; - use std::sync::Arc; - impl crate::room::participant::ParticipantTrait for $x { - fn events(&self) -> Arc { - self.shared.events.clone() - } - fn sid(&self) -> ParticipantSid { self.shared.sid.lock().clone() } @@ -128,6 +220,26 @@ macro_rules! impl_participant_trait { fn metadata(&self) -> String { self.shared.metadata.lock().clone() } + + fn is_speaking(&self) -> bool { + self.shared.speaking.load(Ordering::SeqCst) + } + + fn audio_level(&self) -> f32 { + f32::from_bits(self.shared.audio_level.load(Ordering::SeqCst)) + } + + fn connection_quality(&self) -> ConnectionQuality { + self.shared.connection_quality.load(Ordering::SeqCst).into() + } + + fn tracks(&self) -> RwLockReadGuard> { + self.shared.tracks.read() + } + + fn register_observer(&self) -> mpsc::UnboundedReceiver { + self.shared.register_observer() + } } }; } diff --git a/crates/livekit-core/src/room/participant/remote_participant.rs b/crates/livekit-core/src/room/participant/remote_participant.rs index 12efbae..b5a37c8 100644 --- a/crates/livekit-core/src/room/participant/remote_participant.rs +++ b/crates/livekit-core/src/room/participant/remote_participant.rs @@ -1,10 +1,9 @@ -use crate::events::participant::{ - TrackPublishedEvent, TrackSubscribedEvent, TrackSubscriptionFailedEvent, -}; -use crate::events::TrackError; -use crate::room::id::TrackSid; +use super::ConnectionQuality; +use crate::proto::{data_packet, DataPacket, ParticipantInfo, UserPacket}; +use crate::room::id::{ParticipantIdentity, ParticipantSid, TrackSid}; use crate::room::participant::{ - impl_participant_trait, ParticipantInternalTrait, ParticipantShared, + impl_participant_trait, ParticipantEvent, ParticipantInternalTrait, ParticipantShared, + ParticipantTrait, }; use crate::room::publication::{ RemoteTrackPublication, TrackPublication, TrackPublicationInternalTrait, TrackPublicationTrait, @@ -12,141 +11,38 @@ use crate::room::publication::{ use crate::room::track::remote_audio_track::RemoteAudioTrack; use crate::room::track::remote_track::RemoteTrackHandle; use crate::room::track::remote_video_track::RemoteVideoTrack; -use crate::room::track::{TrackKind, TrackTrait, TrackHandle}; +use crate::room::track::{TrackKind, TrackTrait}; +use crate::room::TrackError; use livekit_webrtc::media_stream::MediaStreamTrackHandle; +use parking_lot::RwLockReadGuard; +use std::collections::HashMap; use std::collections::HashSet; +use std::sync::atomic::Ordering; +use std::sync::Arc; use std::time::Duration; -use tokio::time::{sleep, timeout}; -use tracing::{info, error}; - -use super::ParticipantTrait; +use tokio::sync::mpsc; +use tokio::time::timeout; +use tracing::{debug, error, instrument, Level}; const ADD_TRACK_TIMEOUT: Duration = Duration::from_secs(5); +#[derive(Debug)] pub struct RemoteParticipant { shared: ParticipantShared, } impl RemoteParticipant { - pub(crate) fn new(info: ParticipantInfo) -> Self { + pub(crate) fn new( + sid: ParticipantSid, + identity: ParticipantIdentity, + name: String, + metadata: String, + ) -> Self { Self { - shared: ParticipantShared::new( - info.sid.into(), - info.identity.into(), - info.name, - info.metadata, - ), + shared: ParticipantShared::new(sid, identity, name, metadata), } } - pub(crate) fn add_subscribed_media_track( - self: Arc, - sid: TrackSid, - media_track: MediaStreamTrackHandle, - ) { - tokio::spawn(async move { - let wait_publication = { - let participant = self.clone(); - let sid = sid.clone(); - async move { - loop { - let publication = participant.get_track_publication(&sid); - if let Some(publication) = publication { - return publication; - } - - sleep(Duration::from_millis(50)).await; - } - } - }; - - if let Ok(remote_publication) = timeout(ADD_TRACK_TIMEOUT, wait_publication).await { - let track = match remote_publication.kind() { - TrackKind::Audio => { - if let MediaStreamTrackHandle::Audio(rtc_track) = media_track { - let audio_track = RemoteAudioTrack::new( - remote_publication.sid().into(), - remote_publication.name(), - rtc_track, - ); - RemoteTrackHandle::Audio(Arc::new(audio_track)) - } else { - unreachable!(); - } - } - TrackKind::Video => { - if let MediaStreamTrackHandle::Video(rtc_track) = media_track { - let video_track = RemoteVideoTrack::new( - remote_publication.sid().into(), - remote_publication.name(), - rtc_track, - ); - RemoteTrackHandle::Video(Arc::new(video_track)) - } else { - unreachable!() - } - } - _ => unreachable!(), - }; - - info!("starting track: {:?}", sid); - - remote_publication.update_track(Some(track.clone().into())); - self.shared - .add_track_publication(TrackPublication::Remote(remote_publication.clone())); - track.start(); - - let event = TrackSubscribedEvent { - track, - publication: remote_publication, - participant: self.clone(), - }; - - if let Some(cb) = self - .shared - .internal_events - .on_track_subscribed - .lock() - .as_mut() - { - cb(event.clone()).await; - } - - if let Some(cb) = self.shared.events.on_track_subscribed.lock().as_mut() { - cb(event).await; - } - } else { - error!("could not find published track with sid: {:?}", sid); - - let event = TrackSubscriptionFailedEvent { - sid: sid.clone(), - error: TrackError::TrackNotFound(sid.clone().to_string()), - participant: self.clone(), - }; - - if let Some(cb) = self - .shared - .internal_events - .on_track_subscription_failed - .lock() - .as_mut() - { - cb(event.clone()).await; - } - - if let Some(cb) = self - .shared - .events - .on_track_subscription_failed - .lock() - .as_mut() - { - cb(event).await; - } - } - }); - } - fn get_track_publication(&self, sid: &TrackSid) -> Option { self.shared.tracks.read().get(sid).map(|track| { if let TrackPublication::Remote(remote) = track { @@ -157,11 +53,129 @@ impl RemoteParticipant { }) } - pub(crate) async fn update_info(self: Arc, info: ParticipantInfo) { + /// Called by the RoomSession when receiving data by the RTCSession + /// It is just used to emit the Data event on the participant dispatcher. + pub(crate) fn on_data_received(&self, data: Arc>, kind: data_packet::Kind) { + self.shared + .dispatcher + .lock() + .dispatch(&ParticipantEvent::DataReceived { + payload: data, + kind, + }); + } + + #[instrument(level = Level::DEBUG)] + pub(crate) async fn add_subscribed_media_track( + self: Arc, + sid: TrackSid, + media_track: MediaStreamTrackHandle, + ) { + let wait_publication = { + let participant = self.clone(); + let sid = sid.clone(); + async move { + loop { + let publication = participant.get_track_publication(&sid); + if let Some(publication) = publication { + return publication; + } + + tokio::task::yield_now().await; + } + } + }; + + if let Ok(remote_publication) = timeout(ADD_TRACK_TIMEOUT, wait_publication).await { + let track = match remote_publication.kind() { + TrackKind::Audio => { + if let MediaStreamTrackHandle::Audio(rtc_track) = media_track { + let audio_track = RemoteAudioTrack::new( + remote_publication.sid().into(), + remote_publication.name(), + rtc_track, + ); + RemoteTrackHandle::Audio(Arc::new(audio_track)) + } else { + unreachable!(); + } + } + TrackKind::Video => { + if let MediaStreamTrackHandle::Video(rtc_track) = media_track { + let video_track = RemoteVideoTrack::new( + remote_publication.sid().into(), + remote_publication.name(), + rtc_track, + ); + RemoteTrackHandle::Video(Arc::new(video_track)) + } else { + unreachable!() + } + } + _ => unreachable!(), + }; + + debug!("starting track: {:?}", sid); + + remote_publication.update_track(Some(track.clone().into())); + self.shared + .add_track_publication(TrackPublication::Remote(remote_publication.clone())); + track.start(); + + self.shared + .dispatcher + .lock() + .dispatch(&ParticipantEvent::TrackSubscribed { + track: track, + publication: remote_publication, + }); + } else { + error!("could not find published track with sid: {:?}", sid); + + self.shared + .dispatcher + .lock() + .dispatch(&ParticipantEvent::TrackSubscriptionFailed { + sid: sid.clone(), + error: TrackError::TrackNotFound(sid.clone().to_string()), + }); + } + } + + pub(crate) fn unpublish_track(self: &Arc, sid: &TrackSid, emit_events: bool) { + if let Some(publication) = self.get_track_publication(sid) { + // Unsubscribe to the track if needed + if let Some(track) = publication.track() { + track.stop(); + + self.shared + .dispatcher + .lock() + .dispatch(&ParticipantEvent::TrackUnsubscribed { + track: track.clone(), + publication: publication.clone(), + }); + } + + if emit_events { + self.shared + .dispatcher + .lock() + .dispatch(&ParticipantEvent::TrackUnpublished { + publication: publication.clone(), + }); + } + + publication.update_track(None); + } + } +} + +impl ParticipantInternalTrait for RemoteParticipant { + fn update_info(self: &Arc, info: ParticipantInfo, emit_events: bool) { self.shared.update_info(info.clone()); let mut valid_tracks = HashSet::::new(); - for track in info.tracks { if let Some(publication) = self.get_track_publication(&track.sid.clone().into()) { publication.update_info(track.clone()); @@ -170,35 +184,38 @@ impl RemoteParticipant { self.shared .add_track_publication(TrackPublication::Remote(publication.clone())); - // This is a new track, fire publish events - let event = TrackPublishedEvent { - participant: self.clone(), - publication: publication.clone(), - }; - - if let Some(cb) = self - .shared - .internal_events - .on_track_published - .lock() - .as_mut() - { - cb(event.clone()).await; - } - - if let Some(cb) = self.shared.events.on_track_published.lock().as_mut() { - cb(event).await; + // This is a new track, dispatch publish event + if emit_events { + self.shared + .dispatcher + .lock() + .dispatch(&ParticipantEvent::TrackPublished { publication }); } } valid_tracks.insert(track.sid.into()); } - } -} -impl ParticipantInternalTrait for RemoteParticipant { - fn internal_events(&self) -> Arc { - self.shared.internal_events.clone() + // remove tracks that are no longer valid + for (sid, _) in self.shared.tracks.read().iter() { + if valid_tracks.contains(sid) { + continue; + } + + self.unpublish_track(sid, emit_events); + } + } + + fn set_speaking(&self, speaking: bool) { + self.shared.set_speaking(speaking); + } + + fn set_audio_level(&self, level: f32) { + self.shared.set_audio_level(level); + } + + fn set_connection_quality(&self, quality: ConnectionQuality) { + self.shared.set_connection_quality(quality); } } diff --git a/crates/livekit-core/src/room/publication/mod.rs b/crates/livekit-core/src/room/publication/mod.rs index 547e825..0e8a60e 100644 --- a/crates/livekit-core/src/room/publication/mod.rs +++ b/crates/livekit-core/src/room/publication/mod.rs @@ -4,13 +4,15 @@ use crate::room::id::ParticipantSid; use crate::room::id::TrackSid; use crate::room::track::local_track::LocalTrackHandle; use crate::room::track::remote_track::RemoteTrackHandle; -use crate::room::track::{TrackHandle, TrackKind, TrackSource}; +use crate::room::track::{TrackHandle, TrackKind, TrackSource, TrackTrait}; use livekit_utils::enum_dispatch; +use livekit_utils::observer::Dispatcher; use parking_lot::Mutex; use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; use std::sync::Arc; +use tokio::sync::{mpsc, oneshot}; -use super::track::TrackDimension; +use super::track::{TrackDimension, TrackEvent}; pub(crate) trait TrackPublicationInternalTrait { fn update_track(&self, track: Option); @@ -22,9 +24,11 @@ pub trait TrackPublicationTrait { fn sid(&self) -> TrackSid; fn kind(&self) -> TrackKind; fn source(&self) -> TrackSource; + fn muted(&self) -> bool; fn simulcasted(&self) -> bool; } +#[derive(Debug)] pub(super) struct TrackPublicationShared { pub(super) track: Mutex>, pub(super) name: Mutex, @@ -34,7 +38,10 @@ pub(super) struct TrackPublicationShared { pub(super) simulcasted: AtomicBool, pub(super) dimension: Mutex, pub(super) mime_type: Mutex, - pub(super) participant: ParticipantSid, // TODO(theomonnom) Use WeakParticipant instead + pub(super) muted: AtomicBool, + pub(super) participant: ParticipantSid, + pub(super) dispatcher: Mutex>, + pub(super) close_sender: Mutex>>, } impl TrackPublicationShared { @@ -54,13 +61,56 @@ impl TrackPublicationShared { simulcasted: AtomicBool::new(info.simulcast), dimension: Mutex::new(TrackDimension(info.width, info.height)), mime_type: Mutex::new(info.mime_type), + muted: AtomicBool::new(info.muted), + dispatcher: Default::default(), + close_sender: Default::default(), participant, }) } + pub fn update_track(self: &Arc, track: Option) { + let mut old_track = self.track.lock(); + + if let Some(close_sender) = self.close_sender.lock().take() { + let _ = close_sender.send(()); + } + + *old_track = track.clone(); + if let Some(track) = track { + let (close_sender, close_receiver) = oneshot::channel(); + self.close_sender.lock().replace(close_sender); + + let track_receiver = track.register_observer(); + tokio::spawn( + self.clone() + .publication_task(close_receiver, track_receiver), + ); + } + } + + /// Task used to forward TrackHandle's events to the TrackPublications's dispatcher + async fn publication_task( + self: Arc, + mut close_receiver: oneshot::Receiver<()>, + mut track_receiver: mpsc::UnboundedReceiver, + ) { + loop { + tokio::select! { + Some(event) = track_receiver.recv() => { + self.dispatcher.lock().dispatch(&event); + } + _ = &mut close_receiver => { + break; + } + } + } + } + pub fn update_info(&self, info: TrackInfo) { *self.name.lock() = info.name; *self.sid.lock() = info.sid.into(); + *self.dimension.lock() = TrackDimension(info.width, info.height); + *self.mime_type.lock() = info.mime_type; self.kind.store( TrackKind::from(TrackType::from_i32(info.r#type).unwrap()) as u8, Ordering::SeqCst, @@ -70,12 +120,23 @@ impl TrackPublicationShared { Ordering::SeqCst, ); self.simulcasted.store(info.simulcast, Ordering::SeqCst); - *self.dimension.lock() = TrackDimension(info.width, info.height); - *self.mime_type.lock() = info.mime_type; + self.muted.store(info.muted, Ordering::SeqCst); + + if let Some(track) = self.track.lock().as_ref() { + track.set_muted(info.muted); + } } } -#[derive(Clone)] +impl Drop for TrackPublicationShared { + fn drop(&mut self) { + if let Some(close_sender) = self.close_sender.lock().take() { + let _ = close_sender.send(()); + } + } +} + +#[derive(Clone, Debug)] pub enum TrackPublication { Local(LocalTrackPublication), Remote(RemoteTrackPublication), @@ -106,22 +167,13 @@ impl TrackPublicationTrait for TrackPublication { fnc!(name, &Self, [], String); fnc!(kind, &Self, [], TrackKind); fnc!(source, &Self, [], TrackSource); + fnc!(muted, &Self, [], bool); fnc!(simulcasted, &Self, [], bool); ); } macro_rules! impl_publication_trait { ($x:ident) => { - impl TrackPublicationInternalTrait for $x { - fn update_track(&self, track: Option) { - *self.shared.track.lock() = track; - } - - fn update_info(&self, info: TrackInfo) { - self.shared.update_info(info); - } - } - impl TrackPublicationTrait for $x { fn name(&self) -> String { self.shared.name.lock().clone() @@ -142,11 +194,15 @@ macro_rules! impl_publication_trait { fn simulcasted(&self) -> bool { self.shared.simulcasted.load(Ordering::SeqCst) } + + fn muted(&self) -> bool { + self.shared.muted.load(Ordering::SeqCst) + } } }; } -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct LocalTrackPublication { shared: Arc, } @@ -161,7 +217,17 @@ impl LocalTrackPublication { } } -#[derive(Clone)] +impl TrackPublicationInternalTrait for LocalTrackPublication { + fn update_track(&self, track: Option) { + self.shared.update_track(track); + } + + fn update_info(&self, info: TrackInfo) { + self.shared.update_info(info); + } +} + +#[derive(Clone, Debug)] pub struct RemoteTrackPublication { shared: Arc, } @@ -182,5 +248,15 @@ impl RemoteTrackPublication { } } +impl TrackPublicationInternalTrait for RemoteTrackPublication { + fn update_track(&self, track: Option) { + self.shared.update_track(track); + } + + fn update_info(&self, info: TrackInfo) { + self.shared.update_info(info); + } +} + impl_publication_trait!(LocalTrackPublication); impl_publication_trait!(RemoteTrackPublication); diff --git a/crates/livekit-core/src/room/room_session.rs b/crates/livekit-core/src/room/room_session.rs new file mode 100644 index 0000000..4f284d1 --- /dev/null +++ b/crates/livekit-core/src/room/room_session.rs @@ -0,0 +1,571 @@ +use super::id::{ParticipantIdentity, ParticipantSid}; +use super::participant::local_participant::LocalParticipant; +use super::participant::remote_participant::RemoteParticipant; +use super::participant::{ConnectionQuality, Participant, ParticipantEvent}; +use super::participant::{ParticipantInternalTrait, ParticipantTrait}; +use super::{RoomEmitter, RoomError, RoomEvent, RoomResult, SimulateScenario}; +use crate::proto::{self, participant_info, SpeakerInfo}; +use crate::rtc_engine::{EngineEvent, EngineEvents, EngineResult, RTCEngine}; +use crate::signal_client::SignalOptions; +use parking_lot::{Mutex, RwLock}; +use std::collections::HashMap; +use std::sync::atomic::{AtomicU8, Ordering}; +use std::sync::Arc; +use tokio::sync::mpsc; +use tokio::sync::oneshot; +use tokio::task::JoinHandle; +use tracing::{error, instrument, Level}; + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum ConnectionState { + Disconnected, + Connected, + Reconnecting, + Unknown, +} + +impl From for ConnectionState { + fn from(value: u8) -> Self { + match value { + 0 => ConnectionState::Disconnected, + 1 => ConnectionState::Connected, + 2 => ConnectionState::Reconnecting, + _ => ConnectionState::Unknown, + } + } +} + +/// Internal representation of a RoomSession +#[derive(Debug)] +struct SessionInner { + state: AtomicU8, // ConnectionState + sid: Mutex, + name: Mutex, + participants: RwLock>>, + participants_tasks: RwLock, oneshot::Sender<()>)>>, + active_speakers: RwLock>, + rtc_engine: Arc, + local_participant: Arc, + room_emitter: RoomEmitter, +} + +#[derive(Debug)] +pub(crate) struct SessionHandle { + session: RoomSession, + session_task: JoinHandle<()>, + close_emitter: oneshot::Sender<()>, +} + +/// RoomSession represents a connection to a room. +/// It can be cloned and shared across threads. +#[derive(Debug, Clone)] +pub struct RoomSession { + inner: Arc, +} + +impl SessionHandle { + pub async fn connect(room_emitter: RoomEmitter, url: &str, token: &str) -> RoomResult { + let (rtc_engine, engine_events) = RTCEngine::new(); + let rtc_engine = Arc::new(rtc_engine); + rtc_engine + .connect(url, token, SignalOptions::default()) + .await?; + + let join_response = rtc_engine.join_response().unwrap(); + let pi = join_response.participant.unwrap().clone(); + let local_participant = Arc::new(LocalParticipant::new( + rtc_engine.clone(), + pi.sid.into(), + pi.identity.into(), + pi.name, + pi.metadata, + )); + + let room_info = join_response.room.unwrap(); + let inner = Arc::new(SessionInner { + state: AtomicU8::new(ConnectionState::Disconnected as u8), + sid: Mutex::new(room_info.sid), + name: Mutex::new(room_info.name), + participants: Default::default(), + participants_tasks: Default::default(), + active_speakers: Default::default(), + rtc_engine, + local_participant, + room_emitter, + }); + + for pi in join_response.other_participants { + let participant = { + let pi = pi.clone(); + inner.create_participant(pi.sid.into(), pi.identity.into(), pi.name, pi.metadata) + }; + participant.update_info(pi.clone(), false); + } + + let (close_emitter, close_receiver) = oneshot::channel(); + let session_task = tokio::spawn(inner.clone().room_task(engine_events, close_receiver)); + + inner.update_connection_state(ConnectionState::Connected); + + let session = Self { + session: RoomSession::from(inner), + session_task, + close_emitter, + }; + Ok(session) + } + + pub async fn close(self) { + self.session.inner.close().await; + let _ = self.close_emitter.send(()); + let _ = self.session_task.await; + } + + pub fn session(&self) -> RoomSession { + self.session.clone() + } +} + +impl RoomSession { + fn from(inner: Arc) -> Self { + Self { inner } + } + + pub fn sid(&self) -> String { + self.inner.sid.lock().clone() + } + + pub fn name(&self) -> String { + self.inner.name.lock().clone() + } + + pub fn local_participant(&self) -> Arc { + self.inner.local_participant.clone() + } + + pub fn connection_state(&self) -> ConnectionState { + self.inner.state.load(Ordering::Acquire).try_into().unwrap() + } + + pub fn participants(&self) -> &RwLock>> { + &self.inner.participants + } + + pub async fn simulate_scenario(&self, scenario: SimulateScenario) -> EngineResult<()> { + self.inner.rtc_engine.simulate_scenario(scenario).await + } +} + +impl SessionInner { + #[instrument(level = Level::DEBUG)] + async fn room_task( + self: Arc, + mut engine_events: EngineEvents, + mut close_receiver: oneshot::Receiver<()>, + ) { + loop { + tokio::select! { + res = engine_events.recv() => { + match res { + Some(event) => { + if let Err(err) = self.on_engine_event(event).await { + error!("failed to handle engine event: {:?}", err); + } + }, + _ => panic!("engine_events has been closed unexpectedly") + }; + }, + _ = &mut close_receiver => { + break; + } + } + } + } + + /// Listen to the Participant events and forward them to the Room Dispatcher + #[instrument(level = Level::DEBUG)] + async fn participant_task( + self: Arc, + participant: Participant, + mut participant_events: mpsc::UnboundedReceiver, + mut close_rx: oneshot::Receiver<()>, + ) { + loop { + tokio::select! { + res = participant_events.recv() => { + match res { + Some(event) => { + if let Err(err) = self.on_participant_event(&participant, event).await { + error!("failed to handle participant event for {:?}: {:?}", participant.sid(), err); + } + }, + _ => panic!("engine_events has been closed unexpectedly") + }; + }, + _ = &mut close_rx => { + break; + }, + } + } + } + + #[instrument(level = Level::DEBUG)] + async fn on_participant_event( + self: &Arc, + participant: &Participant, + event: ParticipantEvent, + ) -> RoomResult<()> { + if let Participant::Remote(remote_participant) = participant { + match event { + ParticipantEvent::TrackPublished { publication } => { + let _ = self.room_emitter.send(RoomEvent::TrackPublished { + participant: remote_participant.clone(), + publication, + }); + } + ParticipantEvent::TrackUnpublished { publication } => { + let _ = self.room_emitter.send(RoomEvent::TrackUnpublished { + participant: remote_participant.clone(), + publication, + }); + } + ParticipantEvent::TrackSubscribed { track, publication } => { + let _ = self.room_emitter.send(RoomEvent::TrackSubscribed { + participant: remote_participant.clone(), + track, + publication, + }); + } + ParticipantEvent::TrackUnsubscribed { track, publication } => { + let _ = self.room_emitter.send(RoomEvent::TrackUnsubscribed { + participant: remote_participant.clone(), + track, + publication, + }); + } + _ => {} + }; + } + + Ok(()) + } + + #[instrument(level = Level::DEBUG)] + async fn on_engine_event(self: &Arc, event: EngineEvent) -> RoomResult<()> { + match event { + EngineEvent::ParticipantUpdate(update) => self.handle_participant_update(update), + EngineEvent::MediaTrack { + track, + stream, + receiver: _, + } => { + let stream_id = stream.id(); + let lk_stream_id = unpack_stream_id(&stream_id); + if lk_stream_id.is_none() { + Err(RoomError::Internal(format!( + "MediaTrack event with invalid track_id: {:?}", + &stream_id + )))?; + } + + let (participant_sid, track_sid) = lk_stream_id.unwrap(); + let track_sid = track_sid.to_owned().into(); + let remote_participant = self.get_participant(&participant_sid.to_string().into()); + + if let Some(remote_participant) = remote_participant { + tokio::spawn(async move { + remote_participant + .add_subscribed_media_track(track_sid, track) + .await; + }); + } else { + // The server should send participant updates before sending a new offer + // So this should never happen. + Err(RoomError::Internal(format!( + "AddTrack event with invalid participant_sid: {:?}", + participant_sid + )))?; + } + } + EngineEvent::Resuming => { + if self.update_connection_state(ConnectionState::Reconnecting) { + let _ = self.room_emitter.send(RoomEvent::Reconnecting); + } + } + EngineEvent::Resumed => { + self.update_connection_state(ConnectionState::Connected); + let _ = self.room_emitter.send(RoomEvent::Reconnected); + + // TODO(theomonnom): Update subscriptions settings + // TODO(theomonnom): Send sync state + } + EngineEvent::Restarting => self.handle_restarting(), + EngineEvent::Restarted => self.handle_restarted(), + EngineEvent::Disconnected => self.handle_disconnected(), + EngineEvent::Data { + payload, + kind, + participant_sid, + } => { + let payload = Arc::new(payload); + if let Some(participant) = self.get_participant(&participant_sid.into()) { + let _ = self.room_emitter.send(RoomEvent::DataReceived { + payload: payload.clone(), + kind, + participant: participant.clone(), + }); + + participant.on_data_received(payload, kind); + } + } + EngineEvent::SpeakersChanged { speakers } => self.handle_speakers_changed(speakers), + EngineEvent::ConnectionQuality { updates } => { + self.handle_connection_quality_update(updates) + } + } + + Ok(()) + } + + #[instrument(level = Level::DEBUG)] + async fn close(&self) { + self.rtc_engine.close().await; + } + + /// Change the connection state and emit an event + /// Does nothing if the state is already the same + #[instrument(level = Level::DEBUG)] + fn update_connection_state(&self, state: ConnectionState) -> bool { + let old_state = self.state.load(Ordering::Acquire); + if old_state == state as u8 { + return false; + } + + self.state.store(state as u8, Ordering::Release); + let _ = self + .room_emitter + .send(RoomEvent::ConnectionStateChanged(state)); + return true; + } + + /// Update the participants inside a Room. + /// It'll create, update or remove a participant + /// It also update the participant tracks. + #[instrument(level = Level::DEBUG)] + fn handle_participant_update(self: &Arc, update: proto::ParticipantUpdate) { + for pi in update.participants { + if pi.sid == self.local_participant.sid() + || pi.identity == self.local_participant.identity() + { + self.local_participant.clone().update_info(pi, true); + continue; + } + + let remote_participant = self.get_participant(&pi.sid.clone().into()); + + if let Some(remote_participant) = remote_participant { + if pi.state == participant_info::State::Disconnected as i32 { + // Participant disconnected + self.clone() + .handle_participant_disconnect(remote_participant) + } else { + // Participant is already connected, update the it + remote_participant.update_info(pi.clone(), true); + } + } else { + // Create a new participant + let remote_participant = { + let pi = pi.clone(); + self.create_participant(pi.sid.into(), pi.identity.into(), pi.name, pi.metadata) + }; + + let _ = self + .room_emitter + .send(RoomEvent::ParticipantConnected(remote_participant.clone())); + + remote_participant.update_info(pi.clone(), true); + } + } + } + + /// Active speakers changed + /// Update the participants & sort the active_speakers by audio_level + #[instrument(level = Level::DEBUG)] + fn handle_speakers_changed(&self, speakers_info: Vec) { + let mut speakers = Vec::new(); + + for speaker in speakers_info { + let participant = { + if speaker.sid == self.local_participant.sid() { + Participant::Local(self.local_participant.clone()) + } else { + if let Some(participant) = self.get_participant(&speaker.sid.into()) { + Participant::Remote(participant) + } else { + continue; + } + } + }; + + participant.set_speaking(speaker.active); + participant.set_audio_level(speaker.level); + + if speaker.active { + speakers.push(participant); + } + } + + speakers.sort_by(|a, b| a.audio_level().partial_cmp(&b.audio_level()).unwrap()); + *self.active_speakers.write() = speakers.clone(); + + let _ = self + .room_emitter + .send(RoomEvent::ActiveSpeakersChanged { speakers }); + } + + /// Handle a connection quality update + /// Emit ConnectionQualityChanged event for the concerned participants + #[instrument(level = Level::DEBUG)] + fn handle_connection_quality_update(&self, updates: Vec) { + for update in updates { + let participant = { + if update.participant_sid == self.local_participant.sid() { + Participant::Local(self.local_participant.clone()) + } else { + if let Some(participant) = self.get_participant(&update.participant_sid.into()) + { + Participant::Remote(participant) + } else { + continue; + } + } + }; + + let quality: ConnectionQuality = proto::ConnectionQuality::from_i32(update.quality) + .unwrap() + .into(); + + participant.set_connection_quality(quality); + let _ = self.room_emitter.send(RoomEvent::ConnectionQualityChanged { + participant, + quality, + }); + } + } + + #[instrument(level = Level::DEBUG)] + fn handle_restarting(self: &Arc) { + // Remove existing participants/subscriptions on full reconnect + for (_, participant) in self.participants.read().iter() { + self.clone() + .handle_participant_disconnect(participant.clone()); + } + + if self.update_connection_state(ConnectionState::Reconnecting) { + let _ = self.room_emitter.send(RoomEvent::Reconnecting); + } + } + + #[instrument(level = Level::DEBUG)] + fn handle_restarted(self: &Arc) { + // Full reconnect succeeded! + let join_response = self.rtc_engine.join_response().unwrap(); + + self.update_connection_state(ConnectionState::Connected); + let _ = self.room_emitter.send(RoomEvent::Reconnected); + + if let Some(pi) = join_response.participant { + self.local_participant.update_info(pi, true); // The sid may have changed + } + + self.handle_participant_update(proto::ParticipantUpdate { + participants: join_response.other_participants, + }); + + // TODO(theomonnom): unpublish & republish tracks + } + + #[instrument(level = Level::DEBUG)] + fn handle_disconnected(&self) { + if self.state.load(Ordering::Acquire) == ConnectionState::Disconnected as u8 { + return; + } + + self.update_connection_state(ConnectionState::Disconnected); + let _ = self.room_emitter.send(RoomEvent::Disconnected); + } + + /// Create a new participant + /// Also add it to the participants list + #[instrument(level = Level::DEBUG)] + fn create_participant( + self: &Arc, + sid: ParticipantSid, + identity: ParticipantIdentity, + name: String, + metadata: String, + ) -> Arc { + let participant = Arc::new(RemoteParticipant::new( + sid.clone(), + identity, + name, + metadata, + )); + + // Create the participant task + let (close_tx, close_rx) = oneshot::channel(); + let participant_task = tokio::spawn(self.clone().participant_task( + Participant::Remote(participant.clone()), + participant.register_observer(), + close_rx, + )); + self.participants_tasks + .write() + .insert(sid.clone(), (participant_task, close_tx)); + + self.participants.write().insert(sid, participant.clone()); + participant + } + + /// A participant has disconnected + /// Cleanup the participant and emit an event + #[instrument(level = Level::DEBUG)] + fn handle_participant_disconnect(self: Arc, remote_participant: Arc) { + tokio::spawn(async move { + for (sid, _) in &*remote_participant.tracks() { + remote_participant.unpublish_track(&sid, true); + } + + // Close the participant task + if let Some((task, close_tx)) = self + .participants_tasks + .write() + .remove(&remote_participant.sid()) + { + let _ = close_tx.send(()); + let _ = task.await; + } + + self.participants.write().remove(&remote_participant.sid()); + + let _ = self.room_emitter.send(RoomEvent::ParticipantDisconnected( + remote_participant.clone(), + )); + }); + } + + fn get_participant(&self, sid: &ParticipantSid) -> Option> { + self.participants.read().get(sid).cloned() + } +} + +fn unpack_stream_id(stream_id: &str) -> Option<(&str, &str)> { + let split: Vec<&str> = stream_id.split('|').collect(); + if split.len() == 2 { + let participant_sid = split.get(0).unwrap(); + let track_sid = split.get(1).unwrap(); + Some((participant_sid, track_sid)) + } else { + None + } +} diff --git a/crates/livekit-core/src/room/track/events.rs b/crates/livekit-core/src/room/track/events.rs deleted file mode 100644 index 86d8cad..0000000 --- a/crates/livekit-core/src/room/track/events.rs +++ /dev/null @@ -1 +0,0 @@ -pub struct TrackEvents {} diff --git a/crates/livekit-core/src/room/track/local_audio_track.rs b/crates/livekit-core/src/room/track/local_audio_track.rs index c194c34..057796a 100644 --- a/crates/livekit-core/src/room/track/local_audio_track.rs +++ b/crates/livekit-core/src/room/track/local_audio_track.rs @@ -1,5 +1,6 @@ use crate::room::track::{impl_track_trait, TrackShared}; +#[derive(Debug)] pub struct LocalAudioTrack { shared: TrackShared, } diff --git a/crates/livekit-core/src/room/track/local_video_track.rs b/crates/livekit-core/src/room/track/local_video_track.rs index cb3f0d1..c46b155 100644 --- a/crates/livekit-core/src/room/track/local_video_track.rs +++ b/crates/livekit-core/src/room/track/local_video_track.rs @@ -1,5 +1,6 @@ use crate::room::track::{impl_track_trait, TrackShared}; +#[derive(Debug)] pub struct LocalVideoTrack { shared: TrackShared, } diff --git a/crates/livekit-core/src/room/track/mod.rs b/crates/livekit-core/src/room/track/mod.rs index ad9e218..b9eb468 100644 --- a/crates/livekit-core/src/room/track/mod.rs +++ b/crates/livekit-core/src/room/track/mod.rs @@ -5,13 +5,14 @@ use crate::room::track::local_video_track::LocalVideoTrack; use crate::room::track::remote_audio_track::RemoteAudioTrack; use crate::room::track::remote_video_track::RemoteVideoTrack; use livekit_utils::enum_dispatch; +use livekit_utils::observer::Dispatcher; use livekit_webrtc::media_stream::{MediaStreamTrackHandle, MediaStreamTrackTrait}; use parking_lot::Mutex; -use std::sync::atomic::AtomicU8; +use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; use std::sync::Arc; +use tokio::sync::mpsc; pub mod audio_track; -pub mod events; pub mod local_audio_track; pub mod local_track; pub mod local_video_track; @@ -97,6 +98,7 @@ impl From for TrackSource { } } +#[derive(Clone, Copy, Debug)] pub struct TrackDimension(pub u32, pub u32); pub trait TrackTrait { @@ -106,14 +108,25 @@ pub trait TrackTrait { fn stream_state(&self) -> StreamState; fn start(&self); fn stop(&self); + fn register_observer(&self) -> mpsc::UnboundedReceiver; + fn set_muted(&self, muted: bool); } +#[derive(Debug, Clone)] +pub enum TrackEvent { + Mute, + Unmute, +} + +#[derive(Debug)] pub(super) struct TrackShared { pub(super) sid: Mutex, pub(super) name: Mutex, pub(super) kind: AtomicU8, // TrackKind pub(super) stream_state: AtomicU8, // StreamState + pub(super) muted: AtomicBool, pub(super) rtc_track: MediaStreamTrackHandle, + pub(super) dispatcher: Mutex>, } impl TrackShared { @@ -128,7 +141,9 @@ impl TrackShared { name: Mutex::new(name), kind: AtomicU8::new(kind as u8), stream_state: AtomicU8::new(StreamState::Active as u8), + muted: AtomicBool::new(false), rtc_track, + dispatcher: Default::default(), } } @@ -139,9 +154,28 @@ impl TrackShared { pub(crate) fn stop(&self) { self.rtc_track.set_enabled(false); } + + pub(crate) fn set_muted(&self, muted: bool) { + if self.muted.load(Ordering::SeqCst) == muted { + return; + } + + self.muted.store(muted, Ordering::SeqCst); + self.rtc_track.set_enabled(!muted); + + self.dispatcher.lock().dispatch(if muted { + &TrackEvent::Mute + } else { + &TrackEvent::Unmute + }); + } + + pub(crate) fn register_observer(&self) -> mpsc::UnboundedReceiver { + self.dispatcher.lock().register() + } } -#[derive(Clone)] +#[derive(Clone, Debug)] pub enum TrackHandle { LocalVideo(Arc), LocalAudio(Arc), @@ -158,6 +192,8 @@ impl TrackTrait for TrackHandle { fnc!(stream_state, &Self, [], StreamState); fnc!(start, &Self, [], ()); fnc!(stop, &Self, [], ()); + fnc!(register_observer, &Self, [], mpsc::UnboundedReceiver); + fnc!(set_muted, &Self, [muted: bool], ()); ); } @@ -177,9 +213,10 @@ impl TrackHandle { macro_rules! impl_track_trait { ($x:ident) => { - use crate::room::id::TrackSid; - use crate::room::track::{StreamState, TrackKind, TrackTrait}; use std::sync::atomic::Ordering; + use tokio::sync::mpsc; + use $crate::room::id::TrackSid; + use $crate::room::track::{StreamState, TrackEvent, TrackKind, TrackTrait}; impl TrackTrait for $x { fn sid(&self) -> TrackSid { @@ -205,6 +242,14 @@ macro_rules! impl_track_trait { fn stop(&self) { self.shared.stop(); } + + fn register_observer(&self) -> mpsc::UnboundedReceiver { + self.shared.register_observer() + } + + fn set_muted(&self, muted: bool) { + self.shared.set_muted(muted); + } } }; } diff --git a/crates/livekit-core/src/room/track/remote_audio_track.rs b/crates/livekit-core/src/room/track/remote_audio_track.rs index 933f307..1ce12c6 100644 --- a/crates/livekit-core/src/room/track/remote_audio_track.rs +++ b/crates/livekit-core/src/room/track/remote_audio_track.rs @@ -2,6 +2,7 @@ use crate::room::track::{impl_track_trait, TrackShared}; use livekit_webrtc::media_stream::{AudioTrack, MediaStreamTrackHandle}; use std::sync::Arc; +#[derive(Debug)] pub struct RemoteAudioTrack { shared: TrackShared, } diff --git a/crates/livekit-core/src/room/track/remote_track.rs b/crates/livekit-core/src/room/track/remote_track.rs index 81e53ea..0ed2c09 100644 --- a/crates/livekit-core/src/room/track/remote_track.rs +++ b/crates/livekit-core/src/room/track/remote_track.rs @@ -1,15 +1,15 @@ use std::sync::Arc; - use super::{StreamState, TrackKind}; use crate::room::id::TrackSid; use crate::room::track::remote_audio_track::RemoteAudioTrack; use crate::room::track::remote_video_track::RemoteVideoTrack; -use crate::room::track::TrackHandle; +use crate::room::track::{TrackHandle, TrackEvent}; +use tokio::sync::mpsc; use livekit_utils::enum_dispatch; use super::TrackTrait; -#[derive(Clone)] +#[derive(Clone, Debug)] pub enum RemoteTrackHandle { Audio(Arc), Video(Arc), @@ -24,6 +24,8 @@ impl TrackTrait for RemoteTrackHandle { fnc!(stream_state, &Self, [], StreamState); fnc!(start, &Self, [], ()); fnc!(stop, &Self, [], ()); + fnc!(register_observer, &Self, [], mpsc::UnboundedReceiver); + fnc!(set_muted, &Self, [muted: bool], ()); ); } diff --git a/crates/livekit-core/src/room/track/remote_video_track.rs b/crates/livekit-core/src/room/track/remote_video_track.rs index 57caf67..b3fc13f 100644 --- a/crates/livekit-core/src/room/track/remote_video_track.rs +++ b/crates/livekit-core/src/room/track/remote_video_track.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use crate::room::track::{impl_track_trait, TrackShared}; +#[derive(Debug)] pub struct RemoteVideoTrack { shared: TrackShared, } diff --git a/crates/livekit-core/src/rtc_engine/lk_runtime.rs b/crates/livekit-core/src/rtc_engine/lk_runtime.rs index 061455d..5ff17c6 100644 --- a/crates/livekit-core/src/rtc_engine/lk_runtime.rs +++ b/crates/livekit-core/src/rtc_engine/lk_runtime.rs @@ -1,6 +1,6 @@ use std::fmt::{Debug, Formatter}; -use tracing::{event, Level}; +use tracing::trace; use livekit_webrtc::peer_connection_factory::PeerConnectionFactory; use livekit_webrtc::webrtc::RTCRuntime; @@ -19,9 +19,9 @@ impl Debug for LKRuntime { } } -impl LKRuntime { - pub fn new() -> Self { - event!(Level::TRACE, "LKRuntime::new()"); +impl Default for LKRuntime { + fn default() -> Self { + trace!("LKRuntime::default()"); let rtc_runtime = RTCRuntime::new(); Self { pc_factory: PeerConnectionFactory::new(rtc_runtime.clone()), @@ -32,6 +32,6 @@ impl LKRuntime { impl Drop for LKRuntime { fn drop(&mut self) { - event!(Level::TRACE, "LKRuntime::drop()"); + trace!("LKRuntime::drop()"); } } diff --git a/crates/livekit-core/src/rtc_engine/mod.rs b/crates/livekit-core/src/rtc_engine/mod.rs index 4fbb08a..22b0b3d 100644 --- a/crates/livekit-core/src/rtc_engine/mod.rs +++ b/crates/livekit-core/src/rtc_engine/mod.rs @@ -1,75 +1,55 @@ -use parking_lot::Mutex; -use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; -use std::sync::{Arc, Weak}; -use std::time::Duration; - -use tokio::sync::{mpsc, Mutex as AsyncMutex}; - -use lazy_static::lazy_static; -use prost::Message; -use serde::{Deserialize, Serialize}; -use thiserror::Error; -use tokio::time::sleep; -use tracing::{debug, error, info, trace}; - -use crate::{proto, signal_client}; -use livekit_webrtc::data_channel::{DataChannel, DataChannelInit, DataSendError, DataState}; -use livekit_webrtc::jsep::{IceCandidate, SdpParseError, SessionDescription}; -use livekit_webrtc::media_stream::MediaStream; -use livekit_webrtc::peer_connection::{ - IceConnectionState, PeerConnectionState, RTCOfferAnswerOptions, -}; -use livekit_webrtc::peer_connection_factory::RTCConfiguration; +use futures::future::BoxFuture; +use futures::FutureExt; +use livekit_webrtc::data_channel::DataSendError; +use livekit_webrtc::jsep::SdpParseError; +use livekit_webrtc::media_stream::{MediaStream, MediaStreamTrackHandle}; use livekit_webrtc::rtc_error::RTCError; use livekit_webrtc::rtp_receiver::RtpReceiver; +use parking_lot::Mutex; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Weak}; +use std::time::Duration; +use thiserror::Error; +use tokio::sync::RwLock as AsyncRwLock; +use tokio::task::JoinHandle; +use tokio::time::{interval, Interval}; + +use lazy_static::lazy_static; +use tokio::sync::{mpsc, oneshot}; +use tracing::{error, info, warn}; -use crate::proto::data_packet::Value; use crate::proto::{ - data_packet, signal_request, signal_response, DataPacket, JoinResponse, ParticipantUpdate, - SignalTarget, TrickleRequest, + self as proto, data_packet, DataPacket, JoinResponse, ParticipantUpdate, SpeakerInfo, }; use crate::rtc_engine::lk_runtime::LKRuntime; -use crate::rtc_engine::pc_transport::PCTransport; -use crate::rtc_engine::rtc_events::{RTCEmitter, RTCEvent, RTCEvents}; -use crate::signal_client::{SignalClient, SignalError, SignalEvent, SignalEvents, SignalOptions}; +use crate::signal_client::{SignalError, SignalOptions}; + +use self::rtc_session::{RTCSession, SessionEvent, SessionEvents, SessionInfo}; mod lk_runtime; mod pc_transport; mod rtc_events; - -lazy_static! { - // Share one LKRuntime across all RTCEngine instances - static ref LK_RUNTIME: Mutex> = Mutex::new(Weak::new()); -} +mod rtc_session; pub(crate) type EngineEmitter = mpsc::Sender; pub(crate) type EngineEvents = mpsc::Receiver; pub(crate) type EngineResult = Result; -pub(crate) const MAX_ICE_CONNECT_TIMEOUT: Duration = Duration::from_secs(15); -pub(crate) const LOSSY_DC_LABEL: &str = "_lossy"; -pub(crate) const RELIABLE_DC_LABEL: &str = "_reliable"; - -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub(crate) enum PCState { - New, - Connected, - Disconnected, - Reconnecting, - Closed, -} - -#[derive(Serialize, Deserialize)] -#[allow(non_snake_case)] -struct IceCandidateJSON { - sdpMid: String, - sdpMLineIndex: i32, - candidate: String, +#[derive(Debug, Clone, Eq, PartialEq)] +#[repr(u8)] +pub enum SimulateScenario { + SignalReconnect, + Speaker, + NodeFailure, + ServerLeave, + Migration, + ForceTcp, + ForceTls, } #[derive(Error, Debug)] pub enum EngineError { - #[error("signal failure")] + #[error("signal failure: {0}")] Signal(#[from] SignalError), #[error("internal webrtc failure")] Rtc(#[from] RTCError), @@ -88,545 +68,404 @@ pub enum EngineError { } #[derive(Debug)] -pub(crate) enum EngineEvent { +pub enum EngineEvent { ParticipantUpdate(ParticipantUpdate), - AddTrack { - rtp_receiver: RtpReceiver, - streams: Vec, + MediaTrack { + track: MediaStreamTrackHandle, + stream: MediaStream, + receiver: RtpReceiver, }, + Data { + participant_sid: String, + payload: Vec, + kind: data_packet::Kind, + }, + SpeakersChanged { + speakers: Vec, + }, + ConnectionQuality { + updates: Vec, + }, + Resuming, + Resumed, + Restarting, + Restarted, + Disconnected, +} + +pub const RECONNECT_ATTEMPTS: u32 = 10; +pub const RECONNECT_INTERVAL: Duration = Duration::from_secs(5); + +lazy_static! { + // Share one LKRuntime across all RTCEngine instances + static ref LK_RUNTIME: Mutex> = Mutex::new(Weak::new()); +} +/// +/// Represents a running RTCSession with the ability to close the session +/// and the engine_task +#[derive(Debug)] +struct EngineHandle { + session: RTCSession, + engine_task: JoinHandle<()>, + close_sender: oneshot::Sender<()>, } #[derive(Debug)] struct EngineInner { - has_published: AtomicBool, - join_response: Mutex, - pc_state: AtomicU8, // Casted to PCState enum + lk_runtime: Arc, + session_info: Mutex>, // Last/Current Sessioninfo + running_handle: AsyncRwLock>, + opened: AtomicBool, + engine_emitter: EngineEmitter, - publisher_pc: AsyncMutex, - subscriber_pc: AsyncMutex, - - // Publisher data channels - // Used to send data to other participants ( The SFU forward the messages ) - lossy_dc: Mutex, - reliable_dc: Mutex, - - // Subscriber data channels - // These fields are never used, we just keep a strong reference to them, - // so we can receive data from other participants - sub_reliable_dc: Mutex>, - sub_lossy_dc: Mutex>, + // Reconnecting fields + reconnecting: AtomicBool, + full_reconnect: AtomicBool, + reconnect_interval: Mutex, } #[derive(Debug)] pub struct RTCEngine { - signal_client: Arc, - engine_inner: Arc, - - #[allow(unused)] - lk_runtime: Arc, // Keep a reference while we're using the RTCEngine + inner: Arc, } impl RTCEngine { - #[tracing::instrument(skip(url, token))] - pub(crate) async fn connect( - url: &str, - token: &str, - options: SignalOptions, - ) -> EngineResult<(RTCEngine, EngineEvents)> { + pub fn new() -> (Self, EngineEvents) { let mut lk_runtime = None; { let mut lk_runtime_ref = LK_RUNTIME.lock(); lk_runtime = lk_runtime_ref.upgrade(); if lk_runtime.is_none() { - let new_runtime = Arc::new(LKRuntime::new()); + let new_runtime = Arc::new(LKRuntime::default()); *lk_runtime_ref = Arc::downgrade(&new_runtime); lk_runtime = Some(new_runtime); } } - let lk_runtime = lk_runtime.unwrap(); - let (signal_client, mut signal_events) = SignalClient::connect(url, token, options).await?; + let (engine_emitter, engine_events) = mpsc::channel(8); + let inner = Arc::new(EngineInner { + lk_runtime: lk_runtime.unwrap(), + session_info: Default::default(), + running_handle: Default::default(), + opened: Default::default(), + engine_emitter, + reconnecting: Default::default(), + full_reconnect: Default::default(), + reconnect_interval: Mutex::new(interval(RECONNECT_INTERVAL)), + }); - let join_response = signal_client::utils::next_join_response(&mut signal_events).await?; - debug!("received JoinResponse: {:?}", join_response); - - let (engine_inner, rtc_events) = - Self::configure_engine(lk_runtime.clone(), join_response.clone())?; - let engine_inner = Arc::new(engine_inner); - let signal_client = Arc::new(signal_client); - - let (emitter, events) = mpsc::channel(8); - - tokio::spawn(Self::signal_task( - signal_client.clone(), - engine_inner.clone(), - signal_events, - emitter.clone(), - )); - - tokio::spawn(Self::engine_task( - signal_client.clone(), - engine_inner.clone(), - rtc_events, - emitter.clone(), - )); - - let rtc_engine = Self { - signal_client, - engine_inner, - lk_runtime, - }; - - if !join_response.subscriber_primary { - rtc_engine.negotiate_publisher().await?; - } - - Ok((rtc_engine, events)) + (Self { inner }, engine_events) } #[tracing::instrument] + pub async fn connect( + &self, + url: &str, + token: &str, + options: SignalOptions, + ) -> EngineResult<()> { + self.inner.connect(url, token, options).await + } + + #[tracing::instrument] + pub async fn close(&self) { + self.inner.close().await + } + + #[tracing::instrument(skip(data))] pub async fn publish_data( &self, data: &DataPacket, kind: data_packet::Kind, - ) -> Result<(), EngineError> { - self.ensure_publisher_connected(kind).await?; - self.data_channel(kind) - .lock() - .send(&data.encode_to_vec(), true) - .map_err(Into::into) - } - - pub fn join_response(&self) -> JoinResponse { - self.engine_inner.join_response.lock().clone() - } - - async fn engine_task( - signal_client: Arc, - engine_inner: Arc, - mut rtc_events: RTCEvents, - emitter: EngineEmitter, - ) { - while let Some(event) = rtc_events.recv().await { - if let Err(err) = Self::handle_rtc( - event, - signal_client.clone(), - engine_inner.clone(), - emitter.clone(), - ) - .await - { - error!("failed to handle rtc event: {:?}", err); - } - } - } - - async fn signal_task( - signal_client: Arc, - engine_inner: Arc, - mut signal_events: SignalEvents, - emitter: EngineEmitter, - ) { - while let Some(signal) = signal_events.recv().await { - match signal { - SignalEvent::Open => {} - SignalEvent::Signal(signal) => { - if let Err(err) = Self::handle_signal( - signal, - signal_client.clone(), - engine_inner.clone(), - emitter.clone(), - ) - .await - { - error!("failed to handle signal: {:?}", err); - } - } - SignalEvent::Close => { - // Try reconnect if this isn't expected - } - } - } - } - - async fn handle_rtc( - event: RTCEvent, - signal_client: Arc, - engine_inner: Arc, - emitter: EngineEmitter, ) -> EngineResult<()> { - match event { - RTCEvent::IceCandidate { - ice_candidate, - target, - } => { - let json = serde_json::to_string(&IceCandidateJSON { - sdpMid: ice_candidate.sdp_mid(), - sdpMLineIndex: ice_candidate.sdp_mline_index(), - candidate: ice_candidate.candidate(), - })?; - - trace!("sending ice_candidate ({:?}) - {:?}", target, ice_candidate); - - tokio::spawn(async move { - signal_client - .send(signal_request::Message::Trickle(TrickleRequest { - candidate_init: json, - target: target as i32, - })) - .await; - }); - } - RTCEvent::ConnectionChange { state, target } => { - // Reconnect if we've been disconnected unexpectedly - trace!("Connection change, {:?} {:?}", state, target); - let subscriber_primary = engine_inner.join_response.lock().subscriber_primary; - let is_primary = subscriber_primary && target == SignalTarget::Subscriber; - - if is_primary && state == PeerConnectionState::Disconnected { - let old_state = engine_inner - .pc_state - .swap(PCState::Connected as u8, Ordering::SeqCst); - if old_state == PCState::New as u8 { - // TODO(theomonnom) Handle disconnect - } - } else if state == PeerConnectionState::Failed { - engine_inner - .pc_state - .store(PCState::Disconnected as u8, Ordering::SeqCst); - // TODO(theomonnom) Handle disconnect - } - } - RTCEvent::DataChannel { - data_channel, - target, - } => { - if target == SignalTarget::Subscriber { - if data_channel.label() == RELIABLE_DC_LABEL { - *engine_inner.sub_reliable_dc.lock() = Some(data_channel); - } else { - *engine_inner.sub_lossy_dc.lock() = Some(data_channel); - } - } - } - RTCEvent::Offer { offer, target } => { - if target == SignalTarget::Publisher { - // Send the publisher offer to the server - tokio::spawn(async move { - signal_client - .send(signal_request::Message::Offer(proto::SessionDescription { - r#type: "offer".to_string(), - sdp: offer.to_string(), - })) - .await; - }); - } - } - RTCEvent::AddTrack { - rtp_receiver, - streams, - target, - } => { - if target == SignalTarget::Subscriber { - let _ = emitter - .send(EngineEvent::AddTrack { - rtp_receiver, - streams, - }) - .await; - } - } - RTCEvent::Data { data, binary } => { - if !binary { - Err(EngineError::Internal( - "text messages aren't supported".to_string(), - ))?; - } - - let data = DataPacket::decode(&*data)?; - match data.value.unwrap() { - Value::User(user) => { - // TODO(theomonnom) Send event - } - Value::Speaker(_) => { - // TODO(theomonnonm) - } - } - } - } + self.inner.wait_reconnection().await?; + self.inner + .running_handle + .read() + .await + .as_ref() + .unwrap() + .session + .publish_data(data, kind) + .await + } + pub async fn simulate_scenario(&self, scenario: SimulateScenario) -> EngineResult<()> { + self.inner.wait_reconnection().await?; + self.inner + .running_handle + .read() + .await + .as_ref() + .unwrap() + .session + .simulate_scenario(scenario) + .await; Ok(()) } - async fn handle_signal( - event: signal_response::Message, - signal_client: Arc, - engine_inner: Arc, - emitter: EngineEmitter, - ) -> EngineResult<()> { - match event { - signal_response::Message::Answer(answer) => { - trace!("received answer from the publisher: {:?}", answer); - - let sdp = SessionDescription::from(answer.r#type.parse().unwrap(), &answer.sdp)?; - engine_inner - .publisher_pc - .lock() - .await - .set_remote_description(sdp) - .await?; - } - signal_response::Message::Offer(offer) => { - // Handle the subscriber offer & send an answer to livekit-server - // We always get an offer from the server when connecting - trace!("received offer for the subscriber: {:?}", offer); - let sdp = SessionDescription::from(offer.r#type.parse().unwrap(), &offer.sdp)?; - - engine_inner - .subscriber_pc - .lock() - .await - .set_remote_description(sdp) - .await?; - let answer = engine_inner - .subscriber_pc - .lock() - .await - .peer_connection() - .create_answer(RTCOfferAnswerOptions::default()) - .await?; - engine_inner - .subscriber_pc - .lock() - .await - .peer_connection() - .set_local_description(answer.clone()) - .await?; - - tokio::spawn(async move { - signal_client - .send(signal_request::Message::Answer(proto::SessionDescription { - r#type: "answer".to_string(), - sdp: answer.to_string(), - })) - .await; - }); - } - signal_response::Message::Trickle(trickle) => { - // Add the IceCandidate received from the livekit-server - let json: IceCandidateJSON = serde_json::from_str(&trickle.candidate_init)?; - let ice = IceCandidate::from(&json.sdpMid, json.sdpMLineIndex, &json.candidate)?; - - trace!( - "received ice_candidate {:?} - {:?}", - SignalTarget::from_i32(trickle.target).unwrap(), - ice - ); - - if trickle.target == SignalTarget::Publisher as i32 { - engine_inner - .publisher_pc - .lock() - .await - .add_ice_candidate(ice) - .await?; - } else { - engine_inner - .subscriber_pc - .lock() - .await - .add_ice_candidate(ice) - .await?; - } - } - signal_response::Message::Update(update) => { - let _ = emitter.send(EngineEvent::ParticipantUpdate(update)).await; - } - _ => {} - } - - Ok(()) - } - - async fn ensure_publisher_connected(&self, kind: data_packet::Kind) -> EngineResult<()> { - if !self.join_response().subscriber_primary { - return Ok(()); - } - - let publisher = &self.engine_inner.publisher_pc; - { - let mut publisher = publisher.lock().await; - if !publisher.is_connected() - && publisher.peer_connection().ice_connection_state() - != IceConnectionState::IceConnectionChecking - { - let _ = self.negotiate_publisher().await; - } - } - - let dc = self.data_channel(kind); - if dc.lock().state() == DataState::Open { - return Ok(()); - } - - // Wait until the PeerConnection is connected - let wait_connected = async move { - while publisher.lock().await.is_connected() && dc.lock().state() == DataState::Open { - sleep(Duration::from_millis(50)).await; - } - }; - - tokio::select! { - _ = wait_connected => Ok(()), - _ = sleep(MAX_ICE_CONNECT_TIMEOUT) => { - let err = EngineError::Connection("could not establish publisher connection: timeout".to_string()); - error!(error = ?err); - Err(err) - } - } - } - - async fn negotiate_publisher(&self) -> EngineResult<()> { - self.engine_inner - .has_published - .store(true, Ordering::SeqCst); - if let Err(err) = self - .engine_inner - .publisher_pc - .lock() - .await - .negotiate() - .await - { - error!("failed to negotiate the publisher: {:?}", err); - Err(err)? + pub fn join_response(&self) -> Option { + if let Some(info) = self.inner.session_info.lock().as_ref() { + Some(info.join_response.clone()) } else { - Ok(()) - } - } - - fn configure_engine( - lk_runtime: Arc, - join_response: JoinResponse, - ) -> EngineResult<(EngineInner, RTCEvents)> { - let (rtc_emitter, events) = mpsc::unbounded_channel(); - let rtc_config = RTCConfiguration::from(join_response.clone()); - - let mut publisher_pc = PCTransport::new( - lk_runtime - .pc_factory - .create_peer_connection(rtc_config.clone())?, - ); - - let mut subscriber_pc = PCTransport::new( - lk_runtime - .pc_factory - .create_peer_connection(rtc_config.clone())?, - ); - - let mut lossy_dc = publisher_pc.peer_connection().create_data_channel( - LOSSY_DC_LABEL, - DataChannelInit { - ordered: true, - max_retransmits: Some(0), - ..DataChannelInit::default() - }, - )?; - - let mut reliable_dc = publisher_pc.peer_connection().create_data_channel( - RELIABLE_DC_LABEL, - DataChannelInit { - ordered: true, - ..DataChannelInit::default() - }, - )?; - - publisher_pc - .peer_connection() - .on_ice_candidate(rtc_events::on_ice_candidate( - SignalTarget::Publisher, - rtc_emitter.clone(), - )); - subscriber_pc - .peer_connection() - .on_ice_candidate(rtc_events::on_ice_candidate( - SignalTarget::Subscriber, - rtc_emitter.clone(), - )); - - publisher_pc.on_offer(rtc_events::on_offer( - SignalTarget::Publisher, - rtc_emitter.clone(), - )); - subscriber_pc.on_offer(rtc_events::on_offer( - SignalTarget::Subscriber, - rtc_emitter.clone(), - )); - - publisher_pc - .peer_connection() - .on_data_channel(rtc_events::on_data_channel( - SignalTarget::Publisher, - rtc_emitter.clone(), - )); - subscriber_pc - .peer_connection() - .on_data_channel(rtc_events::on_data_channel( - SignalTarget::Subscriber, - rtc_emitter.clone(), - )); - - publisher_pc - .peer_connection() - .on_add_track(rtc_events::on_add_track( - SignalTarget::Publisher, - rtc_emitter.clone(), - )); - subscriber_pc - .peer_connection() - .on_add_track(rtc_events::on_add_track( - SignalTarget::Subscriber, - rtc_emitter.clone(), - )); - - publisher_pc - .peer_connection() - .on_connection_change(rtc_events::on_connection_change( - SignalTarget::Publisher, - rtc_emitter.clone(), - )); - subscriber_pc - .peer_connection() - .on_connection_change(rtc_events::on_connection_change( - SignalTarget::Subscriber, - rtc_emitter.clone(), - )); - - lossy_dc.on_message(rtc_events::on_message(rtc_emitter.clone())); - reliable_dc.on_message(rtc_events::on_message(rtc_emitter.clone())); - - Ok(( - EngineInner { - has_published: AtomicBool::new(false), - join_response: Mutex::new(join_response), - pc_state: AtomicU8::new(PCState::New as u8), - publisher_pc: AsyncMutex::new(publisher_pc), - subscriber_pc: AsyncMutex::new(subscriber_pc), - lossy_dc: Mutex::new(lossy_dc), - reliable_dc: Mutex::new(reliable_dc), - sub_lossy_dc: Mutex::new(None), - sub_reliable_dc: Mutex::new(None), - }, - events, - )) - } - - fn data_channel(&self, kind: data_packet::Kind) -> &Mutex { - if kind == data_packet::Kind::Reliable { - &self.engine_inner.reliable_dc - } else { - &self.engine_inner.lossy_dc + None } } } + +impl EngineInner { + async fn engine_task( + self: Arc, + mut session_events: SessionEvents, + mut close_receiver: oneshot::Receiver<()>, + ) { + loop { + tokio::select! { + res = session_events.recv() => { + if let Some(event) = res { + if let Err(err) = self.on_session_event(event).await { + error!("failed to handle session event: {:?}", err); + } + } else { + panic!("rtc_sessions has been closed unexpectedly"); + } + }, + _ = &mut close_receiver => { + break; + } + } + } + } + + async fn on_session_event(self: &Arc, event: SessionEvent) -> EngineResult<()> { + match event { + SessionEvent::Close { + source, + reason, + can_reconnect, + retry_now, + full_reconnect, + } => { + info!("received session close: {}, {:?}", source, reason); + if can_reconnect { + self.clone().try_reconnect(retry_now, full_reconnect); + } else { + self.close().await; + } + } + SessionEvent::Data { + participant_sid, + payload, + kind, + } => { + let _ = self + .engine_emitter + .send(EngineEvent::Data { + participant_sid, + payload, + kind, + }) + .await; + } + SessionEvent::MediaTrack { + track, + stream, + receiver, + } => { + let _ = self + .engine_emitter + .send(EngineEvent::MediaTrack { + track, + stream, + receiver, + }) + .await; + } + SessionEvent::SpeakersChanged { speakers } => { + let _ = self + .engine_emitter + .send(EngineEvent::SpeakersChanged { speakers }) + .await; + } + SessionEvent::ConnectionQuality { updates } => { + let _ = self + .engine_emitter + .send(EngineEvent::ConnectionQuality { updates }) + .await; + } + SessionEvent::Connected => {} + } + Ok(()) + } + + fn connect<'a>( + self: &'a Arc, + url: &'a str, + token: &'a str, + options: SignalOptions, + ) -> BoxFuture<'a, EngineResult<()>> { + async { + let (session_emitter, session_events) = mpsc::unbounded_channel(); + let session = RTCSession::connect( + url, + token, + options, + self.lk_runtime.clone(), + session_emitter, + ) + .await?; + + let (close_sender, close_receiver) = oneshot::channel(); + let engine_task = + tokio::spawn(self.clone().engine_task(session_events, close_receiver)); + + *self.session_info.lock() = Some(session.info().clone()); + *self.running_handle.write().await = Some(EngineHandle { + session, + engine_task, + close_sender, + }); + + self.opened.store(true, Ordering::SeqCst); + Ok(()) + } + .boxed() + } + + async fn terminate_session(&self) { + if let Some(handle) = self.running_handle.write().await.take() { + handle.session.close().await; + let _ = handle.close_sender.send(()); + let _ = handle.engine_task.await; + } + } + + async fn close(&self) { + self.opened.store(false, Ordering::SeqCst); + self.terminate_session().await; + let _ = self.engine_emitter.send(EngineEvent::Disconnected).await; + } + + // Wait for the reconnection task to finish + // Return directly if no open RTCSession + async fn wait_reconnection(&self) -> EngineResult<()> { + if !self.opened.load(Ordering::SeqCst) { + Err(EngineError::Connection("not opened".to_owned()))? + } + + while self.reconnecting.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + + if self.running_handle.read().await.is_none() { + Err(EngineError::Connection("reconnection failed".to_owned()))? + } + + Ok(()) + } + + /// Start the reconnect task if not already started + fn try_reconnect(self: Arc, retry_now: bool, full_reconnect: bool) { + if !self.opened.load(Ordering::SeqCst) { + return; + } + + if self.reconnecting.load(Ordering::SeqCst) { + if retry_now { + self.reconnect_interval.lock().reset(); + self.full_reconnect.store(full_reconnect, Ordering::SeqCst); + } + return; + } + + warn!("reconnecting RTCEngine..."); + + self.reconnecting.store(true, Ordering::SeqCst); + self.full_reconnect.store(full_reconnect, Ordering::SeqCst); + self.reconnect_interval.lock().reset(); + tokio::spawn({ + let inner = self.clone(); + async move { + let res = inner.reconnect_task().await; + inner.reconnecting.store(false, Ordering::SeqCst); + + if res.is_ok() { + warn!("RTCEngine successfully reconnected") + } else { + error!("failed to reconnect after {} attemps", RECONNECT_ATTEMPTS); + inner.close().await; + } + } + }); + } + + /// Called every time the PeerConnection or the SignalClient is closed + /// We first try to resume the connection, if it fails, we start a full reconnect. + async fn reconnect_task(self: &Arc) -> EngineResult<()> { + for i in 0..RECONNECT_ATTEMPTS { + if !self.opened.load(Ordering::Acquire) { + // The user closed the RTCEngine, cancel the reconnection task + return Ok(()); + } + + if self.full_reconnect.load(Ordering::SeqCst) { + if i == 0 { + let _ = self.engine_emitter.send(EngineEvent::Restarting).await; + } + + info!("restarting connection... attempt: {}", i); + if let Err(err) = self.try_restart_connection().await { + error!("restarting connection failed: {}", err); + } else { + let _ = self.engine_emitter.send(EngineEvent::Restarted).await; + return Ok(()); + } + } else { + if i == 0 { + let _ = self.engine_emitter.send(EngineEvent::Resuming).await; + } + + info!("resuming connection... attempt: {}", i); + if let Err(err) = self.try_resume_connection().await { + error!("resuming connection failed: {}", err); + if let EngineError::Signal(_) = err { + self.full_reconnect.store(true, Ordering::SeqCst); + } + } else { + let _ = self.engine_emitter.send(EngineEvent::Resumed).await; + return Ok(()); + } + } + + self.reconnect_interval.lock().tick().await; + } + + Err(EngineError::Connection("failed to reconnect".to_owned())) + } + + /// Try to recover the connection by doing a full reconnect. + /// It recreates a new RTCSession + async fn try_restart_connection(self: &Arc) -> EngineResult<()> { + let info = self.session_info.lock().clone().unwrap(); + self.terminate_session().await; + self.connect(&info.url, &info.token, info.options).await?; + self.running_handle + .read() + .await + .as_ref() + .unwrap() + .session + .wait_pc_connection() + .await + + // TODO(theomonnom): Resend SignalClient queue + } + + /// Try to restart the current session + async fn try_resume_connection(&self) -> EngineResult<()> { + let handle = self.running_handle.read().await; + handle.as_ref().unwrap().session.restart().await?; + handle.as_ref().unwrap().session.wait_pc_connection().await + } +} diff --git a/crates/livekit-core/src/rtc_engine/pc_transport.rs b/crates/livekit-core/src/rtc_engine/pc_transport.rs index 8d9676a..34b2fa2 100644 --- a/crates/livekit-core/src/rtc_engine/pc_transport.rs +++ b/crates/livekit-core/src/rtc_engine/pc_transport.rs @@ -11,6 +11,8 @@ use livekit_webrtc::peer_connection::{ }; use livekit_webrtc::rtc_error::RTCError; +use crate::proto::SignalTarget; + const NEGOTIATION_FREQUENCY: Duration = Duration::from_millis(150); pub type OnOfferHandler = Box< @@ -20,11 +22,12 @@ pub type OnOfferHandler = Box< >; pub struct PCTransport { + signal_target: SignalTarget, peer_connection: PeerConnection, pending_candidates: Vec, on_offer_handler: Option, - restarting_ice: bool, renegotiate: bool, + restarting_ice: bool, } impl Debug for PCTransport { @@ -34,8 +37,9 @@ impl Debug for PCTransport { } impl PCTransport { - pub fn new(peer_connection: PeerConnection) -> Self { + pub fn new(peer_connection: PeerConnection, signal_target: SignalTarget) -> Self { Self { + signal_target, peer_connection, pending_candidates: Vec::default(), on_offer_handler: None, @@ -54,24 +58,37 @@ impl PCTransport { &mut self.peer_connection } + pub fn signal_target(&self) -> SignalTarget { + self.signal_target.clone() + } + pub fn on_offer(&mut self, handler: OnOfferHandler) { self.on_offer_handler = Some(handler); } - #[tracing::instrument] + pub fn prepare_ice_restart(&mut self) { + self.restarting_ice = true; + } + + pub fn close(&mut self) { + self.peer_connection.close(); + } + + #[tracing::instrument(level = Level::DEBUG)] pub async fn add_ice_candidate(&mut self, ice_candidate: IceCandidate) -> Result<(), RTCError> { - if self.peer_connection.remote_description().is_none() { - self.pending_candidates.push(ice_candidate); + if self.peer_connection.remote_description().is_some() && !self.restarting_ice { + self.peer_connection + .add_ice_candidate(ice_candidate) + .await?; + return Ok(()); } - self.peer_connection - .add_ice_candidate(ice_candidate) - .await?; + self.pending_candidates.push(ice_candidate); Ok(()) } - #[tracing::instrument] + #[tracing::instrument(level = Level::DEBUG)] pub async fn set_remote_description( &mut self, remote_description: SessionDescription, @@ -94,15 +111,33 @@ impl PCTransport { Ok(()) } - #[tracing::instrument] + #[tracing::instrument(level = Level::DEBUG)] pub async fn negotiate(&mut self) -> Result<(), RTCError> { // TODO(theomonnom) Debounce here with NEGOTIATION_FREQUENCY self.create_and_send_offer(RTCOfferAnswerOptions::default()) .await } - #[tracing::instrument] - async fn create_and_send_offer( + #[tracing::instrument(level = Level::DEBUG)] + pub async fn create_anwser( + &mut self, + offer: SessionDescription, + options: RTCOfferAnswerOptions, + ) -> Result { + self.set_remote_description(offer).await?; + let answer = self + .peer_connection() + .create_answer(RTCOfferAnswerOptions::default()) + .await?; + self.peer_connection() + .set_local_description(answer.clone()) + .await?; + + Ok(answer) + } + + #[tracing::instrument(level = Level::DEBUG)] + pub async fn create_and_send_offer( &mut self, options: RTCOfferAnswerOptions, ) -> Result<(), RTCError> { diff --git a/crates/livekit-core/src/rtc_engine/rtc_events.rs b/crates/livekit-core/src/rtc_engine/rtc_events.rs index 78712d7..6ccd0ed 100644 --- a/crates/livekit-core/src/rtc_engine/rtc_events.rs +++ b/crates/livekit-core/src/rtc_engine/rtc_events.rs @@ -11,11 +11,13 @@ use tokio::sync::mpsc; use crate::proto::SignalTarget; use crate::rtc_engine::pc_transport::OnOfferHandler; -pub(super) type RTCEmitter = mpsc::UnboundedSender; -pub(super) type RTCEvents = mpsc::UnboundedReceiver; +use super::pc_transport::PCTransport; + +pub type RTCEmitter = mpsc::UnboundedSender; +pub type RTCEvents = mpsc::UnboundedReceiver; #[derive(Debug)] -pub(super) enum RTCEvent { +pub enum RTCEvent { IceCandidate { ice_candidate: IceCandidate, target: SignalTarget, @@ -28,6 +30,7 @@ pub(super) enum RTCEvent { data_channel: DataChannel, target: SignalTarget, }, + // TODO (theomonnom): Move Offer to PCTransport Offer { offer: SessionDescription, target: SignalTarget, @@ -43,19 +46,16 @@ pub(super) enum RTCEvent { }, } -/// Handlers used to forward event to a channel +/// Handlers used to forward events to a channel /// Every callback here is called on the signaling thread -pub(super) fn on_connection_change( - target: SignalTarget, - emitter: RTCEmitter, -) -> OnConnectionChangeHandler { +fn on_connection_change(target: SignalTarget, emitter: RTCEmitter) -> OnConnectionChangeHandler { Box::new(move |state| { let _ = emitter.send(RTCEvent::ConnectionChange { state, target }); }) } -pub(super) fn on_ice_candidate(target: SignalTarget, emitter: RTCEmitter) -> OnIceCandidateHandler { +fn on_ice_candidate(target: SignalTarget, emitter: RTCEmitter) -> OnIceCandidateHandler { Box::new(move |ice_candidate| { let _ = emitter.send(RTCEvent::IceCandidate { ice_candidate, @@ -64,7 +64,7 @@ pub(super) fn on_ice_candidate(target: SignalTarget, emitter: RTCEmitter) -> OnI }) } -pub(super) fn on_offer(target: SignalTarget, emitter: RTCEmitter) -> OnOfferHandler { +fn on_offer(target: SignalTarget, emitter: RTCEmitter) -> OnOfferHandler { Box::new(move |offer| { let _ = emitter.send(RTCEvent::Offer { offer, target }); @@ -72,7 +72,7 @@ pub(super) fn on_offer(target: SignalTarget, emitter: RTCEmitter) -> OnOfferHand }) } -pub(super) fn on_data_channel(target: SignalTarget, emitter: RTCEmitter) -> OnDataChannelHandler { +fn on_data_channel(target: SignalTarget, emitter: RTCEmitter) -> OnDataChannelHandler { Box::new(move |mut data_channel| { data_channel.on_message(on_message(emitter.clone())); @@ -83,7 +83,7 @@ pub(super) fn on_data_channel(target: SignalTarget, emitter: RTCEmitter) -> OnDa }) } -pub(super) fn on_add_track(target: SignalTarget, emitter: RTCEmitter) -> OnAddTrackHandler { +fn on_add_track(target: SignalTarget, emitter: RTCEmitter) -> OnAddTrackHandler { Box::new(move |rtp_receiver, streams| { let _ = emitter.send(RTCEvent::AddTrack { rtp_receiver, @@ -93,7 +93,28 @@ pub(super) fn on_add_track(target: SignalTarget, emitter: RTCEmitter) -> OnAddTr }) } -pub(super) fn on_message(emitter: RTCEmitter) -> OnMessageHandler { +pub fn forward_pc_events(transport: &mut PCTransport, rtc_emitter: RTCEmitter) { + let signal_target = transport.signal_target(); + transport + .peer_connection() + .on_ice_candidate(on_ice_candidate(signal_target, rtc_emitter.clone())); + + transport + .peer_connection() + .on_data_channel(on_data_channel(signal_target, rtc_emitter.clone())); + + transport + .peer_connection() + .on_add_track(on_add_track(signal_target, rtc_emitter.clone())); + + transport + .peer_connection() + .on_connection_change(on_connection_change(signal_target, rtc_emitter.clone())); + + transport.on_offer(on_offer(transport.signal_target(), rtc_emitter.clone())); +} + +fn on_message(emitter: RTCEmitter) -> OnMessageHandler { Box::new(move |data, binary| { let _ = emitter.send(RTCEvent::Data { data: data.to_vec(), @@ -101,3 +122,7 @@ pub(super) fn on_message(emitter: RTCEmitter) -> OnMessageHandler { }); }) } + +pub fn forward_dc_events(dc: &mut DataChannel, rtc_emitter: RTCEmitter) { + dc.on_message(on_message(rtc_emitter.clone())); +} diff --git a/crates/livekit-core/src/rtc_engine/rtc_session.rs b/crates/livekit-core/src/rtc_engine/rtc_session.rs new file mode 100644 index 0000000..bde216f --- /dev/null +++ b/crates/livekit-core/src/rtc_engine/rtc_session.rs @@ -0,0 +1,743 @@ +use livekit_webrtc::media_stream::{MediaStream, MediaStreamTrackHandle}; +use livekit_webrtc::rtp_receiver::RtpReceiver; +use parking_lot::Mutex; +use std::convert::TryInto; +use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; +use std::sync::Arc; +use std::time::Duration; +use tokio::task::JoinHandle; + +use tokio::sync::{mpsc, watch, Mutex as AsyncMutex}; +use tokio::time::sleep; + +use prost::Message; +use serde::{Deserialize, Serialize}; +use tracing::{debug, error, trace, warn}; + +use crate::{proto, signal_client}; +use livekit_webrtc::data_channel::{DataChannel, DataChannelInit, DataState}; +use livekit_webrtc::jsep::{IceCandidate, SessionDescription}; +use livekit_webrtc::peer_connection::{ + IceConnectionState, PeerConnectionState, RTCOfferAnswerOptions, +}; +use livekit_webrtc::peer_connection_factory::RTCConfiguration; + +use crate::proto::data_packet::Value; +use crate::proto::{ + data_packet, signal_request, signal_response, CandidateProtocol, DataPacket, DisconnectReason, + JoinResponse, SignalTarget, TrickleRequest, +}; +use crate::rtc_engine::lk_runtime::LKRuntime; +use crate::rtc_engine::pc_transport::PCTransport; +use crate::rtc_engine::rtc_events::{RTCEvent, RTCEvents}; +use crate::signal_client::{SignalClient, SignalEvent, SignalEvents, SignalOptions}; + +use super::{rtc_events, EngineError, EngineResult, SimulateScenario}; + +pub const MAX_ICE_CONNECT_TIMEOUT: Duration = Duration::from_secs(15); +pub const LOSSY_DC_LABEL: &str = "_lossy"; +pub const RELIABLE_DC_LABEL: &str = "_reliable"; + +pub type SessionEmitter = mpsc::UnboundedSender; +pub type SessionEvents = mpsc::UnboundedReceiver; + +#[derive(Debug)] +pub enum SessionEvent { + Data { + participant_sid: String, + payload: Vec, + kind: proto::data_packet::Kind, + }, + MediaTrack { + track: MediaStreamTrackHandle, + stream: MediaStream, + receiver: RtpReceiver, + }, + SpeakersChanged { + speakers: Vec, + }, + ConnectionQuality { + updates: Vec, + }, + // TODO(theomonnom): Move entirely the reconnection logic on mod.rs + Close { + source: String, + reason: DisconnectReason, + can_reconnect: bool, + full_reconnect: bool, + retry_now: bool, + }, + Connected, +} + +#[repr(u8)] +pub enum PCState { + New, + Connected, + Disconnected, + Reconnecting, + Closed, +} + +impl TryInto for u8 { + type Error = &'static str; + + fn try_into(self) -> Result { + match self { + 0 => Ok(PCState::New), + 1 => Ok(PCState::Connected), + 2 => Ok(PCState::Disconnected), + 3 => Ok(PCState::Reconnecting), + 4 => Ok(PCState::Closed), + _ => Err("invalid PCState"), + } + } +} + +#[derive(Serialize, Deserialize)] +#[allow(non_snake_case)] +struct IceCandidateJSON { + sdpMid: String, + sdpMLineIndex: i32, + candidate: String, +} + +#[derive(Debug, Clone, Default)] +pub struct SessionInfo { + pub url: String, + pub token: String, + pub options: SignalOptions, + pub join_response: JoinResponse, +} + +/// Fields shared with rtc_task and signal_task +#[derive(Debug)] +struct SessionInner { + info: SessionInfo, + signal_client: Arc, + pc_state: AtomicU8, // PCState + has_published: AtomicBool, + + publisher_pc: AsyncMutex, + subscriber_pc: AsyncMutex, + + // Publisher data channels + // used to send data to other participants ( The SFU forwards the messages ) + lossy_dc: DataChannel, + reliable_dc: DataChannel, + + // Keep a strong reference to the subscriber datachannels, + // so we can receive data from other participants + subscriber_dc: Mutex>, + + emitter: SessionEmitter, +} + +/// This struct holds a WebRTC session +/// The session changes at every reconnection +/// +/// RTCSession is also responsable for the signaling and the negotation +#[derive(Debug)] +pub struct RTCSession { + lk_runtime: Arc, + inner: Arc, + close_emitter: watch::Sender, // false = is_running + signal_task: JoinHandle<()>, + rtc_task: JoinHandle<()>, +} + +impl RTCSession { + pub async fn connect( + url: &str, + token: &str, + options: SignalOptions, + lk_runtime: Arc, + session_emitter: SessionEmitter, + ) -> EngineResult { + // Connect to the SignalClient + let (signal_client, mut signal_events) = SignalClient::new(); + let signal_client = Arc::new(signal_client); + signal_client.connect(url, token, options.clone()).await?; + let join_response = signal_client::utils::next_join_response(&mut signal_events).await?; + debug!("received JoinResponse: {:?}", join_response); + + let (rtc_emitter, rtc_events) = mpsc::unbounded_channel(); + let rtc_config = RTCConfiguration::from(join_response.clone()); + + let mut publisher_pc = PCTransport::new( + lk_runtime + .pc_factory + .create_peer_connection(rtc_config.clone())?, + SignalTarget::Publisher, + ); + + let mut subscriber_pc = PCTransport::new( + lk_runtime + .pc_factory + .create_peer_connection(rtc_config.clone())?, + SignalTarget::Subscriber, + ); + + let mut lossy_dc = publisher_pc.peer_connection().create_data_channel( + LOSSY_DC_LABEL, + DataChannelInit { + ordered: true, + max_retransmits: Some(0), + ..DataChannelInit::default() + }, + )?; + + let mut reliable_dc = publisher_pc.peer_connection().create_data_channel( + RELIABLE_DC_LABEL, + DataChannelInit { + ordered: true, + ..DataChannelInit::default() + }, + )?; + + // Forward events received in the Signaling Thread to our rtc channel + rtc_events::forward_pc_events(&mut publisher_pc, rtc_emitter.clone()); + rtc_events::forward_pc_events(&mut subscriber_pc, rtc_emitter.clone()); + rtc_events::forward_dc_events(&mut lossy_dc, rtc_emitter.clone()); + rtc_events::forward_dc_events(&mut reliable_dc, rtc_emitter.clone()); + + let session_info = SessionInfo { + url: url.to_owned(), + token: token.to_owned(), + options, + join_response, + }; + + let (close_emitter, close_receiver) = watch::channel(false); + let inner = Arc::new(SessionInner { + info: session_info, + pc_state: AtomicU8::new(PCState::New as u8), + has_published: Default::default(), + signal_client, + publisher_pc: AsyncMutex::new(publisher_pc), + subscriber_pc: AsyncMutex::new(subscriber_pc), + lossy_dc, + reliable_dc, + subscriber_dc: Default::default(), + emitter: session_emitter, + }); + + // Start session tasks + let signal_task = tokio::spawn( + inner + .clone() + .signal_task(signal_events, close_receiver.clone()), + ); + let rtc_task = tokio::spawn(inner.clone().rtc_task(rtc_events, close_receiver.clone())); + + if !inner.info.join_response.subscriber_primary { + inner.negotiate_publisher().await?; + } + + let session = Self { + lk_runtime, + inner: inner.clone(), + close_emitter, + signal_task, + rtc_task, + }; + + Ok(session) + } + + /// Close the PeerConnections and the SignalClient + #[tracing::instrument] + pub async fn close(self) { + // Close the tasks + let _ = self.close_emitter.send(true); + let _ = self.rtc_task.await; + let _ = self.signal_task.await; + self.inner.close().await; + } + + pub async fn publish_data( + &self, + data: &DataPacket, + kind: data_packet::Kind, + ) -> Result<(), EngineError> { + self.inner.publish_data(data, kind).await + } + + pub async fn restart(&self) -> EngineResult<()> { + self.inner.restart_session().await + } + + pub async fn wait_pc_connection(&self) -> EngineResult<()> { + self.inner.wait_pc_connection().await + } + + pub async fn simulate_scenario(&self, scenario: SimulateScenario) { + self.inner.simulate_scenario(scenario).await + } +} + +impl RTCSession { + pub fn info(&self) -> &SessionInfo { + &self.inner.info + } + + pub fn state(&self) -> PCState { + self.inner + .pc_state + .load(Ordering::SeqCst) + .try_into() + .unwrap() + } + + pub fn publisher(&self) -> &AsyncMutex { + &self.inner.publisher_pc + } + + pub fn subscriber(&self) -> &AsyncMutex { + &self.inner.subscriber_pc + } + + pub fn signal_client(&self) -> &Arc { + &self.inner.signal_client + } + + pub fn data_channel(&self, kind: data_packet::Kind) -> &DataChannel { + &self.inner.data_channel(kind) + } +} + +impl SessionInner { + async fn rtc_task( + self: Arc, + mut rtc_events: RTCEvents, + mut close_receiver: watch::Receiver, + ) { + loop { + tokio::select! { + res = rtc_events.recv() => { + if let Some(event) = res { + if let Err(err) = self.on_rtc_event(event).await { + error!("failed to handle rtc event: {:?}", err); + } + } else { + panic!("rtc_events has been closed unexpectedly"); + } + }, + _ = close_receiver.changed() => { + break; + } + } + } + } + + async fn signal_task( + self: Arc, + mut signal_events: SignalEvents, + mut close_receiver: watch::Receiver, + ) { + loop { + tokio::select! { + res = signal_events.recv() => { + if let Some(signal) = res { + match signal { + SignalEvent::Open => {} + SignalEvent::Signal(signal) => { + if let Err(err) = self.on_signal_event(signal).await { + error!("failed to handle signal: {:?}", err); + } + } + SignalEvent::Close => { + self.on_session_disconnected("SignalClient closed", DisconnectReason::UnknownReason, true, false, false); + } + } + } else { + panic!("signal_events has been closed unexpectedly"); + } + + }, + _ = close_receiver.changed() => { + break; + } + } + } + } + + async fn on_signal_event(&self, event: signal_response::Message) -> EngineResult<()> { + match event { + signal_response::Message::Answer(answer) => { + trace!("received publisher answer: {:?}", answer); + let answer = SessionDescription::from(answer.r#type.parse().unwrap(), &answer.sdp)?; + self.publisher_pc + .lock() + .await + .set_remote_description(answer) + .await?; + } + signal_response::Message::Offer(offer) => { + trace!("received subscriber offer: {:?}", offer); + let offer = SessionDescription::from(offer.r#type.parse().unwrap(), &offer.sdp)?; + let answer = self + .subscriber_pc + .lock() + .await + .create_anwser(offer, RTCOfferAnswerOptions::default()) + .await?; + + self.signal_client + .send(signal_request::Message::Answer(proto::SessionDescription { + r#type: "answer".to_string(), + sdp: answer.to_string(), + })) + .await; + } + signal_response::Message::Trickle(trickle) => { + let target = SignalTarget::from_i32(trickle.target).unwrap(); + let ice_candidate = { + let json = serde_json::from_str::(&trickle.candidate_init)?; + IceCandidate::from(&json.sdpMid, json.sdpMLineIndex, &json.candidate)? + }; + + trace!("received ice_candidate {:?} {:?}", target, ice_candidate); + + if target == SignalTarget::Publisher { + self.publisher_pc + .lock() + .await + .add_ice_candidate(ice_candidate) + .await?; + } else { + self.subscriber_pc + .lock() + .await + .add_ice_candidate(ice_candidate) + .await?; + } + } + signal_response::Message::Leave(leave) => { + self.on_session_disconnected( + "received leave", + leave.reason(), + leave.can_reconnect, + true, + true, + ); + } + signal_response::Message::SpeakersChanged(speaker) => { + let _ = self.emitter.send(SessionEvent::SpeakersChanged { + speakers: speaker.speakers, + }); + } + signal_response::Message::ConnectionQuality(quality) => { + let _ = self.emitter.send(SessionEvent::ConnectionQuality { + updates: quality.updates, + }); + } + _ => {} + } + + Ok(()) + } + + async fn on_rtc_event(&self, event: RTCEvent) -> EngineResult<()> { + match event { + RTCEvent::IceCandidate { + ice_candidate, + target, + } => { + self.signal_client + .send(signal_request::Message::Trickle(TrickleRequest { + candidate_init: serde_json::to_string(&IceCandidateJSON { + sdpMid: ice_candidate.sdp_mid(), + sdpMLineIndex: ice_candidate.sdp_mline_index(), + candidate: ice_candidate.candidate(), + })?, + target: target as i32, + })) + .await; + } + RTCEvent::ConnectionChange { state, target } => { + trace!("connection change, {:?} {:?}", state, target); + let is_primary = self.info.join_response.subscriber_primary + && target == SignalTarget::Subscriber; + + if is_primary && state == PeerConnectionState::Connected { + let old_state = self + .pc_state + .swap(PCState::Connected as u8, Ordering::SeqCst); + if old_state == PCState::New as u8 { + let _ = self.emitter.send(SessionEvent::Connected); + } + } else if state == PeerConnectionState::Failed { + self.pc_state + .store(PCState::Disconnected as u8, Ordering::SeqCst); + + self.on_session_disconnected( + "pc_state failed", + DisconnectReason::UnknownReason, + true, + false, + false, + ); + } + } + RTCEvent::DataChannel { + data_channel, + target: _, + } => { + self.subscriber_dc.lock().push(data_channel); + } + RTCEvent::Offer { offer, target: _ } => { + // Send the publisher offer to the server + self.signal_client + .send(signal_request::Message::Offer(proto::SessionDescription { + r#type: "offer".to_string(), + sdp: offer.to_string(), + })) + .await; + } + RTCEvent::AddTrack { + rtp_receiver, + mut streams, + target: _, + } => { + if !streams.is_empty() { + let _ = self.emitter.send(SessionEvent::MediaTrack { + track: rtp_receiver.track(), + stream: streams.remove(0), + receiver: rtp_receiver, + }); + } else { + warn!("AddTrack event with no streams"); + } + } + RTCEvent::Data { data, binary } => { + if !binary { + Err(EngineError::Internal( + "text messages aren't supported".to_string(), + ))?; + } + + let data = DataPacket::decode(&*data)?; + match data.value.unwrap() { + Value::User(user) => { + let _ = self.emitter.send(SessionEvent::Data { + participant_sid: user.participant_sid, + payload: user.payload, + kind: data_packet::Kind::from_i32(data.kind).unwrap(), + }); + } + Value::Speaker(_) => {} + } + } + } + + Ok(()) + } + + /// Called when the SignalClient or one of the PeerConnection has lost the connection + /// The RTCEngine may try a reconnect. + fn on_session_disconnected( + &self, + source: &str, + reason: DisconnectReason, + can_reconnect: bool, + retry_now: bool, + full_reconnect: bool, + ) { + let _ = self.emitter.send(SessionEvent::Close { + source: source.to_owned(), + reason, + can_reconnect, + retry_now, + full_reconnect, + }); + } + + #[tracing::instrument] + async fn close(&self) { + self.signal_client.close().await; + self.publisher_pc.lock().await.close(); + self.subscriber_pc.lock().await.close(); + } + + #[tracing::instrument] + async fn simulate_scenario(&self, scenario: SimulateScenario) { + match scenario { + SimulateScenario::SignalReconnect => { + self.signal_client.close().await; + } + SimulateScenario::Speaker => { + self.signal_client + .send(signal_request::Message::Simulate(proto::SimulateScenario { + scenario: Some(proto::simulate_scenario::Scenario::SpeakerUpdate(3)), + })) + .await; + } + SimulateScenario::NodeFailure => { + self.signal_client + .send(signal_request::Message::Simulate(proto::SimulateScenario { + scenario: Some(proto::simulate_scenario::Scenario::NodeFailure(true)), + })) + .await; + } + SimulateScenario::ServerLeave => { + self.signal_client + .send(signal_request::Message::Simulate(proto::SimulateScenario { + scenario: Some(proto::simulate_scenario::Scenario::ServerLeave(true)), + })) + .await; + } + SimulateScenario::Migration => { + self.signal_client + .send(signal_request::Message::Simulate(proto::SimulateScenario { + scenario: Some(proto::simulate_scenario::Scenario::Migration(true)), + })) + .await; + } + SimulateScenario::ForceTcp => { + self.signal_client + .send(signal_request::Message::Simulate(proto::SimulateScenario { + scenario: Some( + proto::simulate_scenario::Scenario::SwitchCandidateProtocol( + CandidateProtocol::Tcp as i32, + ), + ), + })) + .await; + } + SimulateScenario::ForceTls => { + self.signal_client + .send(signal_request::Message::Simulate(proto::SimulateScenario { + scenario: Some( + proto::simulate_scenario::Scenario::SwitchCandidateProtocol( + CandidateProtocol::Tls as i32, + ), + ), + })) + .await; + } + } + } + + #[tracing::instrument(skip(data))] + async fn publish_data( + &self, + data: &DataPacket, + kind: data_packet::Kind, + ) -> Result<(), EngineError> { + self.ensure_publisher_connected(kind).await?; + self.data_channel(kind) + .send(&data.encode_to_vec(), true) + .map_err(Into::into) + } + + /// Try to restart the session by doing an ICE Restart (The SignalClient is also restarted) + /// This reconnection if more seemless than the full reconnection implemented in ['RTCEngine'] + async fn restart_session(&self) -> EngineResult<()> { + self.signal_client.close().await; + + let mut options = self.info.options.clone(); + options.sid = self.info.join_response.participant.clone().unwrap().sid; + options.reconnect = true; + + self.signal_client + .connect(&self.info.url, &self.info.token, options) + .await?; + + self.subscriber_pc.lock().await.prepare_ice_restart(); + + if self.has_published.load(Ordering::Acquire) { + self.publisher_pc + .lock() + .await + .create_and_send_offer(RTCOfferAnswerOptions { + ice_restart: true, + ..Default::default() + }) + .await?; + } + + self.wait_pc_connection().await?; + self.signal_client.flush_queue().await; + + Ok(()) + } + + // Wait for PCState to become PCState::Connected + // Timeout after ['MAX_ICE_CONNECT_TIMEOUT'] + async fn wait_pc_connection(&self) -> EngineResult<()> { + let wait_connected = async move { + while self.pc_state.load(Ordering::Acquire) != PCState::Connected as u8 { + tokio::task::yield_now().await; + } + }; + + tokio::select! { + _ = wait_connected => Ok(()), + _ = sleep(MAX_ICE_CONNECT_TIMEOUT) => { + let err = EngineError::Connection("wait_pc_connection timed out".to_string()); + Err(err) + } + } + } + + /// Start publisher negotiation + async fn negotiate_publisher(&self) -> EngineResult<()> { + self.has_published.store(true, Ordering::Release); + let res = self.publisher_pc.lock().await.negotiate().await; + if let Err(err) = &res { + error!("failed to negotiate the publisher: {:?}", err); + } + res.map_err(Into::into) + } + + /// Ensure the Publisher PC is connected, if not, start the negotiation + /// This is required when sending data to the server + async fn ensure_publisher_connected(&self, kind: data_packet::Kind) -> EngineResult<()> { + if !self.info.join_response.subscriber_primary { + return Ok(()); + } + + if !self.publisher_pc.lock().await.is_connected() + && self + .publisher_pc + .lock() + .await + .peer_connection() + .ice_connection_state() + != IceConnectionState::IceConnectionChecking + { + let _ = self.negotiate_publisher().await; + } + + let dc = self.data_channel(kind); + if dc.state() == DataState::Open { + return Ok(()); + } + + // Wait until the PeerConnection is connected + let wait_connected = async { + while self.publisher_pc.lock().await.is_connected() && dc.state() == DataState::Open { + tokio::task::yield_now().await; + } + }; + + // TODO(theomonnom) Avoid 15 seconds deadlock on the RTCEngine by recv close here + tokio::select! { + _ = wait_connected => Ok(()), + _ = sleep(MAX_ICE_CONNECT_TIMEOUT) => { + let err = EngineError::Connection("could not establish publisher connection: timeout".to_string()); + error!(error = ?err); + Err(err) + } + } + } + + fn data_channel(&self, kind: data_packet::Kind) -> &DataChannel { + if kind == data_packet::Kind::Reliable { + &self.reliable_dc + } else { + &self.lossy_dc + } + } +} diff --git a/crates/livekit-core/src/signal_client/mod.rs b/crates/livekit-core/src/signal_client/mod.rs index 2958de4..ad58a88 100644 --- a/crates/livekit-core/src/signal_client/mod.rs +++ b/crates/livekit-core/src/signal_client/mod.rs @@ -1,15 +1,18 @@ use std::fmt::Debug; + use std::time::Duration; use livekit_webrtc::peer_connection_factory::{ ContinualGatheringPolicy, ICEServer, IceTransportsType, RTCConfiguration, }; +use parking_lot::RwLock; use thiserror::Error; use tokio::sync::mpsc; use tokio_tungstenite::tungstenite::Error as WsError; use crate::proto::{signal_request, signal_response, JoinResponse}; use crate::signal_client::signal_stream::SignalStream; +use tracing::{instrument, Level}; mod signal_stream; @@ -21,7 +24,7 @@ pub const JOIN_RESPONSE_TIMEOUT: Duration = Duration::from_secs(5); #[derive(Error, Debug)] pub enum SignalError { - #[error("websocket failure")] + #[error("ws failure: {0}")] WsError(#[from] WsError), #[error("failed to parse the url")] UrlParse(#[from] url::ParseError), @@ -33,18 +36,18 @@ pub enum SignalError { /// Events used by the RTCEngine who will handle the reconnection logic #[derive(Debug)] -pub(crate) enum SignalEvent { +pub enum SignalEvent { Open, Signal(signal_response::Message), Close, } -#[derive(Debug)] -pub(crate) struct SignalOptions { - reconnect: bool, - auto_subscribe: bool, - sid: String, - adaptive_stream: bool, +#[derive(Debug, Clone)] +pub struct SignalOptions { + pub(crate) reconnect: bool, + pub(crate) sid: String, + pub auto_subscribe: bool, + pub adaptive_stream: bool, } impl Default for SignalOptions { @@ -60,32 +63,59 @@ impl Default for SignalOptions { #[derive(Debug)] pub struct SignalClient { - stream: SignalStream, + stream: RwLock>, emitter: SignalEmitter, } impl SignalClient { - pub(crate) async fn connect( + pub fn new() -> (Self, SignalEvents) { + let (emitter, events) = mpsc::channel(8); + ( + Self { + stream: Default::default(), + emitter, + }, + events, + ) + } + + #[instrument(level = Level::DEBUG, skip(url, token, options))] + pub async fn connect( + &self, url: &str, token: &str, options: SignalOptions, - ) -> SignalResult<(Self, SignalEvents)> { - let (emitter, events) = mpsc::channel(8); - let stream = SignalStream::connect(url, token, options, emitter.clone()).await?; - - // TODO(theomonnom) Retry initial connection - - Ok((Self { stream, emitter }, events)) + ) -> SignalResult<()> { + let stream = SignalStream::connect(url, token, options, self.emitter.clone()).await?; + *self.stream.write() = Some(stream); + Ok(()) } - pub async fn send(&self, signal: signal_request::Message) { - if let Err(_) = self.stream.send(signal).await { - // TODO(theomonnom) Queue message ( Ignore on full reconnect ) + #[instrument(level = Level::DEBUG)] + pub async fn close(&self) { + if let Some(stream) = self.stream.write().take() { + stream.close().await; } } - pub async fn reconnect(&self) { - // TODO(theomonnom) Close & recreate SignalStream, also send the queue if needed + #[instrument(level = Level::DEBUG)] + pub async fn send(&self, signal: signal_request::Message) { + if let Some(stream) = self.stream.read().as_ref() { + if stream.send(signal).await.is_ok() { + return; + } + } + + // TODO(theomonnom): enqueue message + } + + pub async fn clear_queue(&self) { + // TODO(theomonnom): impl + } + + #[instrument(level = Level::DEBUG)] + pub async fn flush_queue(&self) { + // TODO(theomonnom): impl } } @@ -112,13 +142,15 @@ impl From for RTCConfiguration { pub mod utils { use crate::proto::{signal_response, JoinResponse}; use crate::signal_client::{SignalError, SignalEvent, SignalResult, JOIN_RESPONSE_TIMEOUT}; - use tokio::sync::mpsc; use tokio::time::timeout; use tokio_tungstenite::tungstenite::Error as WsError; - use tracing::{event, Level}; + use tracing::{event, instrument, Level}; + use super::SignalEvents; + + #[instrument(level = Level::DEBUG, skip(receiver))] pub(crate) async fn next_join_response( - receiver: &mut mpsc::Receiver, + receiver: &mut SignalEvents, ) -> SignalResult { let join = async { while let Some(event) = receiver.recv().await { diff --git a/crates/livekit-core/src/signal_client/signal_stream.rs b/crates/livekit-core/src/signal_client/signal_stream.rs index eb0680d..1692397 100644 --- a/crates/livekit-core/src/signal_client/signal_stream.rs +++ b/crates/livekit-core/src/signal_client/signal_stream.rs @@ -4,6 +4,7 @@ use prost::Message as ProstMessage; use tokio::net::TcpStream; use tokio::sync::{mpsc, oneshot}; use tokio::task::JoinHandle; + use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode; use tokio_tungstenite::tungstenite::protocol::CloseFrame; use tokio_tungstenite::tungstenite::Message; @@ -60,6 +61,7 @@ impl SignalStream { .append_pair("access_token", token) .append_pair("protocol", PROTOCOL_VERSION.to_string().as_str()) .append_pair("reconnect", if options.reconnect { "1" } else { "0" }) + .append_pair("sid", &options.sid) .append_pair( "auto_subscribe", if options.auto_subscribe { "1" } else { "0" }, @@ -69,17 +71,15 @@ impl SignalStream { if options.adaptive_stream { "1" } else { "0" }, ); - event!(Level::DEBUG, "connecting to websocket: {}", lk_url); + event!(Level::INFO, "connecting to SignalClient: {}", lk_url); let (ws_stream, _) = connect_async(lk_url).await?; - event!(Level::DEBUG, "connected to websocket"); let _ = emitter.send(SignalEvent::Open).await; let (ws_writer, ws_reader) = ws_stream.split(); let (internal_tx, internal_rx) = mpsc::channel::(8); - let write_handle = - tokio::spawn(Self::handle_write(internal_rx, ws_writer, emitter.clone())); - let read_handle = tokio::spawn(Self::handle_read(internal_tx.clone(), ws_reader, emitter)); + let write_handle = tokio::spawn(Self::write_task(internal_rx, ws_writer, emitter.clone())); + let read_handle = tokio::spawn(Self::read_task(internal_tx.clone(), ws_reader, emitter)); Ok(Self { internal_tx, @@ -119,7 +119,7 @@ impl SignalStream { /// This task is used to send messages to the websocket /// It is also responsible for closing the connection - async fn handle_write( + async fn write_task( mut internal_rx: mpsc::Receiver, mut ws_writer: SplitSink, emitter: SignalEmitter, @@ -170,7 +170,7 @@ impl SignalStream { /// and dispatch them through the EventEmitter. /// /// It can also send messages to [handle_write] task ( Used e.g. answer to pings ) - async fn handle_read( + async fn read_task( internal_tx: mpsc::Sender, mut ws_reader: SplitStream, emitter: SignalEmitter, diff --git a/crates/livekit-utils/Cargo.toml b/crates/livekit-utils/Cargo.toml index 27b9ca7..b2f868c 100644 --- a/crates/livekit-utils/Cargo.toml +++ b/crates/livekit-utils/Cargo.toml @@ -6,3 +6,5 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +parking_lot = "0.12.1" +tokio = { version = "1", features = ["full"] } diff --git a/crates/livekit-utils/src/enum_dispatch.rs b/crates/livekit-utils/src/enum_dispatch.rs index c7f59df..d24eb4d 100644 --- a/crates/livekit-utils/src/enum_dispatch.rs +++ b/crates/livekit-utils/src/enum_dispatch.rs @@ -1,3 +1,6 @@ + +// TODO(theomonnom): Match the complete function signature like: +// - pub(crate) fn update_info(&self, info: ParticipantInfo) -> (); #[macro_export] macro_rules! enum_dispatch { // This arm is used to avoid nested loops with the arguments @@ -10,16 +13,15 @@ macro_rules! enum_dispatch { } }; - ($fnc:ident, $self:ty, [$($arg:ident: $t:ty),*], $ret:ty, [$($variant:ident),+]) => { - fn $fnc(self: $self, $($arg: $t),*) -> $ret { + ($vis:vis$(,)? $fnc:ident, $self:ty, [$($arg:ident: $t:ty),*], $ret:ty, [$($variant:ident),+]) => { + $vis fn $fnc(self: $self, $($arg: $t),*) -> $ret { enum_dispatch!(@match self $fnc ($($arg,)*) [$($variant),+]) } }; - ($variants:tt $(fnc!($fnc:ident, $self:ty, $args:tt, $ret:ty);)+) => { + ($variants:tt $(fnc!($vis:vis$(,)? $fnc:ident, $self:ty, $args:tt, $ret:ty);)+) => { $( - enum_dispatch!($fnc, $self, $args, $ret, $variants); + enum_dispatch!($vis, $fnc, $self, $args, $ret, $variants); )* }; } - diff --git a/crates/livekit-utils/src/lib.rs b/crates/livekit-utils/src/lib.rs index f328b57..a58478e 100644 --- a/crates/livekit-utils/src/lib.rs +++ b/crates/livekit-utils/src/lib.rs @@ -1 +1,2 @@ pub mod enum_dispatch; +pub mod observer; diff --git a/crates/livekit-utils/src/observer.rs b/crates/livekit-utils/src/observer.rs new file mode 100644 index 0000000..46cb800 --- /dev/null +++ b/crates/livekit-utils/src/observer.rs @@ -0,0 +1,39 @@ +// Really basic implementation of the observer pattern using mpsc channels. +// Currently unbounded channels + +use tokio::sync::mpsc; + +#[derive(Debug)] +pub struct Dispatcher +where + T: Clone, +{ + senders: Vec>, +} + +impl Default for Dispatcher +where + T: Clone, +{ + fn default() -> Self { + Self { + senders: Default::default(), + } + } +} + +impl Dispatcher +where + T: Clone, +{ + pub fn register(&mut self) -> mpsc::UnboundedReceiver { + let (tx, rx) = mpsc::unbounded_channel(); + self.senders.push(tx); + rx + } + + pub fn dispatch(&mut self, msg: &T) { + self.senders + .retain(|sender| sender.send(msg.clone()).is_err()); + } +} diff --git a/crates/livekit-webrtc/libwebrtc-sys/build.rs b/crates/livekit-webrtc/libwebrtc-sys/build.rs index 377958a..fb00e3d 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/build.rs +++ b/crates/livekit-webrtc/libwebrtc-sys/build.rs @@ -62,6 +62,7 @@ fn main() { path::PathBuf::from("./include"), libwebrtc_dir.clone(), libwebrtc_dir.join("third_party/abseil-cpp/"), + libwebrtc_dir.join("third_party/libyuv/include/"), libwebrtc_dir.join("third_party/libc++/"), // For mac & ios libwebrtc_dir.join("sdk/objc"), @@ -124,7 +125,7 @@ fn main() { println!("cargo:rustc-link-lib=dylib=d3d11"); println!("cargo:rustc-link-lib=dylib=dxgi"); println!("cargo:rustc-link-lib=dylib=dwmapi"); - println!("cargo:rustc-link-lib=dylib=webrtc"); + println!("cargo:rustc-link-lib=static=webrtc"); builder .flag("/std:c++17") diff --git a/crates/livekit-webrtc/libwebrtc-sys/include/livekit/data_channel.h b/crates/livekit-webrtc/libwebrtc-sys/include/livekit/data_channel.h index 38078ef..6fcc6df 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/include/livekit/data_channel.h +++ b/crates/livekit-webrtc/libwebrtc-sys/include/livekit/data_channel.h @@ -24,10 +24,10 @@ class DataChannel { void register_observer(NativeDataChannelObserver& observer); void unregister_observer(); - bool send(const DataBuffer& buffer); + bool send(const DataBuffer& buffer) const; rust::String label() const; DataState state() const; - void close(); + void close() const; private: std::shared_ptr rtc_runtime_; diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/data_channel.cpp b/crates/livekit-webrtc/libwebrtc-sys/src/data_channel.cpp index 41986f1..e5be4b7 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/src/data_channel.cpp +++ b/crates/livekit-webrtc/libwebrtc-sys/src/data_channel.cpp @@ -24,7 +24,7 @@ void DataChannel::unregister_observer() { data_channel_->UnregisterObserver(); } -bool DataChannel::send(const DataBuffer& buffer) { +bool DataChannel::send(const DataBuffer& buffer) const { return data_channel_->Send(webrtc::DataBuffer{ rtc::CopyOnWriteBuffer(buffer.ptr, buffer.len), buffer.binary}); } @@ -37,7 +37,7 @@ DataState DataChannel::state() const { return static_cast(data_channel_->state()); } -void DataChannel::close() { +void DataChannel::close() const { return data_channel_->Close(); } diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/data_channel.rs b/crates/livekit-webrtc/libwebrtc-sys/src/data_channel.rs index 4ae9983..004be48 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/src/data_channel.rs +++ b/crates/livekit-webrtc/libwebrtc-sys/src/data_channel.rs @@ -68,10 +68,10 @@ pub mod ffi { ); fn unregister_observer(self: Pin<&mut DataChannel>); - fn send(self: Pin<&mut DataChannel>, data: &DataBuffer) -> bool; + fn send(self: &DataChannel, data: &DataBuffer) -> bool; fn label(self: &DataChannel) -> String; fn state(self: &DataChannel) -> DataState; - fn close(self: Pin<&mut DataChannel>); + fn close(self: &DataChannel); fn create_data_channel_init(init: DataChannelInit) -> UniquePtr; fn create_native_data_channel_observer( @@ -83,8 +83,10 @@ pub mod ffi { } unsafe impl Send for ffi::DataChannel {} +unsafe impl Sync for ffi::DataChannel {} unsafe impl Send for ffi::NativeDataChannelObserver {} +unsafe impl Sync for ffi::NativeDataChannelObserver {} // DataChannelObserver diff --git a/crates/livekit-webrtc/libwebrtc-sys/src/webrtc.cpp b/crates/livekit-webrtc/libwebrtc-sys/src/webrtc.cpp index 0afc9d9..9dbddb3 100644 --- a/crates/livekit-webrtc/libwebrtc-sys/src/webrtc.cpp +++ b/crates/livekit-webrtc/libwebrtc-sys/src/webrtc.cpp @@ -8,7 +8,7 @@ namespace livekit { RTCRuntime::RTCRuntime() { - rtc::LogMessage::LogToDebug(rtc::LS_INFO); + // rtc::LogMessage::LogToDebug(rtc::LS_INFO); RTC_LOG(LS_INFO) << "RTCRuntime()"; RTC_CHECK(rtc::InitializeSSL()) << "Failed to InitializeSSL()"; @@ -43,4 +43,4 @@ rtc::Thread* RTCRuntime::signaling_thread() const { std::shared_ptr create_rtc_runtime() { return std::make_shared(); } -} // namespace livekit \ No newline at end of file +} // namespace livekit diff --git a/crates/livekit-webrtc/src/data_channel.rs b/crates/livekit-webrtc/src/data_channel.rs index 284340f..2a7591b 100644 --- a/crates/livekit-webrtc/src/data_channel.rs +++ b/crates/livekit-webrtc/src/data_channel.rs @@ -58,7 +58,7 @@ impl DataChannel { dc } - pub fn send(&mut self, data: &[u8], binary: bool) -> Result<(), DataSendError> { + pub fn send(&self, data: &[u8], binary: bool) -> Result<(), DataSendError> { let buffer = sys_dc::ffi::DataBuffer { ptr: data.as_ptr(), len: data.len(), @@ -66,7 +66,6 @@ impl DataChannel { }; self.cxx_handle - .pin_mut() .send(&buffer) .then_some(()) .ok_or(DataSendError {}) @@ -80,8 +79,8 @@ impl DataChannel { self.cxx_handle.state() } - pub fn close(&mut self) { - self.cxx_handle.pin_mut().close(); + pub fn close(&self) { + self.cxx_handle.close(); } pub fn on_state_change(&mut self, handler: OnStateChangeHandler) { diff --git a/crates/livekit-webrtc/src/media_stream.rs b/crates/livekit-webrtc/src/media_stream.rs index 565c92b..0ad7a7f 100644 --- a/crates/livekit-webrtc/src/media_stream.rs +++ b/crates/livekit-webrtc/src/media_stream.rs @@ -1,6 +1,7 @@ use cxx::UniquePtr; use libwebrtc_sys::media_stream as sys_ms; use libwebrtc_sys::MEDIA_TYPE_VIDEO; +use livekit_utils::enum_dispatch; use std::fmt::{Debug, Formatter}; use std::pin::Pin; use std::sync::{Arc, Mutex}; @@ -25,17 +26,6 @@ pub enum MediaStreamTrackHandle { Video(Arc), } -macro_rules! shared_getter { - ($x:ident, $ret:ty) => { - fn $x(&self) -> $ret { - match self { - Self::Video(inner) => inner.$x(), - Self::Audio(inner) => inner.$x(), - } - } - }; -} - impl MediaStreamTrackHandle { pub(crate) fn new(cxx_handle: UniquePtr) -> Self { unsafe { @@ -66,17 +56,14 @@ impl Debug for MediaStreamTrackHandle { } impl MediaStreamTrackTrait for MediaStreamTrackHandle { - shared_getter!(kind, String); - shared_getter!(id, String); - shared_getter!(enabled, bool); - shared_getter!(state, TrackState); - - fn set_enabled(&self, enabled: bool) -> bool { - match self { - Self::Video(inner) => inner.set_enabled(enabled), - Self::Audio(inner) => inner.set_enabled(enabled), - } - } + enum_dispatch!( + [Audio, Video] + fnc!(kind, &Self, [], String); + fnc!(id, &Self, [], String); + fnc!(enabled, &Self, [], bool); + fnc!(state, &Self, [], TrackState); + fnc!(set_enabled, &Self, [enabled: bool], bool); + ); } pub struct AudioTrack { diff --git a/examples/.cargo/config b/examples/.cargo/config new file mode 100644 index 0000000..ac2b23f --- /dev/null +++ b/examples/.cargo/config @@ -0,0 +1,2 @@ +[target.x86_64-pc-windows-msvc] +rustflags = ["-C", "target-feature=+crt-static"] diff --git a/examples/Cargo.lock b/examples/Cargo.lock index 35e7fa4..5553bfc 100644 --- a/examples/Cargo.lock +++ b/examples/Cargo.lock @@ -48,9 +48,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "0.7.19" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" dependencies = [ "memchr", ] @@ -64,20 +64,11 @@ dependencies = [ "libc", ] -[[package]] -name = "ansi_term" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" -dependencies = [ - "winapi", -] - [[package]] name = "anyhow" -version = "1.0.65" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98161a4e3e2184da77bb14f02184cdd111e83bbbcc9979dfee3c44b9a85f5602" +checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" [[package]] name = "arboard" @@ -138,9 +129,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "base64" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "bit-set" @@ -212,18 +203,18 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" +checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" [[package]] name = "calloop" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bcf530afb40e45e14440701e5e996d7fd139e84a912a4d83a8d6a0fb3e58663" +checksum = "19457a0da465234abd76134a5c2a910c14bd3c5558463e4396ab9a37a328e465" dependencies = [ "log", - "nix 0.25.0", + "nix 0.25.1", "slotmap", "thiserror", "vec_map", @@ -231,9 +222,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.73" +version = "1.0.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" +checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4" [[package]] name = "cesu8" @@ -436,9 +427,9 @@ checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35" [[package]] name = "cxx" -version = "1.0.78" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f39818dcfc97d45b03953c1292efc4e80954e1583c4aa770bac1383e2310a4" +checksum = "bdf07d07d6531bfcdbe9b8b739b104610c6508dcc4d63b410585faf338241daf" dependencies = [ "cc", "cxxbridge-flags", @@ -448,9 +439,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.78" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e580d70777c116df50c390d1211993f62d40302881e54d4b79727acb83d0199" +checksum = "d2eb5b96ecdc99f72657332953d4d9c50135af1bac34277801cc3937906ebd39" dependencies = [ "cc", "codespan-reporting", @@ -463,15 +454,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.78" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56a46460b88d1cec95112c8c363f0e2c39afdb237f60583b0b36343bf627ea9c" +checksum = "ac040a39517fd1674e0f32177648334b0f4074625b5588a64519804ba0553b12" [[package]] name = "cxxbridge-macro" -version = "1.0.78" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "747b608fecf06b0d72d440f27acc99288207324b793be2c17991839f3d4995ea" +checksum = "1362b0ddcfc4eb0a1f57b68bd77dd99f0e826958a96abd0ae9bd092e114ffed6" dependencies = [ "proc-macro2", "quote", @@ -526,9 +517,9 @@ dependencies = [ [[package]] name = "digest" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c" +checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" dependencies = [ "block-buffer", "crypto-common", @@ -569,10 +560,18 @@ dependencies = [ "wio", ] +[[package]] +name = "ecolor" +version = "0.20.0" +source = "git+https://github.com/emilk/egui#e7471f1191081ecb514fe129bb8618938b67e796" +dependencies = [ + "bytemuck", +] + [[package]] name = "egui" -version = "0.19.0" -source = "git+https://github.com/emilk/egui#0336816faf9f361e59ca77f99c19fcfa2bf7c993" +version = "0.20.0" +source = "git+https://github.com/emilk/egui#e7471f1191081ecb514fe129bb8618938b67e796" dependencies = [ "ahash 0.8.2", "epaint", @@ -582,8 +581,8 @@ dependencies = [ [[package]] name = "egui-wgpu" -version = "0.19.0" -source = "git+https://github.com/emilk/egui#0336816faf9f361e59ca77f99c19fcfa2bf7c993" +version = "0.20.0" +source = "git+https://github.com/emilk/egui#e7471f1191081ecb514fe129bb8618938b67e796" dependencies = [ "bytemuck", "egui", @@ -596,8 +595,8 @@ dependencies = [ [[package]] name = "egui-winit" -version = "0.19.0" -source = "git+https://github.com/emilk/egui#0336816faf9f361e59ca77f99c19fcfa2bf7c993" +version = "0.20.0" +source = "git+https://github.com/emilk/egui#e7471f1191081ecb514fe129bb8618938b67e796" dependencies = [ "arboard", "egui", @@ -608,26 +607,6 @@ dependencies = [ "winit", ] -[[package]] -name = "egui_demo_lib" -version = "0.19.0" -source = "git+https://github.com/emilk/egui#0336816faf9f361e59ca77f99c19fcfa2bf7c993" -dependencies = [ - "egui", - "egui_extras", - "enum-map", - "tracing", - "unicode_names2", -] - -[[package]] -name = "egui_extras" -version = "0.19.0" -source = "git+https://github.com/emilk/egui#0336816faf9f361e59ca77f99c19fcfa2bf7c993" -dependencies = [ - "egui", -] - [[package]] name = "either" version = "1.8.0" @@ -636,42 +615,22 @@ checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" [[package]] name = "emath" -version = "0.19.0" -source = "git+https://github.com/emilk/egui#0336816faf9f361e59ca77f99c19fcfa2bf7c993" +version = "0.20.0" +source = "git+https://github.com/emilk/egui#e7471f1191081ecb514fe129bb8618938b67e796" dependencies = [ "bytemuck", ] -[[package]] -name = "enum-map" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5a56d54c8dd9b3ad34752ed197a4eb2a6601bc010808eb097a04a58ae4c43e1" -dependencies = [ - "enum-map-derive", - "serde", -] - -[[package]] -name = "enum-map-derive" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9045e2676cd5af83c3b167d917b0a5c90a4d8e266e2683d6631b235c457fc27" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "epaint" -version = "0.19.0" -source = "git+https://github.com/emilk/egui#0336816faf9f361e59ca77f99c19fcfa2bf7c993" +version = "0.20.0" +source = "git+https://github.com/emilk/egui#e7471f1191081ecb514fe129bb8618938b67e796" dependencies = [ "ab_glyph", "ahash 0.8.2", "atomic_refcell", "bytemuck", + "ecolor", "emath", "nohash-hasher", "parking_lot", @@ -803,9 +762,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f21eda599937fba36daeb58a22e8f5cee2d14c4a17b5b7739c7c8e5e3b8230c" +checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0" dependencies = [ "futures-channel", "futures-core", @@ -818,9 +777,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30bdd20c28fadd505d0fd6712cdfcb0d4b5648baf45faef7f852afb2399bb050" +checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" dependencies = [ "futures-core", "futures-sink", @@ -828,15 +787,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e5aa3de05362c3fb88de6531e6296e85cde7739cccad4b9dfeeb7f6ebce56bf" +checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" [[package]] name = "futures-executor" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ff63c23854bee61b6e9cd331d523909f238fc7636290b96826e9cfa5faa00ab" +checksum = "7acc85df6714c176ab5edf386123fafe217be88c0840ec11f199441134a074e2" dependencies = [ "futures-core", "futures-task", @@ -845,15 +804,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbf4d2a7a308fd4578637c0b17c7e1c7ba127b8f6ba00b29f717e9655d85eb68" +checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" [[package]] name = "futures-macro" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42cd15d1c7456c04dbdf7e88bcd69760d74f3a798d6444e16974b505b0e62f17" +checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d" dependencies = [ "proc-macro2", "quote", @@ -862,21 +821,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b20ba5a92e727ba30e72834706623d94ac93a725410b6a6b6fbc1b07f7ba56" +checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" [[package]] name = "futures-task" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6508c467c73851293f390476d4491cf4d227dbabcd4170f3bb6044959b294f1" +checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" [[package]] name = "futures-util" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44fb6cb1be61cc1d2e43b262516aafcf63b241cffdb1d3fa115f91d9c7b09c90" +checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" dependencies = [ "futures-channel", "futures-core", @@ -921,9 +880,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" +checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ "cfg-if", "libc", @@ -1052,9 +1011,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" +checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" dependencies = [ "autocfg", "hashbrown", @@ -1083,9 +1042,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754" +checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" [[package]] name = "jni" @@ -1135,9 +1094,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.134" +version = "0.2.138" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "329c933548736bc49fd575ee68c89e8be4d260064184389a5b77517cddd99ffb" +checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8" [[package]] name = "libloading" @@ -1298,14 +1257,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57ee1c23c7c63b0c9250c339ffdc69255f110b298b901b9f6c82547b7b87caaf" +checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" dependencies = [ "libc", "log", "wasi", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -1336,9 +1295,9 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" dependencies = [ "lazy_static", "libc", @@ -1412,9 +1371,9 @@ dependencies = [ [[package]] name = "nix" -version = "0.24.2" +version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "195cdbc1741b8134346d515b3a56a1c94b0912758009cfd53f99ea0f57b065fc" +checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" dependencies = [ "bitflags", "cfg-if", @@ -1424,9 +1383,9 @@ dependencies = [ [[package]] name = "nix" -version = "0.25.0" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e322c04a9e3440c327fca7b6c8a63e6890a32fa2ad689db972425f07e0d22abb" +checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4" dependencies = [ "autocfg", "bitflags", @@ -1451,6 +1410,16 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num-traits" version = "0.2.15" @@ -1462,9 +1431,9 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.13.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" dependencies = [ "hermit-abi", "libc", @@ -1532,15 +1501,15 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1" +checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" [[package]] name = "openssl" -version = "0.10.42" +version = "0.10.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12fc0523e3bd51a692c8850d075d74dc062ccf251c0110668cbd921917118a13" +checksum = "29d971fd5722fec23977260f6e81aa67d2f22cadbdc2aa049f1022d9a3be1566" dependencies = [ "bitflags", "cfg-if", @@ -1570,9 +1539,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.76" +version = "0.9.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5230151e44c0f05157effb743e8d517472843121cf9243e8b81393edb5acd9ce" +checksum = "5454462c0eced1e97f2ec09036abc8da362e66802f66fd20f86854d9d8cbcbc4" dependencies = [ "autocfg", "cc", @@ -1581,6 +1550,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "owned_ttf_parser" version = "0.17.1" @@ -1602,15 +1577,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.3" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" +checksum = "7ff9f3fef3968a3ec5945535ed654cb38ff72d7495a25619e2247fb15a2ed9ba" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -1643,9 +1618,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" +checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "png" @@ -1667,9 +1642,19 @@ checksum = "5da3b0203fd7ee5720aa0b5e790b591aa5d3f41c3ed2c34a3a393382198af2f7" [[package]] name = "ppv-lite86" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "prettyplease" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c142c0e46b57171fe0c528bee8c5b7569e80f0c17e377cd0e30ea57dbc11bb51" +dependencies = [ + "proc-macro2", + "syn", +] [[package]] name = "proc-macro-crate" @@ -1684,9 +1669,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94e2ef8dbfc347b10c094890f778ee2e36ca9bb4262e86dc99cd217e35f3470b" +checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" dependencies = [ "unicode-ident", ] @@ -1699,9 +1684,9 @@ checksum = "74605f360ce573babfe43964cbe520294dcb081afbf8c108fc6e23036b4da2df" [[package]] name = "prost" -version = "0.11.0" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "399c3c31cdec40583bb68f0b18403400d01ec4289c383aa047560439952c4dd7" +checksum = "c0b18e655c21ff5ac2084a5ad0611e827b3f92badf79f4910b5a5c58f4d87ff0" dependencies = [ "bytes", "prost-derive", @@ -1709,9 +1694,9 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.11.1" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f835c582e6bd972ba8347313300219fed5bfa52caf175298d860b61ff6069bb" +checksum = "276470f7f281b0ed53d2ae42dd52b4a8d08853a3c70e7fe95882acbb98a6ae94" dependencies = [ "bytes", "heck", @@ -1720,18 +1705,20 @@ dependencies = [ "log", "multimap", "petgraph", + "prettyplease", "prost", "prost-types", "regex", + "syn", "tempfile", "which", ] [[package]] name = "prost-derive" -version = "0.11.0" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7345d5f0e08c0536d7ac7229952590239e77abf0a0100a1b1d890add6ea96364" +checksum = "164ae68b6587001ca506d3bf7f1000bfa248d0e1217b618108fba4ec1d0cc306" dependencies = [ "anyhow", "itertools", @@ -1742,9 +1729,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dfaa718ad76a44b3415e6c4d53b17c8f99160dcb3a99b10470fce8ad43f6e3e" +checksum = "747761bc3dc48f9a34553bf65605cf6cb6288ba219f3450b4275dbd81539551a" dependencies = [ "bytes", "prost", @@ -1824,9 +1811,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" +checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" dependencies = [ "aho-corasick", "memchr", @@ -1835,9 +1822,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.27" +version = "0.6.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" +checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" [[package]] name = "remove_dir_all" @@ -1891,7 +1878,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" dependencies = [ "lazy_static", - "windows-sys", + "windows-sys 0.36.1", ] [[package]] @@ -1949,18 +1936,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.145" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728eb6351430bccb993660dfffc5a72f91ccc1295abaa8ce19b27ebe4f75568b" +checksum = "256b9932320c590e707b94576e3cc1f7c9024d0ee6612dfbcf1cb106cbe8e055" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.145" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fa1584d3d1bcacd84c277a0dfe21f5b0f6accf4a23d04d4c6d61f1af522b4c" +checksum = "b4eae9b04cbffdfd550eb462ed33bc6a1b68c935127d008b27444d08380f94e4" dependencies = [ "proc-macro2", "quote", @@ -1969,9 +1956,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.85" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e55a28e3aaef9d5ce0506d0a14dbba8054ddc7e499ef522dd8b26859ec9d4a44" +checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db" dependencies = [ "itoa", "ryu", @@ -2001,9 +1988,9 @@ dependencies = [ [[package]] name = "sha-1" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" +checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" dependencies = [ "cfg-if", "cpufeatures", @@ -2035,9 +2022,9 @@ dependencies = [ "egui", "egui-wgpu", "egui-winit", - "egui_demo_lib", "futures", "livekit", + "parking_lot", "tokio", "tracing", "tracing-subscriber", @@ -2065,9 +2052,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "smithay-client-toolkit" @@ -2081,7 +2068,7 @@ dependencies = [ "lazy_static", "log", "memmap2", - "nix 0.24.2", + "nix 0.24.3", "pkg-config", "wayland-client", "wayland-cursor", @@ -2138,9 +2125,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.101" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e90cde112c4b9690b8cbe810cba9ddd8bc1d7472e2cae317b69e9438c1cba7d2" +checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908" dependencies = [ "proc-macro2", "quote", @@ -2241,9 +2228,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.21.2" +version = "1.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099" +checksum = "eab6d665857cc6ca78d6e80303a02cea7a7851e85dfbd77cbdc09bd129f1ef46" dependencies = [ "autocfg", "bytes", @@ -2256,14 +2243,14 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "winapi", + "windows-sys 0.42.0", ] [[package]] name = "tokio-macros" -version = "1.8.0" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" +checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" dependencies = [ "proc-macro2", "quote", @@ -2305,9 +2292,9 @@ dependencies = [ [[package]] name = "tracing" -version = "0.1.36" +version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fce9567bd60a67d08a16488756721ba392f24f29006402881e43b19aac64307" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if", "pin-project-lite", @@ -2317,9 +2304,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ "proc-macro2", "quote", @@ -2328,9 +2315,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.29" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeea4303076558a00714b823f9ad67d58a3bbda1df83d8827d21193156e22f7" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" dependencies = [ "once_cell", "valuable", @@ -2349,11 +2336,11 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60db860322da191b40952ad9affe65ea23e7dd6a5c442c2c42865810c6ab8e6b" +checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" dependencies = [ - "ansi_term", + "nu-ansi-term", "sharded-slab", "smallvec", "thread_local", @@ -2398,9 +2385,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "unicode-bidi" @@ -2410,9 +2397,9 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" [[package]] name = "unicode-ident" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd" +checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" [[package]] name = "unicode-normalization" @@ -2435,12 +2422,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" -[[package]] -name = "unicode_names2" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "029df4cc8238cefc911704ff8fa210853a0f3bce2694d8f51181dd41ee0f3301" - [[package]] name = "url" version = "2.3.1" @@ -2574,7 +2555,7 @@ dependencies = [ "bitflags", "downcast-rs", "libc", - "nix 0.24.2", + "nix 0.24.3", "scoped-tls", "wayland-commons", "wayland-scanner", @@ -2587,7 +2568,7 @@ version = "0.29.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8691f134d584a33a6606d9d717b95c4fa20065605f798a3f350d78dced02a902" dependencies = [ - "nix 0.24.2", + "nix 0.24.3", "once_cell", "smallvec", "wayland-sys", @@ -2599,7 +2580,7 @@ version = "0.29.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6865c6b66f13d6257bef1cd40cbfe8ef2f150fb8ebbdb1e8e873455931377661" dependencies = [ - "nix 0.24.2", + "nix 0.24.3", "wayland-client", "xcursor", ] @@ -2666,9 +2647,9 @@ dependencies = [ [[package]] name = "wgpu" -version = "0.14.0" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2272b17bffc8a0c7d53897435da7c1db587c87d3a14e8dae9cdb8d1d210fc0f" +checksum = "81f643110d228fd62a60c5ed2ab56c4d5b3704520bd50561174ec4ec74932937" dependencies = [ "arrayvec 0.7.2", "js-sys", @@ -2688,9 +2669,9 @@ dependencies = [ [[package]] name = "wgpu-core" -version = "0.14.0" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73d14cad393054caf992ee02b7da6a372245d39a484f7461c1f44f6f6359bd28" +checksum = "6000d1284ef8eec6076fd5544a73125fd7eb9b635f18dceeb829d826f41724ca" dependencies = [ "arrayvec 0.7.2", "bit-vec", @@ -2821,43 +2802,100 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" dependencies = [ - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_msvc", + "windows_aarch64_msvc 0.36.1", + "windows_i686_gnu 0.36.1", + "windows_i686_msvc 0.36.1", + "windows_x86_64_gnu 0.36.1", + "windows_x86_64_msvc 0.36.1", ] +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc 0.42.0", + "windows_i686_gnu 0.42.0", + "windows_i686_msvc 0.42.0", + "windows_x86_64_gnu 0.42.0", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc 0.42.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" + [[package]] name = "windows_aarch64_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" + [[package]] name = "windows_i686_gnu" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" +[[package]] +name = "windows_i686_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" + [[package]] name = "windows_i686_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" +[[package]] +name = "windows_i686_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" + [[package]] name = "windows_x86_64_gnu" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" + [[package]] name = "windows_x86_64_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" + [[package]] name = "winit" version = "0.27.5" @@ -2887,7 +2925,7 @@ dependencies = [ "wayland-client", "wayland-protocols", "web-sys", - "windows-sys", + "windows-sys 0.36.1", "x11-dl", ] @@ -2918,7 +2956,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "592b4883219f345e712b3209c62654ebda0bb50887f330cbd018d0f654bfd507" dependencies = [ "gethostname", - "nix 0.24.2", + "nix 0.24.3", "winapi", "winapi-wsapoll", "x11rb-protocol", @@ -2930,7 +2968,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56b245751c0ac9db0e006dc812031482784e434630205a93c73cfefcaabeac67" dependencies = [ - "nix 0.24.2", + "nix 0.24.3", ] [[package]] diff --git a/examples/Cargo.toml b/examples/Cargo.toml index 6696429..20abc68 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -1,4 +1,4 @@ [workspace] members = ["*"] -exclude = ["target"] +exclude = ["target", ".cargo"] resolver = "2" diff --git a/examples/simple_room/Cargo.toml b/examples/simple_room/Cargo.toml index d9e266a..6d59c39 100644 --- a/examples/simple_room/Cargo.toml +++ b/examples/simple_room/Cargo.toml @@ -11,8 +11,8 @@ livekit = { path = "../.." } futures = "0.3" wgpu = "0.14.0" winit = "0.27.5" +parking_lot = "0.12.1" egui = { git = "https://github.com/emilk/egui" } egui-wgpu = { git = "https://github.com/emilk/egui", features = ["winit"] } egui-winit = { git = "https://github.com/emilk/egui" } -egui_demo_lib = { git = "https://github.com/emilk/egui" } diff --git a/examples/simple_room/src/app.rs b/examples/simple_room/src/app.rs new file mode 100644 index 0000000..f17cba7 --- /dev/null +++ b/examples/simple_room/src/app.rs @@ -0,0 +1,361 @@ +use crate::events::UiCmd; +use crate::video_renderer::VideoRenderer; +use crate::{events::AsyncCmd, video_grid::VideoGrid}; +use egui_wgpu::WgpuConfiguration; +use livekit::room::track::remote_track::RemoteTrackHandle; +use parking_lot::Mutex; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; +use tokio::sync::mpsc; + +use livekit::room::{ConnectionState, Room, RoomError, SimulateScenario}; + +// Useful default constants for developing +const DEFAULT_URL: &str = "ws://localhost:7880"; +const DEFAULT_TOKEN : &str = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjIzODQ4MDY0NzMsImlzcyI6IkFQSXpLYkFTaUNWYWtnSiIsIm5hbWUiOiJuYXRpdmUiLCJuYmYiOjE2NjQ4MDY0NzMsInN1YiI6Im5hdGl2ZSIsInZpZGVvIjp7InJvb21DcmVhdGUiOnRydWUsInJvb21Kb2luIjp0cnVlfX0.BgVdBnq3XFD3_BQHoe1azqjifYysubgFl6Qlzu9IQGI"; + +// eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjIzODQ4MDY3MzAsImlzcyI6IkFQSXpLYkFTaUNWYWtnSiIsIm5hbWUiOiJ3ZWIiLCJuYmYiOjE2NjQ4MDY3MzAsInN1YiI6IndlYiIsInZpZGVvIjp7InJvb21DcmVhdGUiOnRydWUsInJvb21Kb2luIjp0cnVlfX0.VbDoULjX1CVGZu2sPy3SvWYlVZUBXxQVPmdB9BnmlN4 + +use winit::{ + event::*, + event_loop::{ControlFlow, EventLoop}, + window::{WindowBuilder, WindowId}, +}; + +struct AppState { + room: Mutex, + connecting: AtomicBool, +} + +struct App { + state: Arc, + + video_renderers: Vec, + egui_context: egui::Context, + egui_state: egui_winit::State, + egui_painter: egui_wgpu::winit::Painter, + window: winit::window::Window, + cmd_tx: mpsc::UnboundedSender, + cmd_rx: mpsc::UnboundedReceiver, + + // UI State + lk_url: String, + lk_token: String, + connection_failure: Option, + room_state: ConnectionState, +} + +pub fn run(rt: tokio::runtime::Runtime) { + rt.block_on(async { + let event_loop = EventLoop::new(); + let window = WindowBuilder::new() + .with_title("LiveKit - NativeSDK") + .build(&event_loop) + .unwrap(); + + let egui_context = egui::Context::default(); + let egui_state = egui_winit::State::new(&event_loop); + let mut egui_painter = egui_wgpu::winit::Painter::new(WgpuConfiguration::default(), 1, 32); + + unsafe { + egui_painter.set_window(Some(&window)); + } + + let (async_cmd_tx, mut async_cmd_rx) = mpsc::unbounded_channel::(); + let (ui_cmd_tx, ui_cmd_rx) = mpsc::unbounded_channel::(); + + let state = Arc::new(AppState { + room: Mutex::new(Room::new()), + connecting: AtomicBool::new(false), + }); + + let mut app = App { + state: state.clone(), + video_renderers: Vec::default(), + egui_context, + egui_state, + egui_painter, + window, + cmd_tx: async_cmd_tx, + cmd_rx: ui_cmd_rx, + lk_url: DEFAULT_URL.to_owned(), + lk_token: DEFAULT_TOKEN.to_owned(), + connection_failure: None, + room_state: ConnectionState::Connected, + }; + + // Async event loop + tokio::spawn(async move { + { + let events = state.room.lock().events(); + events.on_track_subscribed({ + let ui_cmd_tx = ui_cmd_tx.clone(); + move |event| { + let ui_cmd_tx = ui_cmd_tx.clone(); + async move { + ui_cmd_tx.send(UiCmd::TrackSubscribed { event }).unwrap(); + } + } + }); + } + while let Some(event) = async_cmd_rx.recv().await { + match event { + AsyncCmd::RoomConnect { url, token } => { + state.connecting.store(true, Ordering::SeqCst); + + let mut room = state.room.lock(); + ui_cmd_tx + .send(UiCmd::ConnectResult { + result: room.connect(&url, &token).await, + }) + .unwrap(); + + state.connecting.store(false, Ordering::SeqCst); + } + AsyncCmd::SimulateScenario { scenario } => { + if let Some(handle) = state.room.lock().get_handle() { + let _ = handle.simulate_scenario(scenario).await; + } + } + } + } + }); + + tokio::task::block_in_place(move || loop { + // UI/Main Thread + event_loop.run(move |event, _, control_flow| { + app.update(event, control_flow); + }); + }); + }); +} + +impl App { + fn update(&mut self, event: Event<'_, T>, control_flow: &mut ControlFlow) { + if let Ok(cmd) = self.cmd_rx.try_recv() { + match cmd { + UiCmd::ConnectResult { result } => { + if let Err(err) = result { + self.connection_failure = Some(err.to_string()); + } else { + self.connection_failure = None + } + } + UiCmd::TrackSubscribed { event } => { + match event.track { + RemoteTrackHandle::Video(video_track) => { + // Create a new VideoRenderer + let video_renderer = VideoRenderer::new( + self.egui_painter.render_state().clone().unwrap(), + video_track.rtc_track(), + ); + self.video_renderers.push(video_renderer); + } + RemoteTrackHandle::Audio(_) => { + // The demo doesn't support Audio rendering at the moment. + } + }; + } + } + } + + match event { + Event::WindowEvent { window_id, event } => { + if let Some(flow) = self.on_window_event(window_id, event) { + *control_flow = flow; + } + } + Event::RedrawRequested(window_id) if window_id == self.window.id() => { + self.render(); + } + Event::RedrawEventsCleared => { + self.window.request_redraw(); + } + _ => {} + }; + } + + fn on_window_event( + &mut self, + _window_id: WindowId, + event: WindowEvent<'_>, + ) -> Option { + if self + .egui_state + .on_event(&self.egui_context, &event) + .consumed + { + return None; + } + + match event { + WindowEvent::CloseRequested => Some(ControlFlow::Exit), + WindowEvent::Resized(inner_size) => { + self.egui_painter + .on_window_resized(inner_size.width, inner_size.height); + None + } + WindowEvent::ScaleFactorChanged { new_inner_size, .. } => { + self.egui_painter + .on_window_resized(new_inner_size.width, new_inner_size.height); + None + } + _ => None, + } + } + + fn ui(&mut self, ui: &mut egui::Ui) { + egui::TopBottomPanel::top("top_panel").show(ui.ctx(), |ui| { + egui::menu::bar(ui, |ui| { + ui.menu_button("Tools", |ui| { + if ui.button("Logs").clicked() {} + if ui.button("Profiler").clicked() {} + if ui.button("WebRTC Stats").clicked() {} + if ui.button("Events").clicked() {} + }); + ui.menu_button("Simulate", |ui| { + if ui.button("SignalReconnect").clicked() { + let _ = self.cmd_tx.send(AsyncCmd::SimulateScenario { + scenario: SimulateScenario::SignalReconnect, + }); + } + if ui.button("Speaker").clicked() { + let _ = self.cmd_tx.send(AsyncCmd::SimulateScenario { + scenario: SimulateScenario::Speaker, + }); + } + if ui.button("NodeFailure").clicked() { + let _ = self.cmd_tx.send(AsyncCmd::SimulateScenario { + scenario: SimulateScenario::NodeFailure, + }); + } + if ui.button("ServerLeave").clicked() { + let _ = self.cmd_tx.send(AsyncCmd::SimulateScenario { + scenario: SimulateScenario::ServerLeave, + }); + } + if ui.button("Migration").clicked() { + let _ = self.cmd_tx.send(AsyncCmd::SimulateScenario { + scenario: SimulateScenario::Migration, + }); + } + if ui.button("ForceTcp").clicked() { + let _ = self.cmd_tx.send(AsyncCmd::SimulateScenario { + scenario: SimulateScenario::ForceTcp, + }); + } + if ui.button("ForceTls").clicked() { + let _ = self.cmd_tx.send(AsyncCmd::SimulateScenario { + scenario: SimulateScenario::ForceTls, + }); + } + }); + }); + }); + + egui::SidePanel::right("room_panel") + .default_width(256.0) + .show(ui.ctx(), |ui| { + ui.heading("Livekit - Connect to a room"); + ui.separator(); + + ui.horizontal(|ui| { + ui.label("URL: "); + ui.text_edit_singleline(&mut self.lk_url); + }); + + ui.horizontal(|ui| { + ui.label("Token: "); + ui.text_edit_singleline(&mut self.lk_token); + }); + + ui.horizontal(|ui| { + let connecting = self.state.connecting.load(Ordering::SeqCst); + ui.set_enabled(!connecting); + + if ui.button("Connect").clicked() { + self.connection_failure = None; + let _ = self.cmd_tx.send(AsyncCmd::RoomConnect { + url: self.lk_url.clone(), + token: self.lk_token.clone(), + }); + } + + if connecting { + ui.spinner(); + } + }); + + if let Some(err) = &self.connection_failure { + ui.colored_label(egui::Color32::RED, err); + } + + ui.separator(); + }); + + egui::CentralPanel::default().show(ui.ctx(), |ui| { + egui::ScrollArea::vertical().show(ui, |ui| { + VideoGrid::new("default_grid") + .max_columns(6) + .show(ui, |ui| { + if self.room_state == ConnectionState::Disconnected { + for _ in 0..20 { + ui.video_frame(|ui| { + egui::Frame::none().fill(egui::Color32::DARK_GRAY).show( + ui, + |ui| { + ui.allocate_space(ui.available_size()); + }, + ); + }); + } + } else { + for video_renderer in &self.video_renderers { + ui.video_frame(|ui| { + if let Some(tex) = video_renderer.texture_id() { + ui.painter().image( + tex, + ui.available_rect_before_wrap(), + egui::Rect::from_min_max( + egui::pos2(0.0, 0.0), + egui::pos2(1.0, 1.0), + ), + egui::Color32::WHITE, + ); + } + }); + } + } + }); + }); + }); + } + + fn render(&mut self) { + self.egui_state + .set_pixels_per_point(egui_winit::native_pixels_per_point(&self.window)); + + let raw_inputs = self.egui_state.take_egui_input(&self.window); + let full_output = self.egui_context.clone().run(raw_inputs, |ctx| { + egui::CentralPanel::default().show(ctx, |ui| { + self.ui(ui); + }); + }); + let clipped_primitives = self.egui_context.tessellate(full_output.shapes); + + self.egui_painter.paint_and_update_textures( + egui_winit::native_pixels_per_point(&self.window), + egui::Rgba::BLACK, + &clipped_primitives, + &full_output.textures_delta, + ); + + self.egui_state.handle_platform_output( + &self.window, + &self.egui_context, + full_output.platform_output, + ); + } +} diff --git a/examples/simple_room/src/events.rs b/examples/simple_room/src/events.rs new file mode 100644 index 0000000..8efe803 --- /dev/null +++ b/examples/simple_room/src/events.rs @@ -0,0 +1,17 @@ +use livekit::{events::TrackSubscribedEvent, room::SimulateScenario}; + +#[derive(Debug)] +pub enum AsyncCmd { + RoomConnect { url: String, token: String }, + SimulateScenario { scenario: SimulateScenario } +} + +#[derive(Debug)] +pub enum UiCmd { + ConnectResult { + result: livekit::room::RoomResult<()>, + }, + TrackSubscribed { + event: TrackSubscribedEvent, + }, +} diff --git a/examples/simple_room/src/main.rs b/examples/simple_room/src/main.rs index a35bbbf..351ba58 100644 --- a/examples/simple_room/src/main.rs +++ b/examples/simple_room/src/main.rs @@ -1,192 +1,15 @@ -use std::convert::TryInto; -use std::ops::DerefMut; -use std::{num::NonZeroU32, time::Duration}; - -use egui_wgpu::WgpuConfiguration; -use livekit::webrtc::media_stream::VideoTrack; -use livekit::webrtc::video_frame_buffer::{ - PlanarYuv8Buffer, PlanarYuvBuffer, VideoFrameBufferTrait, -}; -use livekit::webrtc::yuv_helper; -use std::sync::{Arc, Mutex}; -use video_renderer::VideoRenderer; -use wgpu::{Device, Queue}; - -use tokio::time::sleep; - -use livekit::room::track::remote_track::RemoteTrackHandle; -use livekit::room::{Room, RoomError}; - -const URL: &str = "ws://localhost:7880"; -const TOKEN : &str = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjIzODQ4MDY0NzMsImlzcyI6IkFQSXpLYkFTaUNWYWtnSiIsIm5hbWUiOiJuYXRpdmUiLCJuYmYiOjE2NjQ4MDY0NzMsInN1YiI6Im5hdGl2ZSIsInZpZGVvIjp7InJvb21DcmVhdGUiOnRydWUsInJvb21Kb2luIjp0cnVlfX0.BgVdBnq3XFD3_BQHoe1azqjifYysubgFl6Qlzu9IQGI"; - -// eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjIzODQ4MDY3MzAsImlzcyI6IkFQSXpLYkFTaUNWYWtnSiIsIm5hbWUiOiJ3ZWIiLCJuYmYiOjE2NjQ4MDY3MzAsInN1YiI6IndlYiIsInZpZGVvIjp7InJvb21DcmVhdGUiOnRydWUsInJvb21Kb2luIjp0cnVlfX0.VbDoULjX1CVGZu2sPy3SvWYlVZUBXxQVPmdB9BnmlN4 - +mod app; +mod events; +mod video_grid; mod video_renderer; -use winit::{ - event::*, - event_loop::{ControlFlow, EventLoop}, - window::{Window, WindowBuilder, WindowId}, -}; - -struct AppState { - room: Room, - demo: egui_demo_lib::DemoWindows, - egui_context: egui::Context, - egui_state: egui_winit::State, - egui_painter: egui_wgpu::winit::Painter, - window: winit::window::Window, -} - -impl AppState { - fn on_event(&mut self, event: Event<'_, T>, control_flow: &mut ControlFlow) { - match event { - Event::WindowEvent { window_id, event } => { - if let Some(flow) = self.on_window_event(window_id, event) { - *control_flow = flow; - } - } - Event::RedrawRequested(window_id) if window_id == self.window.id() => { - self.render(); - } - Event::RedrawEventsCleared => { - self.window.request_redraw(); - } - _ => {} - }; - } - - fn on_window_event( - &mut self, - _window_id: WindowId, - event: WindowEvent<'_>, - ) -> Option { - if self - .egui_state - .on_event(&self.egui_context, &event) - .consumed - { - return None; - } - - match event { - WindowEvent::CloseRequested => Some(ControlFlow::Exit), - WindowEvent::Resized(inner_size) => { - self.egui_painter - .on_window_resized(inner_size.width, inner_size.height); - None - } - WindowEvent::ScaleFactorChanged { new_inner_size, .. } => { - self.egui_painter - .on_window_resized(new_inner_size.width, new_inner_size.height); - None - } - _ => None, - } - } - - fn render(&mut self) { - let raw_inputs = self.egui_state.take_egui_input(&self.window); - let full_output = self.egui_context.run(raw_inputs, |ctx| { - //self.ui(ctx); - }); - let clipped_primitives = self.egui_context.tessellate(full_output.shapes); - - self.egui_painter.paint_and_update_textures( - egui_winit::native_pixels_per_point(&self.window), - egui::Rgba::BLACK, - &clipped_primitives, - &full_output.textures_delta, - ); - - self.egui_state.handle_platform_output( - &self.window, - &self.egui_context, - full_output.platform_output, - ); - } -} - -struct App { - rt: tokio::runtime::Runtime, -} - -impl App { - pub fn new(rt: tokio::runtime::Runtime) -> Self { - Self { rt } - } - - pub fn run(&mut self) { - self.rt.block_on(async { - let event_loop = EventLoop::new(); - let window = WindowBuilder::new().build(&event_loop).unwrap(); - - let egui_context = egui::Context::default(); - let egui_state = egui_winit::State::new(&event_loop); - let mut egui_painter = - egui_wgpu::winit::Painter::new(WgpuConfiguration::default(), 1, 32); - unsafe { - egui_painter.set_window(Some(&window)); - } - - let mut inner = AppState { - room: Room::new(), - demo: egui_demo_lib::DemoWindows::default(), - egui_context, - egui_state, - egui_painter, - window, - }; - - inner - .room - .events() - .on_participant_connected(|_event| async move {}); - - inner.room.events().on_track_subscribed({ - let test = Arc::new(Mutex::new(None)); - - let egui_render = inner.egui_painter.render_state().clone().unwrap(); - - move |event| { - let test = test.clone(); - let egui_render = egui_render.clone(); - - async move { - let track = event.publication.track().unwrap(); - if let RemoteTrackHandle::Video(video_track) = track { - *test.lock().unwrap() = - Some(VideoRenderer::new(egui_render, video_track.rtc_track())) - } - } - } - }); - - inner.room.connect(URL, TOKEN).await.unwrap(); - - tokio::spawn(async { - loop { - println!("Test"); - tokio::time::sleep(Duration::from_secs(5)).await; - } - }); - - tokio::task::block_in_place(move || loop { - event_loop.run(move |event, _, control_flow| { - inner.on_event(event, control_flow); - }); - }); - }); - } -} - fn main() { + tracing_subscriber::fmt::init(); + let rt = tokio::runtime::Builder::new_multi_thread() .enable_all() .build() .unwrap(); - let mut app = App::new(rt); - app.run(); + app::run(rt); } diff --git a/examples/simple_room/src/video_grid.rs b/examples/simple_room/src/video_grid.rs new file mode 100644 index 0000000..bc14d75 --- /dev/null +++ b/examples/simple_room/src/video_grid.rs @@ -0,0 +1,175 @@ +use std::cmp; + +#[derive(Debug, Clone, Default, PartialEq)] +struct State { + num_videos: u32, +} + +impl State { + pub fn load(ctx: &egui::Context, id: egui::Id) -> Option { + ctx.data().get_temp(id) + } + + pub fn store(self, ctx: &egui::Context, id: egui::Id) { + ctx.data().insert_temp(id, self); + } +} + +pub const DEFAULT_VIDEO_SIZE: egui::Vec2 = egui::vec2(320.0, 180.0); +pub const DEFAULT_MAX_COLUMNS: u32 = 4; +pub const DEFAULT_SPACING: f32 = 16.0; + +pub struct VideoGrid { + id: egui::Id, + + // Current frame + available_rect: egui::Rect, + prev_state: State, + curr_state: State, + video_index: u32, // Kinda "cursor" + + // Options + min_video_size: egui::Vec2, + max_columns: u32, + spacing: f32, +} + +impl VideoGrid { + pub fn new(id_source: impl std::hash::Hash) -> Self { + Self { + id: egui::Id::new(id_source), + available_rect: egui::Rect::NAN, + prev_state: State::default(), + curr_state: State::default(), + video_index: 0, + min_video_size: DEFAULT_VIDEO_SIZE, + max_columns: DEFAULT_MAX_COLUMNS, + spacing: DEFAULT_SPACING, + } + } + + pub fn show( + mut self, + ui: &mut egui::Ui, + grid: impl FnOnce(&mut VideoGridContext) -> R, + ) -> egui::InnerResponse { + // TODO(theomonnom): Should I care about the current egui layout? + + let prev_state = State::load(ui.ctx(), self.id); + let is_first_frame = prev_state.is_none(); + + self.prev_state = prev_state.unwrap_or_default(); + self.available_rect = ui.available_rect_before_wrap(); + + ui.ctx() + .check_for_id_clash(self.id, self.available_rect, "VideoGrid"); + + ui.allocate_ui_at_rect(self.available_rect, |ui| { + ui.set_visible(!is_first_frame); + + let mut ctx = VideoGridContext { + layout: &mut self, + ui, + }; + let res = grid(&mut ctx); + + // Save the new state + if self.curr_state != self.prev_state { + self.curr_state.clone().store(ui.ctx(), self.id); + ui.ctx().request_repaint(); + } + + res + }) + } + + fn next_frame_rect(&mut self) -> egui::Rect { + assert!(self.available_rect.is_finite()); + assert!(self.spacing <= self.min_video_size.x); + + // increment the amount of videos for the next frame + self.curr_state.num_videos += 1; + + let num_videos = self.prev_state.num_videos; + if num_videos == 0 { + return egui::Rect::NOTHING; + } + + let max_columns = self.max_columns; + let minimum_size = self.min_video_size; + let available_size = self.available_rect.size(); + + let calc_min_width = + |columns: u32| columns as f32 * minimum_size.x + (columns - 1) as f32 * self.spacing; + + let total_columns = { + let mut est = (available_size.x / minimum_size.x) as u32 + 1; + if available_size.x < calc_min_width(est) { + est -= 1; + } + cmp::max(1, cmp::min(est, max_columns)) + }; + + let aspect_ratio = minimum_size.x / minimum_size.y; + let remaining_width = available_size.x - calc_min_width(total_columns); + let w = minimum_size.x + remaining_width / total_columns as f32; + let h = w / aspect_ratio; + + let x_index = self.video_index % total_columns; + let y_index = self.video_index / total_columns; + + let x = { + let mut x = x_index as f32 * (w + self.spacing); + + // vertically center the last row + let total_rows = num_videos / total_columns + 1; + if (y_index + 1) == total_rows { + let nb_items = num_videos - (total_rows - 1) * total_columns; // nb. of items on the last row + x += (total_columns - nb_items) as f32 * (w + self.spacing) / 2.0; + } + + x + }; + let y = y_index as f32 * (h + self.spacing); + + let min = egui::pos2(x, y) + self.available_rect.left_top().to_vec2(); + let max = egui::pos2(w, h) + min.to_vec2(); + + self.video_index += 1; + + egui::Rect { min, max } + } +} + +impl VideoGrid { + pub fn min_video_size(mut self, min_video_size: egui::Vec2) -> Self { + self.min_video_size = min_video_size; + self + } + + pub fn max_columns(mut self, max_columns: u32) -> Self { + self.max_columns = max_columns; + self + } + + pub fn spacing(mut self, spacing: f32) -> Self { + self.spacing = spacing; + self + } +} + +pub struct VideoGridContext<'a> { + layout: &'a mut VideoGrid, + ui: &'a mut egui::Ui, +} + +impl<'a> VideoGridContext<'a> { + pub fn video_frame(&mut self, add_contents: impl FnOnce(&mut egui::Ui)) -> egui::Response { + let frame_rect = self.layout.next_frame_rect(); + + let mut child_ui = self.ui.child_ui(frame_rect, egui::Layout::default()); + add_contents(&mut child_ui); + + self.ui.allocate_rect(frame_rect, egui::Sense::hover()) + } +} diff --git a/examples/simple_room/src/video_renderer.rs b/examples/simple_room/src/video_renderer.rs index 124322b..0f0da9e 100644 --- a/examples/simple_room/src/video_renderer.rs +++ b/examples/simple_room/src/video_renderer.rs @@ -3,6 +3,7 @@ use livekit::webrtc::video_frame_buffer::PlanarYuv8Buffer; use livekit::webrtc::video_frame_buffer::PlanarYuvBuffer; use livekit::webrtc::video_frame_buffer::VideoFrameBufferTrait; use livekit::webrtc::yuv_helper; +use tracing::debug_span; use std::convert::TryInto; use std::num::NonZeroU32; use std::{ @@ -101,6 +102,9 @@ impl VideoRenderer { let internal = internal.clone(); Box::new(move |_frame, buffer| { + let span = debug_span!("texture_upload"); + let _enter = span.enter(); + let mut internal = internal.lock().unwrap(); let buffer = buffer.to_i420(); @@ -149,8 +153,6 @@ impl VideoRenderer { copy_layout, copy_size, ); - - println!("wrote"); }) });