use parking_lot::Mutex; use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; use std::sync::{Arc, Weak}; use std::time::Duration; use tokio::sync::{mpsc, Mutex as AsyncMutex}; use lazy_static::lazy_static; use prost::Message; use serde::{Deserialize, Serialize}; use thiserror::Error; use tokio::time::sleep; use tracing::{debug, error, trace}; use crate::{proto, signal_client}; use livekit_webrtc::data_channel::{DataChannel, DataChannelInit, DataSendError, DataState}; use livekit_webrtc::jsep::{IceCandidate, SdpParseError, SessionDescription}; use livekit_webrtc::media_stream::MediaStream; use livekit_webrtc::peer_connection::{ IceConnectionState, PeerConnectionState, RTCOfferAnswerOptions, }; use livekit_webrtc::peer_connection_factory::RTCConfiguration; use livekit_webrtc::rtc_error::RTCError; use livekit_webrtc::rtp_receiver::RtpReceiver; use crate::proto::data_packet::Value; use crate::proto::{ data_packet, signal_request, signal_response, DataPacket, JoinResponse, ParticipantUpdate, SignalTarget, TrickleRequest, }; use crate::rtc_engine::lk_runtime::LKRuntime; use crate::rtc_engine::pc_transport::PCTransport; use crate::rtc_engine::rtc_events::{RTCEmitter, RTCEvent, RTCEvents}; use crate::signal_client::{SignalClient, SignalError, SignalEvent, SignalEvents, SignalOptions}; mod lk_runtime; mod pc_transport; mod rtc_events; lazy_static! { // Share one LKRuntime across all RTCEngine instances static ref LK_RUNTIME: Mutex> = Mutex::new(Weak::new()); } pub(crate) type EngineEmitter = mpsc::Sender; pub(crate) type EngineEvents = mpsc::Receiver; pub(crate) type EngineResult = Result; pub(crate) const MAX_ICE_CONNECT_TIMEOUT: Duration = Duration::from_secs(15); pub(crate) const LOSSY_DC_LABEL: &str = "_lossy"; pub(crate) const RELIABLE_DC_LABEL: &str = "_reliable"; #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub(crate) enum PCState { New, Connected, Disconnected, Reconnecting, Closed, } #[derive(Serialize, Deserialize)] #[allow(non_snake_case)] struct IceCandidateJSON { sdpMid: String, sdpMLineIndex: i32, candidate: String, } #[derive(Error, Debug)] pub enum EngineError { #[error("signal failure")] Signal(#[from] SignalError), #[error("internal webrtc failure")] Rtc(#[from] RTCError), #[error("failed to parse sdp")] Parse(#[from] SdpParseError), #[error("serde error")] Serde(#[from] serde_json::Error), #[error("failed to send data to the datachannel")] Data(#[from] DataSendError), #[error("connection error: {0}")] Connection(String), #[error("decode error")] Decode(#[from] prost::DecodeError), #[error("internal error: {0}")] Internal(String), // Unexpected error } #[derive(Debug)] pub(crate) enum EngineEvent { ParticipantUpdate(ParticipantUpdate), AddTrack { rtp_receiver: RtpReceiver, streams: Vec, }, } #[derive(Debug)] struct EngineInner { has_published: AtomicBool, join_response: Mutex, pc_state: AtomicU8, // Casted to PCState enum publisher_pc: AsyncMutex, subscriber_pc: AsyncMutex, // Publisher data channels // Used to send data to other participants ( The SFU forward the messages ) lossy_dc: Mutex, reliable_dc: Mutex, // Subscriber data channels // These fields are never used, we just keep a strong reference to them, // so we can receive data from other participants sub_reliable_dc: Mutex>, sub_lossy_dc: Mutex>, } #[derive(Debug)] pub struct RTCEngine { signal_client: Arc, engine_inner: Arc, #[allow(unused)] lk_runtime: Arc, // Keep a reference while we're using the RTCEngine } impl RTCEngine { #[tracing::instrument(skip(url, token))] pub(crate) async fn connect( url: &str, token: &str, options: SignalOptions, ) -> EngineResult<(RTCEngine, EngineEvents)> { let mut lk_runtime = None; { let mut lk_runtime_ref = LK_RUNTIME.lock(); lk_runtime = lk_runtime_ref.upgrade(); if lk_runtime.is_none() { let new_runtime = Arc::new(LKRuntime::new()); *lk_runtime_ref = Arc::downgrade(&new_runtime); lk_runtime = Some(new_runtime); } } let lk_runtime = lk_runtime.unwrap(); let (signal_client, mut signal_events) = SignalClient::connect(url, token, options).await?; let join_response = signal_client::utils::next_join_response(&mut signal_events).await?; debug!("received JoinResponse: {:?}", join_response); let (engine_inner, rtc_events) = Self::configure_engine(lk_runtime.clone(), join_response.clone())?; let engine_inner = Arc::new(engine_inner); let signal_client = Arc::new(signal_client); let (emitter, events) = mpsc::channel(8); tokio::spawn(Self::signal_task( signal_client.clone(), engine_inner.clone(), signal_events, emitter.clone(), )); tokio::spawn(Self::engine_task( signal_client.clone(), engine_inner.clone(), rtc_events, emitter.clone(), )); let rtc_engine = Self { signal_client, engine_inner, lk_runtime, }; if !join_response.subscriber_primary { rtc_engine.negotiate_publisher().await?; } Ok((rtc_engine, events)) } #[tracing::instrument] pub async fn publish_data( &self, data: &DataPacket, kind: data_packet::Kind, ) -> Result<(), EngineError> { self.ensure_publisher_connected(kind).await?; self.data_channel(kind) .lock() .send(&data.encode_to_vec(), true) .map_err(Into::into) } pub fn join_response(&self) -> JoinResponse { self.engine_inner.join_response.lock().clone() } async fn engine_task( signal_client: Arc, engine_inner: Arc, mut rtc_events: RTCEvents, emitter: EngineEmitter, ) { while let Some(event) = rtc_events.recv().await { if let Err(err) = Self::handle_rtc( event, signal_client.clone(), engine_inner.clone(), emitter.clone(), ) .await { error!("failed to handle rtc event: {:?}", err); } } } async fn signal_task( signal_client: Arc, engine_inner: Arc, mut signal_events: SignalEvents, emitter: EngineEmitter, ) { while let Some(signal) = signal_events.recv().await { match signal { SignalEvent::Open => {} SignalEvent::Signal(signal) => { if let Err(err) = Self::handle_signal( signal, signal_client.clone(), engine_inner.clone(), emitter.clone(), ) .await { error!("failed to handle signal: {:?}", err); } } SignalEvent::Close => { // Try reconnect if this isn't expected } } } } async fn handle_rtc( event: RTCEvent, signal_client: Arc, engine_inner: Arc, emitter: EngineEmitter, ) -> EngineResult<()> { match event { RTCEvent::IceCandidate { ice_candidate, target, } => { let json = serde_json::to_string(&IceCandidateJSON { sdpMid: ice_candidate.sdp_mid(), sdpMLineIndex: ice_candidate.sdp_mline_index(), candidate: ice_candidate.candidate(), })?; trace!("sending ice_candidate ({:?}) - {:?}", target, ice_candidate); tokio::spawn(async move { signal_client .send(signal_request::Message::Trickle(TrickleRequest { candidate_init: json, target: target as i32, })) .await; }); } RTCEvent::ConnectionChange { state, target } => { // Reconnect if we've been disconnected unexpectedly let subscriber_primary = engine_inner.join_response.lock().subscriber_primary; let is_primary = subscriber_primary && target == SignalTarget::Subscriber; if is_primary && state == PeerConnectionState::Disconnected { let old_state = engine_inner .pc_state .swap(PCState::Connected as u8, Ordering::SeqCst); if old_state == PCState::New as u8 { // TODO(theomonnom) Handle disconnect } } else if state == PeerConnectionState::Failed { engine_inner .pc_state .store(PCState::Disconnected as u8, Ordering::SeqCst); // TODO(theomonnom) Handle disconnect } } RTCEvent::DataChannel { data_channel, target, } => { if target == SignalTarget::Subscriber { if data_channel.label() == RELIABLE_DC_LABEL { *engine_inner.sub_reliable_dc.lock() = Some(data_channel); } else { *engine_inner.sub_lossy_dc.lock() = Some(data_channel); } } } RTCEvent::Offer { offer, target } => { if target == SignalTarget::Publisher { // Send the publisher offer to the server tokio::spawn(async move { signal_client .send(signal_request::Message::Offer(proto::SessionDescription { r#type: "offer".to_string(), sdp: offer.to_string(), })) .await; }); } } RTCEvent::AddTrack { rtp_receiver, streams, target, } => { if target == SignalTarget::Subscriber { let _ = emitter.send(EngineEvent::AddTrack { rtp_receiver, streams, }); } } RTCEvent::Data { data, binary } => { if !binary { Err(EngineError::Internal( "text messages aren't supported".to_string(), ))?; } let data = DataPacket::decode(&*data)?; match data.value.unwrap() { Value::User(user) => { // TODO(theomonnom) Send event } Value::Speaker(_) => { // TODO(theomonnonm) } } } } Ok(()) } async fn handle_signal( event: signal_response::Message, signal_client: Arc, engine_inner: Arc, emitter: EngineEmitter, ) -> EngineResult<()> { match event { signal_response::Message::Answer(answer) => { trace!("received answer from the publisher: {:?}", answer); let sdp = SessionDescription::from(answer.r#type.parse().unwrap(), &answer.sdp)?; engine_inner .publisher_pc .lock() .await .set_remote_description(sdp) .await?; } signal_response::Message::Offer(offer) => { // Handle the subscriber offer & send an answer to livekit-server // We always get an offer from the server when connecting trace!("received offer from the publisher: {:?}", offer); let sdp = SessionDescription::from(offer.r#type.parse().unwrap(), &offer.sdp)?; engine_inner .subscriber_pc .lock() .await .set_remote_description(sdp) .await?; let answer = engine_inner .subscriber_pc .lock() .await .peer_connection() .create_answer(RTCOfferAnswerOptions::default()) .await?; engine_inner .subscriber_pc .lock() .await .peer_connection() .set_local_description(answer.clone()) .await?; tokio::spawn(async move { signal_client .send(signal_request::Message::Answer(proto::SessionDescription { r#type: "answer".to_string(), sdp: answer.to_string(), })) .await; }); } signal_response::Message::Trickle(trickle) => { // Add the IceCandidate received from the livekit-server let json: IceCandidateJSON = serde_json::from_str(&trickle.candidate_init)?; let ice = IceCandidate::from(&json.sdpMid, json.sdpMLineIndex, &json.candidate)?; trace!( "received ice_candidate {:?} - {:?}", SignalTarget::from_i32(trickle.target).unwrap(), ice ); if trickle.target == SignalTarget::Publisher as i32 { engine_inner .publisher_pc .lock() .await .add_ice_candidate(ice) .await?; } else { engine_inner .subscriber_pc .lock() .await .add_ice_candidate(ice) .await?; } } signal_response::Message::Update(update) => { let _ = emitter.send(EngineEvent::ParticipantUpdate(update)); } _ => {} } Ok(()) } async fn ensure_publisher_connected(&self, kind: data_packet::Kind) -> EngineResult<()> { if !self.join_response().subscriber_primary { return Ok(()); } let publisher = &self.engine_inner.publisher_pc; { let mut publisher = publisher.lock().await; if !publisher.is_connected() && publisher.peer_connection().ice_connection_state() != IceConnectionState::IceConnectionChecking { let _ = self.negotiate_publisher().await; } } let dc = self.data_channel(kind); if dc.lock().state() == DataState::Open { return Ok(()); } // Wait until the PeerConnection is connected let wait_connected = async move { while publisher.lock().await.is_connected() && dc.lock().state() == DataState::Open { sleep(Duration::from_millis(50)).await; } }; tokio::select! { _ = wait_connected => Ok(()), _ = sleep(MAX_ICE_CONNECT_TIMEOUT) => { let err = EngineError::Connection("could not establish publisher connection: timeout".to_string()); error!(error = ?err); Err(err) } } } async fn negotiate_publisher(&self) -> EngineResult<()> { self.engine_inner .has_published .store(true, Ordering::SeqCst); if let Err(err) = self .engine_inner .publisher_pc .lock() .await .negotiate() .await { error!("failed to negotiate the publisher: {:?}", err); Err(err)? } else { Ok(()) } } fn configure_engine( lk_runtime: Arc, join_response: JoinResponse, ) -> EngineResult<(EngineInner, RTCEvents)> { let (rtc_emitter, events) = mpsc::unbounded_channel(); let rtc_config = RTCConfiguration::from(join_response.clone()); let mut publisher_pc = PCTransport::new( lk_runtime .pc_factory .create_peer_connection(rtc_config.clone())?, ); let mut subscriber_pc = PCTransport::new( lk_runtime .pc_factory .create_peer_connection(rtc_config.clone())?, ); let mut lossy_dc = publisher_pc.peer_connection().create_data_channel( LOSSY_DC_LABEL, DataChannelInit { ordered: true, max_retransmits: Some(0), ..DataChannelInit::default() }, )?; let mut reliable_dc = publisher_pc.peer_connection().create_data_channel( RELIABLE_DC_LABEL, DataChannelInit { ordered: true, ..DataChannelInit::default() }, )?; publisher_pc .peer_connection() .on_ice_candidate(rtc_events::on_ice_candidate( SignalTarget::Publisher, rtc_emitter.clone(), )); subscriber_pc .peer_connection() .on_ice_candidate(rtc_events::on_ice_candidate( SignalTarget::Subscriber, rtc_emitter.clone(), )); publisher_pc.on_offer(rtc_events::on_offer( SignalTarget::Publisher, rtc_emitter.clone(), )); subscriber_pc.on_offer(rtc_events::on_offer( SignalTarget::Subscriber, rtc_emitter.clone(), )); publisher_pc .peer_connection() .on_data_channel(rtc_events::on_data_channel( SignalTarget::Publisher, rtc_emitter.clone(), )); subscriber_pc .peer_connection() .on_data_channel(rtc_events::on_data_channel( SignalTarget::Subscriber, rtc_emitter.clone(), )); publisher_pc .peer_connection() .on_add_track(rtc_events::on_add_track( SignalTarget::Publisher, rtc_emitter.clone(), )); subscriber_pc .peer_connection() .on_add_track(rtc_events::on_add_track( SignalTarget::Subscriber, rtc_emitter.clone(), )); publisher_pc .peer_connection() .on_connection_change(rtc_events::on_connection_change( SignalTarget::Publisher, rtc_emitter.clone(), )); subscriber_pc .peer_connection() .on_connection_change(rtc_events::on_connection_change( SignalTarget::Subscriber, rtc_emitter.clone(), )); lossy_dc.on_message(rtc_events::on_message(rtc_emitter.clone())); reliable_dc.on_message(rtc_events::on_message(rtc_emitter.clone())); Ok(( EngineInner { has_published: AtomicBool::new(false), join_response: Mutex::new(join_response), pc_state: AtomicU8::new(PCState::New as u8), publisher_pc: AsyncMutex::new(publisher_pc), subscriber_pc: AsyncMutex::new(subscriber_pc), lossy_dc: Mutex::new(lossy_dc), reliable_dc: Mutex::new(reliable_dc), sub_lossy_dc: Mutex::new(None), sub_reliable_dc: Mutex::new(None), }, events, )) } fn data_channel(&self, kind: data_packet::Kind) -> &Mutex { if kind == data_packet::Kind::Reliable { &self.engine_inner.reliable_dc } else { &self.engine_inner.lossy_dc } } }