Merge pull request #7 from livekit/theo/tracks

Initial downstream tracks + VideoRenderer example

Cargo.lock

@@ -484,6 +484,7 @@ name = "livekit"
 version = "0.1.0"
 dependencies = [
  "livekit-core",
+ "livekit-webrtc",
 ]
 
 [[package]]
@@ -493,7 +494,9 @@ dependencies = [
  "futures",
  "futures-util",
  "lazy_static",
+ "livekit-utils",
  "livekit-webrtc",
+ "parking_lot",
  "prost",
  "prost-build",
  "prost-types",
@@ -506,6 +509,10 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "livekit-utils"
+version = "0.1.0"
+
 [[package]]
 name = "livekit-webrtc"
 version = "0.1.0"
@@ -513,6 +520,7 @@ dependencies = [
  "cxx",
  "env_logger",
  "libwebrtc-sys",
+ "livekit-utils",
  "log",
  "thiserror",
  "tokio",

Cargo.toml

@@ -9,8 +9,10 @@ exclude = ["libwebrtc"]
 members = [
     "crates/livekit-core",
     "crates/livekit-webrtc",
+    "crates/livekit-utils",
     "crates/livekit-webrtc/libwebrtc-sys"
 ]
 
 [dependencies]
-livekit-core = { path = "crates/livekit-core" }
+livekit-core = { path = "crates/livekit-core" }
+livekit-webrtc = { path = "crates/livekit-webrtc" }

crates/livekit-core/Cargo.toml

@@ -5,19 +5,21 @@ edition = "2021"
 license = "Apache-2.0"
 
 [dependencies]
+livekit-webrtc = { path = "../livekit-webrtc" }
+livekit-utils = { path = "../livekit-utils" }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1.0"
 tokio-tungstenite = { version = "0.17.2", features = ["native-tls"] }
 tokio = { version = "1", features = ["full"] }
 futures = "0.3"
+parking_lot = { version = "0.12.1", features = ["send_guard"] }
 url = "2.2.2"
 futures-util = "0.3.23"
 thiserror = "1.0"
 prost = "0.11.0"
 prost-types = "0.11.1"
-livekit-webrtc = { path = "../livekit-webrtc" }
 lazy_static = "1.4.0"
 tracing = "0.1"
 
 [build-dependencies]
-prost-build = { version = "0.11.1" }
+prost-build = { version = "0.11.1" }

crates/livekit-core/src/event.rs

@@ -1,43 +0,0 @@
-use std::pin::Pin;
-use std::task::{Context, Poll};
-use futures::Stream;
-use tokio::sync::mpsc;
-
-/// Using unbounded channels to prevent users from blocking internal logic ( e.g: ws heartbeat )
-/// Users must listen to all events to avoid the process from running out of memory
-
-#[derive(Clone, Debug)]
-pub struct Emitter<T> {
-    tx: mpsc::UnboundedSender<T>,
-}
-
-impl<T> Emitter<T> {
-    pub fn new() -> (Self, mpsc::UnboundedReceiver<T>) {
-        let (tx, rx) = mpsc::unbounded_channel();
-
-        (Self { tx }, rx)
-    }
-
-    pub fn event(&self, event: T) {
-        let _ = self.tx.send(event);
-    }
-}
-
-#[derive(Debug)]
-pub struct Events<T> {
-    rx: mpsc::UnboundedReceiver<T>,
-}
-
-impl<T> Events<T> {
-    pub fn new(rx: mpsc::UnboundedReceiver<T>) -> Self {
-        Self { rx }
-    }
-}
-
-impl<T> Stream for Events<T> {
-    type Item = T;
-
-    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
-        self.rx.poll_recv(cx)
-    }
-}

crates/livekit-core/src/events.rs

@@ -0,0 +1,140 @@
+use futures_util::future::BoxFuture;
+use thiserror::Error;
+
+type EventHandler<T> = Box<dyn FnMut(T) -> BoxFuture<'static, ()> + Send + Sync>;
+
+macro_rules! event_setter {
+    ($fnc:ident, $event:ty) => {
+        pub fn $fnc<F, Fut>(&self, mut callback: F)
+        where
+            F: FnMut($event) -> Fut + Send + Sync + 'static,
+            Fut: Future<Output = ()> + Send + 'static,
+        {
+            *self.$fnc.lock() = Some(Box::new(move |event| Box::pin(callback(event))));
+        }
+    };
+}
+
+#[derive(Error, Debug, Clone)]
+pub enum TrackError {
+    #[error("could not find published track with sid: {0}")]
+    TrackNotFound(String),
+}
+
+pub mod room {
+    use super::{EventHandler, TrackError};
+    use crate::room::id::TrackSid;
+    use crate::room::participant::remote_participant::RemoteParticipant;
+    use crate::room::publication::RemoteTrackPublication;
+    use crate::room::track::remote_track::RemoteTrackHandle;
+    use crate::room::RoomHandle;
+    use futures::future::Future;
+    use parking_lot::Mutex;
+    use std::sync::Arc;
+
+    #[derive(Clone)]
+    pub struct ParticipantConnectedEvent {
+        pub room_handle: RoomHandle,
+        pub participant: Arc<RemoteParticipant>,
+    }
+
+    #[derive(Clone)]
+    pub struct ParticipantDisconnectedEvent {
+        pub room_handle: RoomHandle,
+        pub participant: Arc<RemoteParticipant>,
+    }
+
+    #[derive(Clone)]
+    pub struct TrackSubscribedEvent {
+        pub room_handle: RoomHandle,
+        pub track: RemoteTrackHandle,
+        pub publication: RemoteTrackPublication,
+        pub participant: Arc<RemoteParticipant>,
+    }
+
+    #[derive(Clone)]
+    pub struct TrackPublishedEvent {
+        pub room_handle: RoomHandle,
+        pub publication: RemoteTrackPublication,
+        pub participant: Arc<RemoteParticipant>,
+    }
+
+    #[derive(Clone)]
+    pub struct TrackSubscriptionFailedEvent {
+        pub room_handle: RoomHandle,
+        pub error: TrackError,
+        pub sid: TrackSid,
+        pub participant: Arc<RemoteParticipant>,
+    }
+
+    pub(crate) type OnParticipantConnectedHandler = EventHandler<ParticipantConnectedEvent>;
+    pub(crate) type OnParticipantDisconnectedHandler = EventHandler<ParticipantDisconnectedEvent>;
+    pub(crate) type OnTrackSubscribedEventHandler = EventHandler<TrackSubscribedEvent>;
+    pub(crate) type OnTrackPublishedEventHandler = EventHandler<TrackPublishedEvent>;
+    pub(crate) type OnTrackSubscriptionFailedHandler = EventHandler<TrackSubscriptionFailedEvent>;
+
+    #[derive(Default)]
+    pub struct RoomEvents {
+        pub(crate) on_participant_connected: Mutex<Option<OnParticipantConnectedHandler>>,
+        pub(crate) on_participant_disconnected: Mutex<Option<OnParticipantDisconnectedHandler>>,
+        pub(crate) on_track_subscribed: Mutex<Option<OnTrackSubscribedEventHandler>>,
+        pub(crate) on_track_published: Mutex<Option<OnTrackPublishedEventHandler>>,
+        pub(crate) on_track_subscription_failed: Mutex<Option<OnTrackSubscriptionFailedHandler>>,
+    }
+
+    impl RoomEvents {
+        event_setter!(on_participant_connected, ParticipantConnectedEvent);
+        event_setter!(on_participant_disconnected, ParticipantDisconnectedEvent);
+        event_setter!(on_track_subscribed, TrackSubscribedEvent);
+        event_setter!(on_track_published, TrackPublishedEvent);
+        event_setter!(on_track_subscription_failed, TrackSubscriptionFailedEvent);
+    }
+}
+
+pub mod participant {
+    use super::{EventHandler, TrackError};
+    use crate::room::id::TrackSid;
+    use crate::room::participant::remote_participant::RemoteParticipant;
+    use crate::room::publication::RemoteTrackPublication;
+    use crate::room::track::remote_track::RemoteTrackHandle;
+    use futures::future::Future;
+    use parking_lot::Mutex;
+    use std::sync::Arc;
+
+    #[derive(Clone)]
+    pub struct TrackPublishedEvent {
+        pub publication: RemoteTrackPublication,
+        pub participant: Arc<RemoteParticipant>,
+    }
+
+    #[derive(Clone)]
+    pub struct TrackSubscribedEvent {
+        pub track: RemoteTrackHandle,
+        pub publication: RemoteTrackPublication,
+        pub participant: Arc<RemoteParticipant>,
+    }
+
+    #[derive(Clone)]
+    pub struct TrackSubscriptionFailedEvent {
+        pub sid: TrackSid,
+        pub error: TrackError,
+        pub participant: Arc<RemoteParticipant>,
+    }
+
+    pub(crate) type TrackPublishedHandler = EventHandler<TrackPublishedEvent>;
+    pub(crate) type TrackSubscribedHandler = EventHandler<TrackSubscribedEvent>;
+    pub(crate) type TrackSubscriptionFailedHandler = EventHandler<TrackSubscriptionFailedEvent>;
+
+    #[derive(Default)]
+    pub struct ParticipantEvents {
+        pub(crate) on_track_published: Mutex<Option<TrackPublishedHandler>>,
+        pub(crate) on_track_subscribed: Mutex<Option<TrackSubscribedHandler>>,
+        pub(crate) on_track_subscription_failed: Mutex<Option<TrackSubscriptionFailedHandler>>,
+    }
+
+    impl ParticipantEvents {
+        event_setter!(on_track_published, TrackPublishedEvent);
+        event_setter!(on_track_subscribed, TrackSubscribedEvent);
+        event_setter!(on_track_subscription_failed, TrackSubscriptionFailedEvent);
+    }
+}

crates/livekit-core/src/lib.rs

@@ -1,12 +1,11 @@
+extern crate core;
+
 pub mod proto {
     include!(concat!(env!("OUT_DIR"), "/livekit.rs"));
 }
 
-mod lk_runtime;
-mod signal_client;
-mod pc_transport;
+mod events;
 mod rtc_engine;
-mod local_participant;
-mod event;
+mod signal_client;
 
-pub mod room;
+pub mod room;

crates/livekit-core/src/local_participant.rs

@@ -1,43 +0,0 @@
-use std::sync::Arc;
-use tokio::sync::Mutex;
-
-use crate::proto::{data_packet, DataPacket, ParticipantInfo, UserPacket};
-use crate::room::RoomError;
-use crate::rtc_engine::RTCEngine;
-
-pub struct LocalParticipant {
-    sid: String,
-    identity: String,
-    name: String,
-
-    engine: Arc<Mutex<RTCEngine>>,
-}
-
-impl LocalParticipant {
-    pub(crate) fn from(info: ParticipantInfo, engine: Arc<Mutex<RTCEngine>>) -> Self {
-        Self {
-            sid: info.sid,
-            identity: info.identity,
-            name: info.name,
-            engine,
-        }
-    }
-
-    pub(crate) fn update(info: ParticipantInfo) {
-        // TODO(theomonnom)
-    }
-
-    // TODO(theomonnom) Add the destinations parameter
-    pub async fn publish_data(&mut self, data: &[u8], kind: data_packet::Kind) -> Result<(), RoomError> {
-        let data = DataPacket {
-            kind: kind as i32,
-            value: Some(data_packet::Value::User(UserPacket {
-                participant_sid: self.sid.clone(),
-                payload: data.to_vec(),
-                destination_sids: vec![], // TODO(theomonnom)
-            })),
-        };
-
-        self.engine.lock().await.publish_data(&data, kind).await.map_err(Into::into)
-    }
-}

crates/livekit-core/src/room.rs

@@ -1,78 +0,0 @@
-use std::sync::Arc;
-
-use thiserror::Error;
-use tokio::sync::Mutex;
-
-use crate::local_participant::LocalParticipant;
-use crate::rtc_engine;
-use crate::rtc_engine::{EngineError, RTCEngine};
-
-#[derive(Error, Debug)]
-pub enum RoomError {
-    #[error("internal RTCEngine failure")]
-    Engine(#[from] EngineError),
-}
-
-#[derive(Debug)]
-pub enum RoomEvent { 
-    
-}
-
-pub struct Room {
-    sid: String,
-    name: String,
-    local_participant: LocalParticipant,
-    internal: Arc<RoomInternal>
-}
-
-#[tracing::instrument(skip(url, token))]
-pub async fn connect(url: &str, token: &str) -> Result<Room, RoomError> {
-    let engine = rtc_engine::connect(url, token).await?;
-    let join = engine.join_response().await;
-    let engine = Arc::new(Mutex::new(engine));
-    let local_participant = LocalParticipant::from(join.participant.unwrap(), engine.clone());
-    let internal = Arc::new(RoomInternal::new(engine));
-    let room_info = join.room.unwrap();
-
-    tokio::spawn(async move {
-
-    });
-
-    Ok(Room {
-        sid: room_info.sid,
-        name: room_info.name,
-        local_participant,
-        internal,
-    })
-}
-
-impl Room {
-    pub fn local_participant(&self) -> &LocalParticipant {
-        &self.local_participant
-    }
-
-    pub fn local_participant_mut(&mut self) -> &mut LocalParticipant {
-        &mut self.local_participant
-    }
-
-    pub fn sid(&self) -> &str {
-        &self.sid
-    }
-
-    pub fn name(&self) -> &str {
-        &self.name
-    }
-}
-
-
-struct RoomInternal {
-    engine: Arc<Mutex<RTCEngine>>,
-}
-
-impl RoomInternal {
-    pub fn new(engine: Arc<Mutex<RTCEngine>>) -> Self {
-        Self {
-            engine
-        }
-    }
-}

crates/livekit-core/src/room/id.rs

@@ -0,0 +1,56 @@
+use std::fmt;
+
+macro_rules! id_str {
+    ($($name:ident;)*) => {
+        $(
+            impl $name {
+                pub fn new(str: String) -> Self {
+                    Self(str)
+                }
+
+                pub fn as_str(&self) -> &str {
+                    &self.0
+                }
+            }
+
+            impl From<String> for $name {
+                fn from(str: String) -> $name {
+                    $name(str)
+                }
+            }
+
+            impl From<$name> for String {
+                fn from(id: $name) -> String {
+                    id.0
+                }
+            }
+
+            impl PartialEq<$name> for String {
+                fn eq(&self, u: &$name) -> bool {
+                    *self == *u.0
+                }
+            }
+
+            impl fmt::Display for $name {
+                fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+                    f.write_str(&self.0)
+                }
+            }
+        )*
+    }
+}
+
+#[derive(Clone, Default, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)]
+pub struct ParticipantSid(String);
+
+#[derive(Clone, Default, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)]
+pub struct ParticipantIdentity(String);
+
+#[derive(Clone, Default, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)]
+pub struct TrackSid(String);
+
+id_str! {
+    ParticipantSid;
+    ParticipantIdentity;
+    TrackSid;
+}

crates/livekit-core/src/room/mod.rs

@@ -0,0 +1,317 @@
+use parking_lot::lock_api::RwLockUpgradableReadGuard;
+use parking_lot::{Mutex, RwLock};
+use std::collections::HashMap;
+use std::sync::atomic::AtomicU8;
+use std::sync::Arc;
+
+use self::id::ParticipantSid;
+use self::participant::local_participant::LocalParticipant;
+use self::participant::remote_participant::RemoteParticipant;
+use self::participant::ParticipantInternalTrait;
+use self::participant::ParticipantTrait;
+use crate::events::room::{
+    ParticipantConnectedEvent, ParticipantDisconnectedEvent, RoomEvents, TrackSubscribedEvent,
+};
+use crate::proto;
+use crate::proto::participant_info;
+use thiserror::Error;
+use tracing::{debug, error};
+
+use crate::rtc_engine::{EngineError, EngineEvent, EngineEvents, RTCEngine};
+use crate::signal_client::SignalOptions;
+
+pub mod id;
+pub mod participant;
+pub mod publication;
+pub mod track;
+
+#[derive(Error, Debug)]
+pub enum RoomError {
+    #[error("internal RTCEngine failure")]
+    Engine(#[from] EngineError),
+    #[error("internal Room failure")]
+    Internal(String),
+}
+
+type RoomResult<T> = Result<T, RoomError>;
+
+#[derive(Debug)]
+pub enum ConnectionState {
+    Disconnected,
+    Connecting,
+    Connected,
+    Reconnecting,
+}
+
+struct RoomInner {
+    state: AtomicU8, // ConnectionState
+    sid: Mutex<String>,
+    name: Mutex<String>,
+    participants: RwLock<HashMap<ParticipantSid, Arc<RemoteParticipant>>>,
+    rtc_engine: Arc<RTCEngine>,
+    local_participant: Arc<LocalParticipant>,
+}
+
+pub struct Room {
+    inner: Option<Arc<RoomInner>>,
+    events: Arc<RoomEvents>,
+}
+
+impl Room {
+    pub fn new() -> Room {
+        Self {
+            inner: None,
+            events: Default::default(),
+        }
+    }
+
+    pub async fn connect(&mut self, url: &str, token: &str) -> RoomResult<()> {
+        let (rtc_engine, engine_events) =
+            RTCEngine::connect(url, token, SignalOptions::default()).await?;
+        let rtc_engine = Arc::new(rtc_engine);
+        let join_response = rtc_engine.join_response();
+        let local_participant = Arc::new(LocalParticipant::new(
+            rtc_engine.clone(),
+            join_response.participant.unwrap().clone(),
+        ));
+        let room_info = join_response.room.unwrap();
+        let inner = Arc::new(RoomInner {
+            state: AtomicU8::new(ConnectionState::Connecting as u8),
+            sid: Mutex::new(room_info.sid),
+            name: Mutex::new(room_info.name),
+            participants: Default::default(),
+            rtc_engine,
+            local_participant,
+        });
+
+        self.inner = Some(inner.clone());
+
+        // Add already connected participants
+        for pi in join_response.other_participants {
+            let p = Self::create_participant(inner.clone(), self.events.clone(), pi.clone());
+            p.update_info(pi).await;
+        }
+
+        tokio::spawn(Self::room_task(inner, self.events.clone(), engine_events));
+
+        Ok(())
+    }
+
+    pub fn events(&self) -> Arc<RoomEvents> {
+        self.events.clone()
+    }
+
+    pub fn get_handle(&self) -> Option<RoomHandle> {
+        self.inner.as_ref().map(|inner| RoomHandle {
+            inner: inner.clone(),
+        })
+    }
+
+    async fn room_task(
+        room_inner: Arc<RoomInner>,
+        room_events: Arc<RoomEvents>,
+        mut engine_events: EngineEvents,
+    ) {
+        while let Some(event) = engine_events.recv().await {
+            if let Err(err) =
+                Self::handle_event(room_inner.clone(), room_events.clone(), event).await
+            {
+                error!("failed to handle engine event: {:?}", err);
+            }
+        }
+    }
+
+    async fn handle_event(
+        room_inner: Arc<RoomInner>,
+        room_events: Arc<RoomEvents>,
+        event: EngineEvent,
+    ) -> RoomResult<()> {
+        match event {
+            EngineEvent::ParticipantUpdate(update) => {
+                Self::handle_participant_update(room_inner.clone(), room_events.clone(), update)
+                    .await
+            }
+            EngineEvent::AddTrack {
+                rtp_receiver,
+                streams,
+            } => {
+                if streams.is_empty() {
+                    Err(RoomError::Internal(
+                        "AddTrack event with empty streams".to_string(),
+                    ))?;
+                }
+
+                let first_stream_id = streams.first().unwrap().id();
+                let stream_id = unpack_stream_id(&first_stream_id);
+                if stream_id.is_none() {
+                    Err(RoomError::Internal(format!(
+                        "AddTrack event with invalid track_id: {:?}",
+                        first_stream_id
+                    )))?;
+                }
+
+                let (participant_sid, track_sid) = stream_id.unwrap();
+                let remote_participant =
+                    Self::get_participant(room_inner.clone(), &participant_sid.to_string().into());
+
+                if let Some(remote_participant) = remote_participant {
+                    remote_participant.add_subscribed_media_track(
+                        track_sid.to_string().into(),
+                        rtp_receiver.track(),
+                    );
+                } else {
+                    // The server should send participant updates before sending a new offer
+                    // So this should not happen.
+                    Err(RoomError::Internal(format!(
+                        "AddTrack event with invalid participant_sid: {:?}",
+                        participant_sid
+                    )))?;
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    async fn handle_participant_update(
+        room_inner: Arc<RoomInner>,
+        room_events: Arc<RoomEvents>,
+        update: proto::ParticipantUpdate,
+    ) {
+        for pi in update.participants {
+            if pi.sid == room_inner.local_participant.sid()
+                || pi.identity == room_inner.local_participant.identity()
+            {
+                room_inner.local_participant.clone().update_info(pi).await;
+                continue;
+            }
+
+            let remote_participant =
+                Self::get_participant(room_inner.clone(), &pi.sid.clone().into());
+
+            if let Some(remote_participant) = remote_participant {
+                if pi.state == participant_info::State::Disconnected as i32 {
+                    // Participant disconencted
+                    Self::handle_participant_disconnect(
+                        room_inner.clone(),
+                        room_events.clone(),
+                        remote_participant,
+                    )
+                } else {
+                    // Participant is already connected, update the informations
+                    remote_participant.update_info(pi).await;
+                }
+            } else {
+                // Create a new participant and call OnConnect event
+                let remote_participant =
+                    Self::create_participant(room_inner.clone(), room_events.clone(), pi);
+                let mut handler = room_events.on_participant_connected.lock();
+                if let Some(cb) = handler.as_mut() {
+                    cb(ParticipantConnectedEvent {
+                        room_handle: RoomHandle::from(room_inner.clone()),
+                        participant: remote_participant.clone(),
+                    });
+                }
+            }
+        }
+    }
+
+    fn handle_participant_disconnect(
+        room_inner: Arc<RoomInner>,
+        room_events: Arc<RoomEvents>,
+        remote_participant: Arc<RemoteParticipant>,
+    ) {
+        room_inner
+            .participants
+            .write()
+            .remove(&remote_participant.sid());
+
+        // TODO(theomonnom): Unpublish all tracks
+
+        let mut handler = room_events.on_participant_disconnected.lock();
+        if let Some(cb) = handler.as_mut() {
+            cb(ParticipantDisconnectedEvent {
+                room_handle: RoomHandle::from(room_inner.clone()),
+                participant: remote_participant.clone(),
+            });
+        }
+    }
+
+    fn get_participant(
+        room_inner: Arc<RoomInner>,
+        sid: &ParticipantSid,
+    ) -> Option<Arc<RemoteParticipant>> {
+        room_inner.participants.read().get(sid).cloned()
+    }
+
+    fn create_participant(
+        room_inner: Arc<RoomInner>,
+        room_events: Arc<RoomEvents>,
+        pi: proto::ParticipantInfo,
+    ) -> Arc<RemoteParticipant> {
+        let p = Arc::new(RemoteParticipant::new(pi.clone()));
+
+        // Forward participantevents to room events
+        p.internal_events().on_track_subscribed({
+            let room_events = room_events.clone();
+            let room_inner = room_inner.clone();
+
+            move |event| {
+                let room_events = room_events.clone();
+                let room_inner = room_inner.clone();
+
+                async move {
+                    if let Some(cb) = room_events.clone().on_track_subscribed.lock().as_mut() {
+                        cb(TrackSubscribedEvent {
+                            room_handle: RoomHandle::from(room_inner.clone()),
+                            track: event.track,
+                            participant: event.participant,
+                            publication: event.publication,
+                        })
+                        .await;
+                    }
+                }
+            }
+        });
+
+        room_inner
+            .participants
+            .write()
+            .insert(pi.sid.into(), p.clone());
+        p
+    }
+}
+
+#[derive(Clone)]
+pub struct RoomHandle {
+    inner: Arc<RoomInner>,
+}
+
+impl RoomHandle {
+    fn from(room_inner: Arc<RoomInner>) -> Self {
+        Self { inner: room_inner }
+    }
+
+    pub fn sid(&self) -> String {
+        self.inner.sid.lock().clone()
+    }
+
+    pub fn name(&self) -> String {
+        self.inner.name.lock().clone()
+    }
+
+    pub fn local_participant(&self) -> Arc<LocalParticipant> {
+        self.inner.local_participant.clone()
+    }
+}
+
+fn unpack_stream_id(stream_id: &str) -> Option<(&str, &str)> {
+    let split: Vec<&str> = stream_id.split('|').collect();
+    if split.len() == 2 {
+        let participant_sid = split.get(0).unwrap();
+        let track_sid = split.get(1).unwrap();
+        Some((participant_sid, track_sid))
+    } else {
+        None
+    }
+}

crates/livekit-core/src/room/participant/local_participant.rs

@@ -0,0 +1,55 @@
+use crate::proto::{data_packet, DataPacket, UserPacket};
+use crate::room::participant::{impl_participant_trait, ParticipantShared, ParticipantInternalTrait};
+use crate::room::RoomError;
+use crate::rtc_engine::RTCEngine;
+
+pub struct LocalParticipant {
+    shared: ParticipantShared,
+    rtc_engine: Arc<RTCEngine>,
+}
+
+impl LocalParticipant {
+    pub(crate) fn new(rtc_engine: Arc<RTCEngine>, info: ParticipantInfo) -> Self {
+        Self {
+            shared: ParticipantShared::new(
+                info.sid.into(),
+                info.identity.into(),
+                info.name,
+                info.metadata,
+            ),
+            rtc_engine,
+        }
+    }
+
+    pub async fn publish_data(
+        &self,
+        data: &[u8],
+        kind: data_packet::Kind,
+    ) -> Result<(), RoomError> {
+        let data = DataPacket {
+            kind: kind as i32,
+            value: Some(data_packet::Value::User(UserPacket {
+                participant_sid: "".to_string(), /*self.sid().to_owned()*/
+                payload: data.to_vec(),
+                destination_sids: vec![],
+            })),
+        };
+
+        self.rtc_engine
+            .publish_data(&data, kind)
+            .await
+            .map_err(Into::into)
+    }
+
+    pub(crate) async fn update_info(self: Arc<Self>, info: ParticipantInfo) {
+        self.shared.update_info(info);
+    }
+}
+
+impl ParticipantInternalTrait for LocalParticipant {
+    fn internal_events(&self) -> Arc<ParticipantEvents> {
+        self.shared.internal_events.clone()
+    }
+}
+
+impl_participant_trait!(LocalParticipant);

crates/livekit-core/src/room/participant/mod.rs

@@ -0,0 +1,135 @@
+use crate::events::participant::ParticipantEvents;
+use crate::proto::ParticipantInfo;
+use crate::room::id::{ParticipantIdentity, ParticipantSid, TrackSid};
+use crate::room::participant::local_participant::LocalParticipant;
+use crate::room::participant::remote_participant::RemoteParticipant;
+use crate::room::publication::{TrackPublication, TrackPublicationTrait};
+use futures_util::future::BoxFuture;
+use livekit_utils::enum_dispatch;
+use parking_lot::{Mutex, RwLock};
+use std::collections::HashMap;
+use std::sync::Arc;
+
+pub mod local_participant;
+pub mod remote_participant;
+
+type OnTrackSubscribed = Box<dyn FnMut(ParticipantHandle) -> BoxFuture<'static, ()> + Send + Sync>;
+
+pub(super) struct ParticipantShared {
+    pub(super) events: Arc<ParticipantEvents>,
+    pub(super) internal_events: Arc<ParticipantEvents>,
+    pub(super) sid: Mutex<ParticipantSid>,
+    pub(super) identity: Mutex<ParticipantIdentity>,
+    pub(super) name: Mutex<String>,
+    pub(super) metadata: Mutex<String>,
+    pub(super) tracks: RwLock<HashMap<TrackSid, TrackPublication>>,
+}
+
+impl ParticipantShared {
+    pub(super) fn new(
+        sid: ParticipantSid,
+        identity: ParticipantIdentity,
+        name: String,
+        metadata: String,
+    ) -> Self {
+        Self {
+            events: Default::default(),
+            internal_events: Default::default(),
+            sid: Mutex::new(sid),
+            identity: Mutex::new(identity),
+            name: Mutex::new(name),
+            metadata: Mutex::new(metadata),
+            tracks: Default::default(),
+        }
+    }
+
+    pub(crate) fn update_info(&self, info: ParticipantInfo) {
+        *self.sid.lock() = info.sid.into();
+        *self.identity.lock() = info.identity.into();
+        *self.name.lock() = info.name;
+        *self.metadata.lock() = info.metadata; // TODO(theomonnom): callback MetadataChanged
+    }
+
+    pub(crate) fn add_track_publication(&self, publication: TrackPublication) {
+        self.tracks.write().insert(publication.sid(), publication);
+    }
+}
+
+pub(crate) trait ParticipantInternalTrait {
+    fn internal_events(&self) -> Arc<ParticipantEvents>;
+}
+
+pub trait ParticipantTrait {
+    fn events(&self) -> Arc<ParticipantEvents>;
+    fn sid(&self) -> ParticipantSid;
+    fn identity(&self) -> ParticipantIdentity;
+    fn name(&self) -> String;
+    fn metadata(&self) -> String;
+}
+
+#[derive(Clone)]
+pub enum ParticipantHandle {
+    Local(Arc<LocalParticipant>),
+    Remote(Arc<RemoteParticipant>),
+}
+
+impl ParticipantHandle {
+    // TODO(theomonnom): Add async support to wrap_variants ...
+    pub(crate) async fn update_info(&self, info: ParticipantInfo) {
+        match self {
+            Self::Local(inner) => inner.clone().update_info(info).await,
+            Self::Remote(inner) => inner.clone().update_info(info).await,
+        }
+    }
+}
+
+impl ParticipantInternalTrait for ParticipantHandle {
+    enum_dispatch!(
+        [Local, Remote]
+        fnc!(internal_events, &Self, [], Arc<ParticipantEvents>);
+    );
+}
+
+impl ParticipantTrait for ParticipantHandle {
+    enum_dispatch!(
+        [Local, Remote]
+        fnc!(events, &Self, [], Arc<ParticipantEvents>);
+        fnc!(sid, &Self, [], ParticipantSid);
+        fnc!(identity, &Self, [], ParticipantIdentity);
+        fnc!(name, &Self, [], String);
+        fnc!(metadata, &Self, [], String);
+    );
+}
+
+macro_rules! impl_participant_trait {
+    ($x:ty) => {
+        use crate::events::participant::ParticipantEvents;
+        use crate::proto::ParticipantInfo;
+        use crate::room::id::{ParticipantIdentity, ParticipantSid};
+        use std::sync::Arc;
+
+        impl crate::room::participant::ParticipantTrait for $x {
+            fn events(&self) -> Arc<ParticipantEvents> {
+                self.shared.events.clone()
+            }
+
+            fn sid(&self) -> ParticipantSid {
+                self.shared.sid.lock().clone()
+            }
+
+            fn identity(&self) -> ParticipantIdentity {
+                self.shared.identity.lock().clone()
+            }
+
+            fn name(&self) -> String {
+                self.shared.name.lock().clone()
+            }
+
+            fn metadata(&self) -> String {
+                self.shared.metadata.lock().clone()
+            }
+        }
+    };
+}
+
+pub(super) use impl_participant_trait;

crates/livekit-core/src/room/participant/remote_participant.rs

@@ -0,0 +1,205 @@
+use crate::events::participant::{
+    TrackPublishedEvent, TrackSubscribedEvent, TrackSubscriptionFailedEvent,
+};
+use crate::events::TrackError;
+use crate::room::id::TrackSid;
+use crate::room::participant::{
+    impl_participant_trait, ParticipantInternalTrait, ParticipantShared,
+};
+use crate::room::publication::{
+    RemoteTrackPublication, TrackPublication, TrackPublicationInternalTrait, TrackPublicationTrait,
+};
+use crate::room::track::remote_audio_track::RemoteAudioTrack;
+use crate::room::track::remote_track::RemoteTrackHandle;
+use crate::room::track::remote_video_track::RemoteVideoTrack;
+use crate::room::track::{TrackKind, TrackTrait, TrackHandle};
+use livekit_webrtc::media_stream::MediaStreamTrackHandle;
+use std::collections::HashSet;
+use std::time::Duration;
+use tokio::time::{sleep, timeout};
+use tracing::{info, error};
+
+use super::ParticipantTrait;
+
+const ADD_TRACK_TIMEOUT: Duration = Duration::from_secs(5);
+
+pub struct RemoteParticipant {
+    shared: ParticipantShared,
+}
+
+impl RemoteParticipant {
+    pub(crate) fn new(info: ParticipantInfo) -> Self {
+        Self {
+            shared: ParticipantShared::new(
+                info.sid.into(),
+                info.identity.into(),
+                info.name,
+                info.metadata,
+            ),
+        }
+    }
+
+    pub(crate) fn add_subscribed_media_track(
+        self: Arc<Self>,
+        sid: TrackSid,
+        media_track: MediaStreamTrackHandle,
+    ) {
+        tokio::spawn(async move {
+            let wait_publication = {
+                let participant = self.clone();
+                let sid = sid.clone();
+                async move {
+                    loop {
+                        let publication = participant.get_track_publication(&sid);
+                        if let Some(publication) = publication {
+                            return publication;
+                        }
+
+                        sleep(Duration::from_millis(50)).await;
+                    }
+                }
+            };
+
+            if let Ok(remote_publication) = timeout(ADD_TRACK_TIMEOUT, wait_publication).await {
+                let track = match remote_publication.kind() {
+                    TrackKind::Audio => {
+                        if let MediaStreamTrackHandle::Audio(rtc_track) = media_track {
+                            let audio_track = RemoteAudioTrack::new(
+                                remote_publication.sid().into(),
+                                remote_publication.name(),
+                                rtc_track,
+                            );
+                            RemoteTrackHandle::Audio(Arc::new(audio_track))
+                        } else {
+                            unreachable!();
+                        }
+                    }
+                    TrackKind::Video => {
+                        if let MediaStreamTrackHandle::Video(rtc_track) = media_track {
+                            let video_track = RemoteVideoTrack::new(
+                                remote_publication.sid().into(),
+                                remote_publication.name(),
+                                rtc_track,
+                            );
+                            RemoteTrackHandle::Video(Arc::new(video_track))
+                        } else {
+                            unreachable!()
+                        }
+                    }
+                    _ => unreachable!(),
+                };
+
+                info!("starting track: {:?}", sid);
+
+                remote_publication.update_track(Some(track.clone().into()));
+                self.shared
+                    .add_track_publication(TrackPublication::Remote(remote_publication.clone()));
+                track.start();
+
+                let event = TrackSubscribedEvent {
+                    track,
+                    publication: remote_publication,
+                    participant: self.clone(),
+                };
+
+                if let Some(cb) = self
+                    .shared
+                    .internal_events
+                    .on_track_subscribed
+                    .lock()
+                    .as_mut()
+                {
+                    cb(event.clone()).await;
+                }
+
+                if let Some(cb) = self.shared.events.on_track_subscribed.lock().as_mut() {
+                    cb(event).await;
+                }
+            } else {
+                error!("could not find published track with sid: {:?}", sid);
+
+                let event = TrackSubscriptionFailedEvent {
+                    sid: sid.clone(),
+                    error: TrackError::TrackNotFound(sid.clone().to_string()),
+                    participant: self.clone(),
+                };
+
+                if let Some(cb) = self
+                    .shared
+                    .internal_events
+                    .on_track_subscription_failed
+                    .lock()
+                    .as_mut()
+                {
+                    cb(event.clone()).await;
+                }
+
+                if let Some(cb) = self
+                    .shared
+                    .events
+                    .on_track_subscription_failed
+                    .lock()
+                    .as_mut()
+                {
+                    cb(event).await;
+                }
+            }
+        });
+    }
+
+    fn get_track_publication(&self, sid: &TrackSid) -> Option<RemoteTrackPublication> {
+        self.shared.tracks.read().get(sid).map(|track| {
+            if let TrackPublication::Remote(remote) = track {
+                remote.clone()
+            } else {
+                unreachable!()
+            }
+        })
+    }
+
+    pub(crate) async fn update_info(self: Arc<Self>, info: ParticipantInfo) {
+        self.shared.update_info(info.clone());
+
+        let mut valid_tracks = HashSet::<TrackSid>::new();
+
+        for track in info.tracks {
+            if let Some(publication) = self.get_track_publication(&track.sid.clone().into()) {
+                publication.update_info(track.clone());
+            } else {
+                let publication = RemoteTrackPublication::new(track.clone(), self.sid(), None);
+                self.shared
+                    .add_track_publication(TrackPublication::Remote(publication.clone()));
+
+                // This is a new track, fire publish events
+                let event = TrackPublishedEvent {
+                    participant: self.clone(),
+                    publication: publication.clone(),
+                };
+
+                if let Some(cb) = self
+                    .shared
+                    .internal_events
+                    .on_track_published
+                    .lock()
+                    .as_mut()
+                {
+                    cb(event.clone()).await;
+                }
+
+                if let Some(cb) = self.shared.events.on_track_published.lock().as_mut() {
+                    cb(event).await;
+                }
+            }
+
+            valid_tracks.insert(track.sid.into());
+        }
+    }
+}
+
+impl ParticipantInternalTrait for RemoteParticipant {
+    fn internal_events(&self) -> Arc<ParticipantEvents> {
+        self.shared.internal_events.clone()
+    }
+}
+
+impl_participant_trait!(RemoteParticipant);

crates/livekit-core/src/room/publication/mod.rs

@@ -0,0 +1,186 @@
+use crate::proto::TrackType;
+use crate::proto::{TrackInfo, TrackSource as ProtoTrackSource};
+use crate::room::id::ParticipantSid;
+use crate::room::id::TrackSid;
+use crate::room::track::local_track::LocalTrackHandle;
+use crate::room::track::remote_track::RemoteTrackHandle;
+use crate::room::track::{TrackHandle, TrackKind, TrackSource};
+use livekit_utils::enum_dispatch;
+use parking_lot::Mutex;
+use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
+use std::sync::Arc;
+
+use super::track::TrackDimension;
+
+pub(crate) trait TrackPublicationInternalTrait {
+    fn update_track(&self, track: Option<TrackHandle>);
+    fn update_info(&self, info: TrackInfo);
+}
+
+pub trait TrackPublicationTrait {
+    fn name(&self) -> String;
+    fn sid(&self) -> TrackSid;
+    fn kind(&self) -> TrackKind;
+    fn source(&self) -> TrackSource;
+    fn simulcasted(&self) -> bool;
+}
+
+pub(super) struct TrackPublicationShared {
+    pub(super) track: Mutex<Option<TrackHandle>>,
+    pub(super) name: Mutex<String>,
+    pub(super) sid: Mutex<TrackSid>,
+    pub(super) kind: AtomicU8,   // Casted to TrackKind
+    pub(super) source: AtomicU8, // Casted to TrackSource
+    pub(super) simulcasted: AtomicBool,
+    pub(super) dimension: Mutex<TrackDimension>,
+    pub(super) mime_type: Mutex<String>,
+    pub(super) participant: ParticipantSid, // TODO(theomonnom) Use WeakParticipant instead
+}
+
+impl TrackPublicationShared {
+    pub fn new(
+        info: TrackInfo,
+        participant: ParticipantSid,
+        track: Option<TrackHandle>,
+    ) -> Arc<Self> {
+        Arc::new(Self {
+            track: Mutex::new(track),
+            name: Mutex::new(info.name),
+            sid: Mutex::new(info.sid.into()),
+            kind: AtomicU8::new(TrackKind::from(TrackType::from_i32(info.r#type).unwrap()) as u8),
+            source: AtomicU8::new(TrackSource::from(
+                ProtoTrackSource::from_i32(info.source).unwrap(),
+            ) as u8),
+            simulcasted: AtomicBool::new(info.simulcast),
+            dimension: Mutex::new(TrackDimension(info.width, info.height)),
+            mime_type: Mutex::new(info.mime_type),
+            participant,
+        })
+    }
+
+    pub fn update_info(&self, info: TrackInfo) {
+        *self.name.lock() = info.name;
+        *self.sid.lock() = info.sid.into();
+        self.kind.store(
+            TrackKind::from(TrackType::from_i32(info.r#type).unwrap()) as u8,
+            Ordering::SeqCst,
+        );
+        self.source.store(
+            TrackSource::from(ProtoTrackSource::from_i32(info.source).unwrap()) as u8,
+            Ordering::SeqCst,
+        );
+        self.simulcasted.store(info.simulcast, Ordering::SeqCst);
+        *self.dimension.lock() = TrackDimension(info.width, info.height);
+        *self.mime_type.lock() = info.mime_type;
+    }
+}
+
+#[derive(Clone)]
+pub enum TrackPublication {
+    Local(LocalTrackPublication),
+    Remote(RemoteTrackPublication),
+}
+
+impl TrackPublication {
+    pub fn track(&self) -> Option<TrackHandle> {
+        // Not calling Local/Remote function here, we don't need "cast"
+        match self {
+            TrackPublication::Local(p) => p.shared.track.lock().clone(),
+            TrackPublication::Remote(p) => p.shared.track.lock().clone(),
+        }
+    }
+}
+
+impl TrackPublicationInternalTrait for TrackPublication {
+    enum_dispatch!(
+        [Local, Remote]
+        fnc!(update_track, &Self, [track: Option<TrackHandle>], ());
+        fnc!(update_info, &Self, [info: TrackInfo], ());
+    );
+}
+
+impl TrackPublicationTrait for TrackPublication {
+    enum_dispatch!(
+        [Local, Remote]
+        fnc!(sid, &Self, [], TrackSid);
+        fnc!(name, &Self, [], String);
+        fnc!(kind, &Self, [], TrackKind);
+        fnc!(source, &Self, [], TrackSource);
+        fnc!(simulcasted, &Self, [], bool);
+    );
+}
+
+macro_rules! impl_publication_trait {
+    ($x:ident) => {
+        impl TrackPublicationInternalTrait for $x {
+            fn update_track(&self, track: Option<TrackHandle>) {
+                *self.shared.track.lock() = track;
+            }
+
+            fn update_info(&self, info: TrackInfo) {
+                self.shared.update_info(info);
+            }
+        }
+
+        impl TrackPublicationTrait for $x {
+            fn name(&self) -> String {
+                self.shared.name.lock().clone()
+            }
+
+            fn sid(&self) -> TrackSid {
+                self.shared.sid.lock().clone()
+            }
+
+            fn kind(&self) -> TrackKind {
+                self.shared.kind.load(Ordering::SeqCst).into()
+            }
+
+            fn source(&self) -> TrackSource {
+                self.shared.source.load(Ordering::SeqCst).into()
+            }
+
+            fn simulcasted(&self) -> bool {
+                self.shared.simulcasted.load(Ordering::SeqCst)
+            }
+        }
+    };
+}
+
+#[derive(Clone)]
+pub struct LocalTrackPublication {
+    shared: Arc<TrackPublicationShared>,
+}
+
+impl LocalTrackPublication {
+    pub fn track(&self) -> Option<LocalTrackHandle> {
+        self.shared
+            .track
+            .lock()
+            .clone()
+            .map(|local_track| local_track.try_into().unwrap())
+    }
+}
+
+#[derive(Clone)]
+pub struct RemoteTrackPublication {
+    shared: Arc<TrackPublicationShared>,
+}
+
+impl RemoteTrackPublication {
+    pub fn new(info: TrackInfo, participant: ParticipantSid, track: Option<TrackHandle>) -> Self {
+        Self {
+            shared: TrackPublicationShared::new(info, participant, track),
+        }
+    }
+
+    pub fn track(&self) -> Option<RemoteTrackHandle> {
+        self.shared
+            .track
+            .lock()
+            .clone()
+            .map(|track| track.try_into().unwrap())
+    }
+}
+
+impl_publication_trait!(LocalTrackPublication);
+impl_publication_trait!(RemoteTrackPublication);

crates/livekit-core/src/room/track/audio_track.rs

@@ -0,0 +1,31 @@
+use crate::room::track::local_audio_track::LocalAudioTrack;
+use crate::room::track::remote_audio_track::RemoteAudioTrack;
+use crate::room::track::TrackHandle;
+use std::sync::Arc;
+
+#[derive(Clone)]
+pub enum AudioTrackHandle {
+    Local(Arc<LocalAudioTrack>),
+    Remote(Arc<RemoteAudioTrack>),
+}
+
+impl From<AudioTrackHandle> for TrackHandle {
+    fn from(audio_track: AudioTrackHandle) -> Self {
+        match audio_track {
+            AudioTrackHandle::Local(local_audio) => Self::LocalAudio(local_audio),
+            AudioTrackHandle::Remote(remote_audio) => Self::RemoteAudio(remote_audio),
+        }
+    }
+}
+
+impl TryFrom<TrackHandle> for AudioTrackHandle {
+    type Error = &'static str;
+
+    fn try_from(track: TrackHandle) -> Result<Self, Self::Error> {
+        match track {
+            TrackHandle::LocalAudio(local_audio) => Ok(Self::Local(local_audio)),
+            TrackHandle::RemoteAudio(remote_audio) => Ok(Self::Remote(remote_audio)),
+            _ => Err("not a audio track"),
+        }
+    }
+}

crates/livekit-core/src/room/track/events.rs

@@ -0,0 +1 @@
+pub struct TrackEvents {}

crates/livekit-core/src/room/track/local_audio_track.rs

@@ -0,0 +1,7 @@
+use crate::room::track::{impl_track_trait, TrackShared};
+
+pub struct LocalAudioTrack {
+    shared: TrackShared,
+}
+
+impl_track_trait!(LocalAudioTrack);

crates/livekit-core/src/room/track/local_track.rs

@@ -0,0 +1,31 @@
+use crate::room::track::local_audio_track::LocalAudioTrack;
+use crate::room::track::local_video_track::LocalVideoTrack;
+use crate::room::track::TrackHandle;
+use std::sync::Arc;
+
+#[derive(Clone)]
+pub enum LocalTrackHandle {
+    Audio(Arc<LocalAudioTrack>),
+    Video(Arc<LocalVideoTrack>),
+}
+
+impl From<LocalTrackHandle> for TrackHandle {
+    fn from(local_track: LocalTrackHandle) -> Self {
+        match local_track {
+            LocalTrackHandle::Audio(local_audio) => Self::LocalAudio(local_audio),
+            LocalTrackHandle::Video(local_video) => Self::LocalVideo(local_video),
+        }
+    }
+}
+
+impl TryFrom<TrackHandle> for LocalTrackHandle {
+    type Error = &'static str;
+
+    fn try_from(track: TrackHandle) -> Result<Self, Self::Error> {
+        match track {
+            TrackHandle::LocalAudio(local_audio) => Ok(Self::Audio(local_audio)),
+            TrackHandle::LocalVideo(local_video) => Ok(Self::Video(local_video)),
+            _ => Err("not a local track"),
+        }
+    }
+}

crates/livekit-core/src/room/track/local_video_track.rs

@@ -0,0 +1,7 @@
+use crate::room::track::{impl_track_trait, TrackShared};
+
+pub struct LocalVideoTrack {
+    shared: TrackShared,
+}
+
+impl_track_trait!(LocalVideoTrack);

crates/livekit-core/src/room/track/mod.rs

@@ -0,0 +1,212 @@
+use crate::proto::{TrackSource as ProtoTrackSource, TrackType};
+use crate::room::id::TrackSid;
+use crate::room::track::local_audio_track::LocalAudioTrack;
+use crate::room::track::local_video_track::LocalVideoTrack;
+use crate::room::track::remote_audio_track::RemoteAudioTrack;
+use crate::room::track::remote_video_track::RemoteVideoTrack;
+use livekit_utils::enum_dispatch;
+use livekit_webrtc::media_stream::{MediaStreamTrackHandle, MediaStreamTrackTrait};
+use parking_lot::Mutex;
+use std::sync::atomic::AtomicU8;
+use std::sync::Arc;
+
+pub mod audio_track;
+pub mod events;
+pub mod local_audio_track;
+pub mod local_track;
+pub mod local_video_track;
+pub mod remote_audio_track;
+pub mod remote_track;
+pub mod remote_video_track;
+pub mod video_track;
+
+#[derive(Debug)]
+pub enum TrackKind {
+    Unknown,
+    Audio,
+    Video,
+}
+
+impl From<u8> for TrackKind {
+    fn from(val: u8) -> Self {
+        match val {
+            1 => Self::Audio,
+            2 => Self::Video,
+            _ => Self::Unknown,
+        }
+    }
+}
+
+impl From<TrackType> for TrackKind {
+    fn from(r#type: TrackType) -> Self {
+        match r#type {
+            TrackType::Audio => Self::Audio,
+            TrackType::Video => Self::Video,
+            _ => Self::Unknown,
+        }
+    }
+}
+
+#[derive(Debug)]
+pub enum StreamState {
+    Unknown,
+    Active,
+    Paused,
+}
+
+impl From<u8> for StreamState {
+    fn from(val: u8) -> Self {
+        match val {
+            1 => Self::Active,
+            2 => Self::Paused,
+            _ => Self::Unknown,
+        }
+    }
+}
+
+#[derive(Debug)]
+pub enum TrackSource {
+    Unknown,
+    Camera,
+    Microphone,
+    Screenshare,
+    ScreenshareAudio,
+}
+
+impl From<u8> for TrackSource {
+    fn from(val: u8) -> Self {
+        match val {
+            1 => Self::Camera,
+            2 => Self::Microphone,
+            3 => Self::Screenshare,
+            4 => Self::ScreenshareAudio,
+            _ => Self::Unknown,
+        }
+    }
+}
+
+impl From<ProtoTrackSource> for TrackSource {
+    fn from(source: ProtoTrackSource) -> Self {
+        match source {
+            ProtoTrackSource::Camera => Self::Camera,
+            ProtoTrackSource::Microphone => Self::Microphone,
+            ProtoTrackSource::ScreenShare => Self::Screenshare,
+            ProtoTrackSource::ScreenShareAudio => Self::ScreenshareAudio,
+            ProtoTrackSource::Unknown => Self::Unknown,
+        }
+    }
+}
+
+pub struct TrackDimension(pub u32, pub u32);
+
+pub trait TrackTrait {
+    fn sid(&self) -> TrackSid;
+    fn name(&self) -> String;
+    fn kind(&self) -> TrackKind;
+    fn stream_state(&self) -> StreamState;
+    fn start(&self);
+    fn stop(&self);
+}
+
+pub(super) struct TrackShared {
+    pub(super) sid: Mutex<TrackSid>,
+    pub(super) name: Mutex<String>,
+    pub(super) kind: AtomicU8,         // TrackKind
+    pub(super) stream_state: AtomicU8, // StreamState
+    pub(super) rtc_track: MediaStreamTrackHandle,
+}
+
+impl TrackShared {
+    pub(crate) fn new(
+        sid: TrackSid,
+        name: String,
+        kind: TrackKind,
+        rtc_track: MediaStreamTrackHandle,
+    ) -> Self {
+        Self {
+            sid: Mutex::new(sid),
+            name: Mutex::new(name),
+            kind: AtomicU8::new(kind as u8),
+            stream_state: AtomicU8::new(StreamState::Active as u8),
+            rtc_track,
+        }
+    }
+
+    pub(crate) fn start(&self) {
+        self.rtc_track.set_enabled(true);
+    }
+
+    pub(crate) fn stop(&self) {
+        self.rtc_track.set_enabled(false);
+    }
+}
+
+#[derive(Clone)]
+pub enum TrackHandle {
+    LocalVideo(Arc<LocalVideoTrack>),
+    LocalAudio(Arc<LocalAudioTrack>),
+    RemoteVideo(Arc<RemoteVideoTrack>),
+    RemoteAudio(Arc<RemoteAudioTrack>),
+}
+
+impl TrackTrait for TrackHandle {
+    enum_dispatch!(
+        [LocalVideo, LocalAudio, RemoteVideo, RemoteAudio]
+        fnc!(sid, &Self, [], TrackSid);
+        fnc!(name, &Self, [], String);
+        fnc!(kind, &Self, [], TrackKind);
+        fnc!(stream_state, &Self, [], StreamState);
+        fnc!(start, &Self, [], ());
+        fnc!(stop, &Self, [], ());
+    );
+}
+
+impl TrackHandle {
+    pub fn rtc_track(&self) -> MediaStreamTrackHandle {
+        match self {
+            Self::RemoteVideo(remote_video) => {
+                MediaStreamTrackHandle::Video(remote_video.rtc_track())
+            }
+            Self::RemoteAudio(remote_audio) => {
+                MediaStreamTrackHandle::Audio(remote_audio.rtc_track())
+            }
+            _ => todo!(),
+        }
+    }
+}
+
+macro_rules! impl_track_trait {
+    ($x:ident) => {
+        use crate::room::id::TrackSid;
+        use crate::room::track::{StreamState, TrackKind, TrackTrait};
+        use std::sync::atomic::Ordering;
+
+        impl TrackTrait for $x {
+            fn sid(&self) -> TrackSid {
+                self.shared.sid.lock().clone()
+            }
+
+            fn name(&self) -> String {
+                self.shared.name.lock().clone()
+            }
+
+            fn kind(&self) -> TrackKind {
+                self.shared.kind.load(Ordering::SeqCst).into()
+            }
+
+            fn stream_state(&self) -> StreamState {
+                self.shared.stream_state.load(Ordering::SeqCst).into()
+            }
+
+            fn start(&self) {
+                self.shared.start();
+            }
+
+            fn stop(&self) {
+                self.shared.stop();
+            }
+        }
+    };
+}
+
+pub(super) use impl_track_trait;

crates/livekit-core/src/room/track/remote_audio_track.rs

@@ -0,0 +1,30 @@
+use crate::room::track::{impl_track_trait, TrackShared};
+use livekit_webrtc::media_stream::{AudioTrack, MediaStreamTrackHandle};
+use std::sync::Arc;
+
+pub struct RemoteAudioTrack {
+    shared: TrackShared,
+}
+
+impl RemoteAudioTrack {
+    pub(crate) fn new(sid: TrackSid, name: String, track: Arc<AudioTrack>) -> Self {
+        Self {
+            shared: TrackShared::new(
+                sid,
+                name,
+                TrackKind::Audio,
+                MediaStreamTrackHandle::Audio(track),
+            ),
+        }
+    }
+
+    pub fn rtc_track(&self) -> Arc<AudioTrack> {
+        if let MediaStreamTrackHandle::Audio(audio) = &self.shared.rtc_track {
+            audio.clone()
+        } else {
+            unreachable!()
+        }
+    }
+}
+
+impl_track_trait!(RemoteAudioTrack);

crates/livekit-core/src/room/track/remote_track.rs

@@ -0,0 +1,49 @@
+use std::sync::Arc;
+
+use super::{StreamState, TrackKind};
+use crate::room::id::TrackSid;
+use crate::room::track::remote_audio_track::RemoteAudioTrack;
+use crate::room::track::remote_video_track::RemoteVideoTrack;
+use crate::room::track::TrackHandle;
+use livekit_utils::enum_dispatch;
+
+use super::TrackTrait;
+
+#[derive(Clone)]
+pub enum RemoteTrackHandle {
+    Audio(Arc<RemoteAudioTrack>),
+    Video(Arc<RemoteVideoTrack>),
+}
+
+impl TrackTrait for RemoteTrackHandle {
+    enum_dispatch!(
+        [Audio, Video]
+        fnc!(sid, &Self, [], TrackSid);
+        fnc!(name, &Self, [], String);
+        fnc!(kind, &Self, [], TrackKind);
+        fnc!(stream_state, &Self, [], StreamState);
+        fnc!(start, &Self, [], ());
+        fnc!(stop, &Self, [], ());
+    );
+}
+
+impl From<RemoteTrackHandle> for TrackHandle {
+    fn from(remote_track: RemoteTrackHandle) -> Self {
+        match remote_track {
+            RemoteTrackHandle::Audio(remote_audio) => Self::RemoteAudio(remote_audio),
+            RemoteTrackHandle::Video(remote_video) => Self::RemoteVideo(remote_video),
+        }
+    }
+}
+
+impl TryFrom<TrackHandle> for RemoteTrackHandle {
+    type Error = &'static str;
+
+    fn try_from(track: TrackHandle) -> Result<Self, Self::Error> {
+        match track {
+            TrackHandle::RemoteAudio(remote_audio) => Ok(Self::Audio(remote_audio)),
+            TrackHandle::RemoteVideo(remote_video) => Ok(Self::Video(remote_video)),
+            _ => Err("not a remote track"),
+        }
+    }
+}

crates/livekit-core/src/room/track/remote_video_track.rs

@@ -0,0 +1,31 @@
+use livekit_webrtc::media_stream::{MediaStreamTrackHandle, VideoTrack};
+use std::sync::Arc;
+
+use crate::room::track::{impl_track_trait, TrackShared};
+
+pub struct RemoteVideoTrack {
+    shared: TrackShared,
+}
+
+impl RemoteVideoTrack {
+    pub(crate) fn new(sid: TrackSid, name: String, track: Arc<VideoTrack>) -> Self {
+        Self {
+            shared: TrackShared::new(
+                sid,
+                name,
+                TrackKind::Video,
+                MediaStreamTrackHandle::Video(track),
+            ),
+        }
+    }
+
+    pub fn rtc_track(&self) -> Arc<VideoTrack> {
+        if let MediaStreamTrackHandle::Video(video) = &self.shared.rtc_track {
+            video.clone()
+        } else {
+            unreachable!()
+        }
+    }
+}
+
+impl_track_trait!(RemoteVideoTrack);

crates/livekit-core/src/room/track/video_track.rs

@@ -0,0 +1,31 @@
+use crate::room::track::local_video_track::LocalVideoTrack;
+use crate::room::track::remote_video_track::RemoteVideoTrack;
+use crate::room::track::TrackHandle;
+use std::sync::Arc;
+
+#[derive(Clone)]
+pub enum VideoTrackHandle {
+    Local(Arc<LocalVideoTrack>),
+    Remote(Arc<RemoteVideoTrack>),
+}
+
+impl From<VideoTrackHandle> for TrackHandle {
+    fn from(video_track: VideoTrackHandle) -> Self {
+        match video_track {
+            VideoTrackHandle::Local(local_video) => Self::LocalVideo(local_video),
+            VideoTrackHandle::Remote(remote_video) => Self::RemoteVideo(remote_video),
+        }
+    }
+}
+
+impl TryFrom<TrackHandle> for VideoTrackHandle {
+    type Error = &'static str;
+
+    fn try_from(track: TrackHandle) -> Result<Self, Self::Error> {
+        match track {
+            TrackHandle::LocalVideo(local_video) => Ok(Self::Local(local_video)),
+            TrackHandle::RemoteVideo(remote_video) => Ok(Self::Remote(remote_video)),
+            _ => Err("not a video track"),
+        }
+    }
+}

crates/livekit-core/src/rtc_engine/engine_internal.rs

@@ -1,576 +0,0 @@
-use std::fmt::{Debug, Formatter};
-use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
-use std::sync::Arc;
-use std::time::Duration;
-
-use prost::Message;
-use serde::{Deserialize, Serialize};
-use tokio::sync::{mpsc, Mutex};
-use tokio::time;
-use tracing::{event, Level};
-
-use livekit_webrtc::data_channel::{DataChannel, DataChannelInit, DataState};
-use livekit_webrtc::jsep::{IceCandidate, SessionDescription};
-use livekit_webrtc::peer_connection::{
-    IceConnectionState, PeerConnectionState, RTCOfferAnswerOptions,
-};
-use livekit_webrtc::peer_connection_factory::{
-    ContinualGatheringPolicy, ICEServer, IceTransportsType, RTCConfiguration,
-};
-
-use crate::lk_runtime::LKRuntime;
-use crate::pc_transport::PCTransport;
-use crate::proto;
-use crate::proto::data_packet::Value;
-use crate::proto::{
-    data_packet, signal_request, signal_response, DataPacket, JoinResponse, SignalTarget,
-    TrickleRequest,
-};
-use crate::rtc_engine::{EngineError, MAX_ICE_CONNECT_TIMEOUT};
-use crate::signal_client::SignalClient;
-
-const LOSSY_DC_LABEL: &str = "_lossy";
-const RELIABLE_DC_LABEL: &str = "_reliable";
-
-// Used to communicate IceCandidate with the server
-#[derive(Serialize, Deserialize)]
-#[allow(non_snake_case)]
-struct IceCandidateJSON {
-    sdpMid: String,
-    sdpMLineIndex: i32,
-    candidate: String,
-}
-
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-pub(crate) enum PCState {
-    New,
-    Connected,
-    Disconnected,
-    Reconnecting,
-    Closed,
-}
-
-#[derive(Debug)]
-pub(crate) enum InternalMessage {
-    IceCandidate {
-        ice_candidate: IceCandidate,
-        publisher: bool,
-    },
-    ConnectionChange {
-        state: PeerConnectionState,
-        primary: bool,
-    },
-    PrimaryDataChannel {
-        data_channel: DataChannel,
-    },
-    PublisherOffer {
-        offer: SessionDescription,
-    },
-    Data {
-        data: Vec<u8>,
-        binary: bool,
-    },
-}
-
-pub(crate) struct EngineInternal {
-    pub(super) publisher_pc: Arc<Mutex<PCTransport>>,
-    pub(super) subscriber_pc: Arc<Mutex<PCTransport>>,
-    pub(super) lossy_dc: Arc<Mutex<DataChannel>>,
-    pub(super) reliable_dc: Arc<Mutex<DataChannel>>,
-    pub(super) lossy_dc_sub: Arc<Mutex<Option<DataChannel>>>,
-    pub(super) reliable_dc_sub: Arc<Mutex<Option<DataChannel>>>,
-
-    pub(super) msg_sender: mpsc::Sender<InternalMessage>,
-    pub(super) join_response: Mutex<JoinResponse>,
-    pub(super) pc_state: AtomicU8, // casted to PCState
-    pub(super) has_published: AtomicBool,
-}
-
-impl Debug for EngineInternal {
-    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
-        write!(f, "EngineInternal")
-    }
-}
-
-impl EngineInternal {
-    /// Configure the PeerConnections
-    ///
-    /// This is called on connect & on full reconnect.
-    /// Create the PeerConnections & the DataChannels.
-    /// Register listeners and send the internal messages
-    /// to the event_loop.
-    #[tracing::instrument]
-    pub(super) fn configure(
-        lk_runtime: Arc<LKRuntime>,
-        sender: mpsc::Sender<InternalMessage>,
-        join: JoinResponse,
-    ) -> Result<Self, EngineError> {
-        let rtc_config = RTCConfiguration {
-            ice_servers: {
-                let mut servers = vec![];
-                for is in join.ice_servers.clone() {
-                    servers.push(ICEServer {
-                        urls: is.urls,
-                        username: is.username,
-                        password: is.credential,
-                    })
-                }
-                servers
-            },
-            continual_gathering_policy: ContinualGatheringPolicy::GatherContinually,
-            ice_transport_type: IceTransportsType::All,
-        };
-
-        let mut publisher_pc = PCTransport::new(
-            lk_runtime
-                .pc_factory
-                .create_peer_connection(rtc_config.clone())?,
-        );
-        let mut subscriber_pc =
-            PCTransport::new(lk_runtime.pc_factory.create_peer_connection(rtc_config)?);
-
-        publisher_pc.peer_connection().on_ice_candidate(Box::new({
-            let sender = sender.clone();
-            move |ice_candidate| {
-                let _ = sender.blocking_send(InternalMessage::IceCandidate {
-                    ice_candidate,
-                    publisher: true,
-                });
-            }
-        }));
-
-        subscriber_pc.peer_connection().on_ice_candidate(Box::new({
-            let sender = sender.clone();
-            move |ice_candidate| {
-                let _ = sender.blocking_send(InternalMessage::IceCandidate {
-                    ice_candidate,
-                    publisher: false,
-                });
-            }
-        }));
-
-        publisher_pc.on_offer({
-            let sender = sender.clone();
-            Box::new(move |offer| {
-                let sender = sender.clone();
-
-                tokio::spawn(async move {
-                    let _ = sender.send(InternalMessage::PublisherOffer { offer }).await;
-                });
-
-                Box::pin(async move {})
-            })
-        });
-
-        let mut primary_pc = &mut publisher_pc;
-        let mut secondary_pc = &mut subscriber_pc;
-        if join.subscriber_primary {
-            primary_pc = &mut subscriber_pc;
-            secondary_pc = &mut publisher_pc;
-
-            primary_pc.peer_connection().on_data_channel(Box::new({
-                let sender = sender.clone();
-                move |data_channel| {
-                    let _ =
-                        sender.blocking_send(InternalMessage::PrimaryDataChannel { data_channel });
-                }
-            }));
-        }
-
-        primary_pc.peer_connection().on_connection_change(Box::new({
-            let sender = sender.clone();
-            move |state| {
-                let _ = sender.blocking_send(InternalMessage::ConnectionChange {
-                    state,
-                    primary: true,
-                });
-            }
-        }));
-
-        secondary_pc
-            .peer_connection()
-            .on_connection_change(Box::new({
-                let sender = sender.clone();
-                move |state| {
-                    let _ = sender.blocking_send(InternalMessage::ConnectionChange {
-                        state,
-                        primary: false,
-                    });
-                }
-            }));
-
-        let mut lossy_dc = publisher_pc.peer_connection().create_data_channel(
-            LOSSY_DC_LABEL,
-            DataChannelInit {
-                ordered: true,
-                max_retransmits: Some(0),
-                ..DataChannelInit::default()
-            },
-        )?;
-
-        let mut reliable_dc = publisher_pc.peer_connection().create_data_channel(
-            RELIABLE_DC_LABEL,
-            DataChannelInit {
-                ordered: true,
-                ..DataChannelInit::default()
-            },
-        )?;
-
-        Self::configure_dc(&mut lossy_dc, sender.clone());
-        Self::configure_dc(&mut reliable_dc, sender.clone());
-
-        Ok(Self {
-            publisher_pc: Arc::new(Mutex::new(publisher_pc)),
-            subscriber_pc: Arc::new(Mutex::new(subscriber_pc)),
-            lossy_dc: Arc::new(Mutex::new(lossy_dc)),
-            reliable_dc: Arc::new(Mutex::new(reliable_dc)),
-            lossy_dc_sub: Default::default(),
-            reliable_dc_sub: Default::default(),
-            msg_sender: sender,
-            join_response: Mutex::new(join),
-            pc_state: AtomicU8::new(PCState::New as u8),
-            has_published: AtomicBool::new(false),
-        })
-    }
-
-    /// Send InternalMessage when a datachannel receives data
-    #[tracing::instrument]
-    fn configure_dc(data_channel: &mut DataChannel, sender: mpsc::Sender<InternalMessage>) {
-        data_channel.on_message(Box::new(move |data, binary| {
-            let _ = sender.blocking_send(InternalMessage::Data {
-                data: data.to_vec(),
-                binary,
-            });
-        }));
-    }
-
-    /// Ensure the publisher PeerConnection is connected
-    ///
-    /// When subscriber_primary is enabled, only the subscriber PeerConnection is negotiated.
-    /// This allows for faster connection when we don't need the publisher
-    #[tracing::instrument]
-    pub(super) async fn ensure_publisher_connected(
-        self: &Arc<Self>,
-        kind: data_packet::Kind,
-    ) -> Result<(), EngineError> {
-        if !self.join_response.lock().await.subscriber_primary {
-            return Ok(());
-        }
-
-        let publisher = &self.publisher_pc;
-        {
-            let mut publisher = publisher.lock().await;
-            if !publisher.is_connected()
-                && publisher.peer_connection().ice_connection_state()
-                    != IceConnectionState::IceConnectionChecking
-            {
-                tokio::spawn({
-                    let internal = self.clone();
-                    async move {
-                        let _ = internal.negotiate_publisher().await;
-                    }
-                });
-            }
-        }
-
-        let dc = self.data_channel(kind);
-        if dc.lock().await.state() == DataState::Open {
-            return Ok(());
-        }
-
-        let res = time::timeout(MAX_ICE_CONNECT_TIMEOUT, async move {
-            let mut interval = time::interval(Duration::from_millis(50));
-
-            loop {
-                if publisher.lock().await.is_connected()
-                    && dc.lock().await.state() == DataState::Open
-                {
-                    break;
-                }
-
-                interval.tick().await;
-            }
-        })
-        .await;
-
-        if res.is_err() {
-            let err =
-                EngineError::Connection("could not establish publisher connection".to_string());
-            event!(Level::ERROR, error = ?err);
-            Err(err)
-        } else {
-            Ok(())
-        }
-    }
-
-    /// Run the event_loop of the RTCEngine
-    #[tracing::instrument]
-    pub(super) async fn run(
-        self: &Arc<Self>,
-        mut receiver: mpsc::Receiver<InternalMessage>,
-        signal_client: Arc<SignalClient>,
-    ) {
-        loop {
-            tokio::select! {
-                signal = signal_client.recv() => {
-                    match signal {
-                        Some(signal) => {
-                            if let Err(err) = self.handle_signal(signal, signal_client.clone()).await {
-                                event!(
-                                    Level::ERROR,
-                                    "failed to handle signal: {:?}",
-                                    err,
-                                );
-                            }
-                        }
-                        None => {
-                            // TODO(theomonnom) Trigger reconnect
-                        }
-                    }
-                },
-                Some(msg) = receiver.recv() => {
-                    if let Err(err) = self.handle_message(msg, signal_client.clone()).await {
-                        event!(
-                            Level::ERROR,
-                            "failed to handle engine message: {:?}",
-                            err,
-                        );
-                    }
-                },
-            }
-        }
-    }
-
-    /// Handle SignalResponse messages coming from the server
-    ///
-    /// Run the needed livekit-protocol
-    #[tracing::instrument]
-    async fn handle_signal(
-        self: &Arc<Self>,
-        signal: signal_response::Message,
-        signal_client: Arc<SignalClient>,
-    ) -> Result<(), EngineError> {
-        match signal {
-            signal_response::Message::Answer(answer) => {
-                event!(Level::TRACE, "received answer for publisher: {:?}", answer);
-                let sdp = SessionDescription::from(answer.r#type.parse().unwrap(), &answer.sdp)?;
-                self.publisher_pc
-                    .lock()
-                    .await
-                    .set_remote_description(sdp)
-                    .await?;
-            }
-            signal_response::Message::Offer(offer) => {
-                // Handle the subscriber offer & send an answer to livekit-server
-                // We always get an offer from the server when connecting
-                event!(Level::TRACE, "received offer for subscriber: {:?}", offer);
-                let sdp = SessionDescription::from(offer.r#type.parse().unwrap(), &offer.sdp)?;
-
-                self.subscriber_pc
-                    .lock()
-                    .await
-                    .set_remote_description(sdp)
-                    .await?;
-                let answer = self
-                    .subscriber_pc
-                    .lock()
-                    .await
-                    .peer_connection()
-                    .create_answer(RTCOfferAnswerOptions::default())
-                    .await?;
-                self.subscriber_pc
-                    .lock()
-                    .await
-                    .peer_connection()
-                    .set_local_description(answer.clone())
-                    .await?;
-
-                tokio::spawn(async move {
-                    let _ = signal_client
-                        .send(signal_request::Message::Answer(proto::SessionDescription {
-                            r#type: "answer".to_string(),
-                            sdp: answer.to_string(),
-                        }))
-                        .await;
-                });
-            }
-            signal_response::Message::Trickle(trickle) => {
-                // Add the IceCandidate received from the livekit-server
-                let json: IceCandidateJSON = serde_json::from_str(&trickle.candidate_init)?;
-                let ice = IceCandidate::from(&json.sdpMid, json.sdpMLineIndex, &json.candidate)?;
-
-                event!(
-                    Level::TRACE,
-                    "received ice_candidate ({:?}) - {:?}",
-                    SignalTarget::from_i32(trickle.target).unwrap(),
-                    ice
-                );
-
-                if trickle.target == SignalTarget::Publisher as i32 {
-                    self.publisher_pc
-                        .lock()
-                        .await
-                        .add_ice_candidate(ice)
-                        .await?;
-                } else {
-                    self.subscriber_pc
-                        .lock()
-                        .await
-                        .add_ice_candidate(ice)
-                        .await?;
-                }
-            }
-            _ => {}
-        }
-
-        Ok(())
-    }
-
-    /// Handle libwebrtc messages
-    ///
-    /// Every message used inside this function comes from libwebrtc.
-    /// The messages are received in [EngineInternal](#run)
-    /// We're not handling the messages inside the signaling_thread, to return
-    /// as quickly as possible.
-    #[tracing::instrument]
-    async fn handle_message(
-        self: &Arc<Self>,
-        msg: InternalMessage,
-        signal_client: Arc<SignalClient>,
-    ) -> Result<(), EngineError> {
-        match msg {
-            InternalMessage::IceCandidate {
-                ice_candidate,
-                publisher,
-            } => {
-                // Send the IceCandidate to livekit-server
-                // Note that ContinualGatheringPolicy is set to GatherContinually
-                let json = serde_json::to_string(&IceCandidateJSON {
-                    sdpMid: ice_candidate.sdp_mid(),
-                    sdpMLineIndex: ice_candidate.sdp_mline_index(),
-                    candidate: ice_candidate.candidate(),
-                })?;
-
-                let target = if publisher {
-                    SignalTarget::Publisher
-                } else {
-                    SignalTarget::Subscriber
-                };
-
-                event!(
-                    Level::TRACE,
-                    "sending ice_candidate ({:?}) - {:?}",
-                    target,
-                    ice_candidate
-                );
-
-                tokio::spawn(async move {
-                    let _ = signal_client
-                        .send(signal_request::Message::Trickle(TrickleRequest {
-                            candidate_init: json,
-                            target: target as i32,
-                        }))
-                        .await;
-                });
-            }
-            InternalMessage::ConnectionChange { state, primary } => {
-                // PeerConnectionState changed
-                // Reconnect if we've been disconnected unexpectedly
-                // If connected for the first time, send OnConnect event
-                if primary && state == PeerConnectionState::Connected {
-                    let old_state = self.pc_state.load(Ordering::SeqCst);
-                    self.pc_state
-                        .store(PCState::Connected as u8, Ordering::SeqCst);
-
-                    if old_state == PCState::New as u8 {
-                        // TODO(theomonnom) OnConnected
-                    }
-                } else if state == PeerConnectionState::Failed {
-                    self.pc_state
-                        .store(PCState::Disconnected as u8, Ordering::SeqCst);
-
-                    // TODO(theomonnom) handle Disconnect
-                }
-            }
-            InternalMessage::PrimaryDataChannel { mut data_channel } => {
-                // Received datachannel from the primary PeerConnection.
-                // If subscriber_primary is enabled, the datachannel is used for downstream data
-                let reliable = data_channel.label() == RELIABLE_DC_LABEL;
-                Self::configure_dc(&mut data_channel, self.msg_sender.clone());
-
-                event!(
-                    Level::TRACE,
-                    "received primary data_channel - {:?}",
-                    data_channel
-                );
-
-                if reliable {
-                    *self.reliable_dc_sub.lock().await = Some(data_channel);
-                } else {
-                    *self.lossy_dc_sub.lock().await = Some(data_channel);
-                }
-            }
-            InternalMessage::PublisherOffer { offer } => {
-                // Send the publisher offer to livekit-server
-                event!(Level::TRACE, "sending publisher offer - {:?}", offer);
-
-                tokio::spawn(async move {
-                    let _ = signal_client
-                        .send(signal_request::Message::Offer(proto::SessionDescription {
-                            r#type: "offer".to_string(),
-                            sdp: offer.to_string(),
-                        }))
-                        .await;
-                });
-            }
-            InternalMessage::Data { data, binary } => {
-                // Received data from a datachannel
-                // If this is a Speaker DataPacket, update the active speakers
-                // Send SpeakersChanged/OnData event
-                if !binary {
-                    return Err(EngineError::Internal(
-                        "text messages aren't supported".to_string(),
-                    ));
-                }
-
-                let data = DataPacket::decode(&*data)?;
-                match data.value.unwrap() {
-                    Value::User(user) => {
-                        /*let mut handler = self.on_data_handler.lock().await;
-                        if let Some(f) = &mut *handler {
-                            f(Packet {
-                                data: user,
-                                kind: data_packet::Kind::from_i32(data.kind).unwrap(),
-                            })
-                                .await;
-                        }*/
-                    }
-                    Value::Speaker(_) => {
-                        // TODO(theomonnonm)
-                    }
-                }
-            }
-        }
-
-        Ok(())
-    }
-
-    #[tracing::instrument]
-    async fn negotiate_publisher(self: &Arc<Self>) -> Result<(), EngineError> {
-        self.has_published.store(true, Ordering::SeqCst);
-        if let Err(err) = self.publisher_pc.lock().await.negotiate().await {
-            event!(Level::ERROR, "failed to negotiate the publisher: {:?}", err,);
-            Err(EngineError::Rtc(err))
-        } else {
-            Ok(())
-        }
-    }
-
-    pub(super) fn data_channel(&self, kind: data_packet::Kind) -> Arc<Mutex<DataChannel>> {
-        if kind == data_packet::Kind::Reliable {
-            self.reliable_dc.clone()
-        } else {
-            self.lossy_dc.clone()
-        }
-    }
-}

crates/livekit-core/src/rtc_engine/mod.rs

@@ -1,29 +1,71 @@
-use crate::lk_runtime::LKRuntime;
-use crate::proto::{data_packet, signal_response, DataPacket, JoinResponse, UserPacket};
-use crate::rtc_engine::engine_internal::EngineInternal;
-use crate::signal_client::{SignalClient, SignalError, SignalEvent, SignalOptions};
-use futures_util::{FutureExt, StreamExt};
-use lazy_static::lazy_static;
-use livekit_webrtc::data_channel::DataSendError;
-use livekit_webrtc::jsep::SdpParseError;
-use livekit_webrtc::rtc_error::RTCError;
-use prost::Message;
+use parking_lot::Mutex;
+use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
 use std::sync::{Arc, Weak};
 use std::time::Duration;
-use thiserror::Error;
-use tokio::sync::{mpsc, Mutex};
-use tokio::time;
-use tracing::{event, Level};
 
-mod engine_internal;
+use tokio::sync::{mpsc, Mutex as AsyncMutex};
+
+use lazy_static::lazy_static;
+use prost::Message;
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+use tokio::time::sleep;
+use tracing::{debug, error, info, trace};
+
+use crate::{proto, signal_client};
+use livekit_webrtc::data_channel::{DataChannel, DataChannelInit, DataSendError, DataState};
+use livekit_webrtc::jsep::{IceCandidate, SdpParseError, SessionDescription};
+use livekit_webrtc::media_stream::MediaStream;
+use livekit_webrtc::peer_connection::{
+    IceConnectionState, PeerConnectionState, RTCOfferAnswerOptions,
+};
+use livekit_webrtc::peer_connection_factory::RTCConfiguration;
+use livekit_webrtc::rtc_error::RTCError;
+use livekit_webrtc::rtp_receiver::RtpReceiver;
+
+use crate::proto::data_packet::Value;
+use crate::proto::{
+    data_packet, signal_request, signal_response, DataPacket, JoinResponse, ParticipantUpdate,
+    SignalTarget, TrickleRequest,
+};
+use crate::rtc_engine::lk_runtime::LKRuntime;
+use crate::rtc_engine::pc_transport::PCTransport;
+use crate::rtc_engine::rtc_events::{RTCEmitter, RTCEvent, RTCEvents};
+use crate::signal_client::{SignalClient, SignalError, SignalEvent, SignalEvents, SignalOptions};
+
+mod lk_runtime;
+mod pc_transport;
+mod rtc_events;
 
 lazy_static! {
     // Share one LKRuntime across all RTCEngine instances
     static ref LK_RUNTIME: Mutex<Weak<LKRuntime>> = Mutex::new(Weak::new());
 }
 
+pub(crate) type EngineEmitter = mpsc::Sender<EngineEvent>;
+pub(crate) type EngineEvents = mpsc::Receiver<EngineEvent>;
+pub(crate) type EngineResult<T> = Result<T, EngineError>;
+
 pub(crate) const MAX_ICE_CONNECT_TIMEOUT: Duration = Duration::from_secs(15);
-pub(crate) const JOIN_RESPONSE_TIMEOUT: Duration = Duration::from_secs(5);
+pub(crate) const LOSSY_DC_LABEL: &str = "_lossy";
+pub(crate) const RELIABLE_DC_LABEL: &str = "_reliable";
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub(crate) enum PCState {
+    New,
+    Connected,
+    Disconnected,
+    Reconnecting,
+    Closed,
+}
+
+#[derive(Serialize, Deserialize)]
+#[allow(non_snake_case)]
+struct IceCandidateJSON {
+    sdpMid: String,
+    sdpMLineIndex: i32,
+    candidate: String,
+}
 
 #[derive(Error, Debug)]
 pub enum EngineError {
@@ -46,108 +88,545 @@ pub enum EngineError {
 }
 
 #[derive(Debug)]
-pub struct Packet {
-    pub data: UserPacket,
-    pub kind: data_packet::Kind,
+pub(crate) enum EngineEvent {
+    ParticipantUpdate(ParticipantUpdate),
+    AddTrack {
+        rtp_receiver: RtpReceiver,
+        streams: Vec<MediaStream>,
+    },
 }
 
 #[derive(Debug)]
-pub enum EngineEvent {
-    DataReceived(Packet),
+struct EngineInner {
+    has_published: AtomicBool,
+    join_response: Mutex<JoinResponse>,
+    pc_state: AtomicU8, // Casted to PCState enum
+
+    publisher_pc: AsyncMutex<PCTransport>,
+    subscriber_pc: AsyncMutex<PCTransport>,
+
+    // Publisher data channels
+    // Used to send data to other participants ( The SFU forward the messages )
+    lossy_dc: Mutex<DataChannel>,
+    reliable_dc: Mutex<DataChannel>,
+
+    // Subscriber data channels
+    // These fields are never used, we just keep a strong reference to them,
+    // so we can receive data from other participants
+    sub_reliable_dc: Mutex<Option<DataChannel>>,
+    sub_lossy_dc: Mutex<Option<DataChannel>>,
 }
 
 #[derive(Debug)]
 pub struct RTCEngine {
     signal_client: Arc<SignalClient>,
-    internal: Arc<EngineInternal>,
+    engine_inner: Arc<EngineInner>,
 
     #[allow(unused)]
     lk_runtime: Arc<LKRuntime>, // Keep a reference while we're using the RTCEngine
 }
 
-#[tracing::instrument(skip(url, token))]
-pub async fn connect(
-    url: &str,
-    token: &str,
-    options: SignalOptions,
-) -> Result<RTCEngine, EngineError> {
-    // Acquire an existing/a new LKRuntime
-    let mut lk_runtime_ref = LK_RUNTIME.lock().await;
-    let mut lk_runtime = lk_runtime_ref.upgrade();
+impl RTCEngine {
+    #[tracing::instrument(skip(url, token))]
+    pub(crate) async fn connect(
+        url: &str,
+        token: &str,
+        options: SignalOptions,
+    ) -> EngineResult<(RTCEngine, EngineEvents)> {
+        let mut lk_runtime = None;
+        {
+            let mut lk_runtime_ref = LK_RUNTIME.lock();
+            lk_runtime = lk_runtime_ref.upgrade();
 
-    if lk_runtime.is_none() {
-        let new_runtime = Arc::new(LKRuntime::new());
-        *lk_runtime_ref = Arc::downgrade(&new_runtime);
-        lk_runtime = Some(new_runtime);
-    }
-    let lk_runtime = lk_runtime.unwrap();
-    let (signal_client, mut signal_events) = SignalClient::connect(url, token, options).await?;
-    let signal_client = Arc::new(signal_client);
-
-    let join_response = time::timeout(JOIN_RESPONSE_TIMEOUT, async move {
-        while let Some(event) = signal_events.next().await {
-            match event {
-                SignalEvent::Signal(signal_response::Message::Join(join)) => return join,
-                _ => {
-                    // Should we try a reconnect on close here?
-                    continue;
-                }
+            if lk_runtime.is_none() {
+                let new_runtime = Arc::new(LKRuntime::new());
+                *lk_runtime_ref = Arc::downgrade(&new_runtime);
+                lk_runtime = Some(new_runtime);
             }
         }
 
-        unreachable!();
-    })
-    .await
-    .map_err(|_| EngineError::Internal("failed to receive JoinResponse".to_string()))?;
+        let lk_runtime = lk_runtime.unwrap();
+        let (signal_client, mut signal_events) = SignalClient::connect(url, token, options).await?;
 
-    event!(Level::DEBUG, "received JoinResponse: {:?}", join_response);
+        let join_response = signal_client::utils::next_join_response(&mut signal_events).await?;
+        debug!("received JoinResponse: {:?}", join_response);
 
-    let (sender, receiver) = mpsc::channel(8);
-    let internal = Arc::new(EngineInternal::configure(
-        lk_runtime.clone(),
-        sender,
-        join_response.clone(),
-    )?);
+        let (engine_inner, rtc_events) =
+            Self::configure_engine(lk_runtime.clone(), join_response.clone())?;
+        let engine_inner = Arc::new(engine_inner);
+        let signal_client = Arc::new(signal_client);
 
-    if !join_response.subscriber_primary {
-        internal.publisher_pc.lock().await.negotiate().await?;
+        let (emitter, events) = mpsc::channel(8);
+
+        tokio::spawn(Self::signal_task(
+            signal_client.clone(),
+            engine_inner.clone(),
+            signal_events,
+            emitter.clone(),
+        ));
+
+        tokio::spawn(Self::engine_task(
+            signal_client.clone(),
+            engine_inner.clone(),
+            rtc_events,
+            emitter.clone(),
+        ));
+
+        let rtc_engine = Self {
+            signal_client,
+            engine_inner,
+            lk_runtime,
+        };
+
+        if !join_response.subscriber_primary {
+            rtc_engine.negotiate_publisher().await?;
+        }
+
+        Ok((rtc_engine, events))
     }
 
-    tokio::spawn({
-        let signal_client = signal_client.clone();
-        let internal = internal.clone();
-
-        async move {
-            internal.run(receiver, signal_client).await;
-        }
-    });
-
-    Ok(RTCEngine {
-        lk_runtime,
-        signal_client,
-        internal,
-    })
-}
-
-impl RTCEngine {
-    /// Send data to other participants in the Room
     #[tracing::instrument]
     pub async fn publish_data(
-        &mut self,
+        &self,
         data: &DataPacket,
         kind: data_packet::Kind,
     ) -> Result<(), EngineError> {
-        self.internal.ensure_publisher_connected(kind).await?;
-        self.internal
-            .data_channel(kind)
+        self.ensure_publisher_connected(kind).await?;
+        self.data_channel(kind)
             .lock()
-            .await
             .send(&data.encode_to_vec(), true)
             .map_err(Into::into)
     }
 
-    /// Return the last received JoinResponse
-    pub async fn join_response(&self) -> JoinResponse {
-        self.internal.join_response.lock().await.clone()
+    pub fn join_response(&self) -> JoinResponse {
+        self.engine_inner.join_response.lock().clone()
+    }
+
+    async fn engine_task(
+        signal_client: Arc<SignalClient>,
+        engine_inner: Arc<EngineInner>,
+        mut rtc_events: RTCEvents,
+        emitter: EngineEmitter,
+    ) {
+        while let Some(event) = rtc_events.recv().await {
+            if let Err(err) = Self::handle_rtc(
+                event,
+                signal_client.clone(),
+                engine_inner.clone(),
+                emitter.clone(),
+            )
+            .await
+            {
+                error!("failed to handle rtc event: {:?}", err);
+            }
+        }
+    }
+
+    async fn signal_task(
+        signal_client: Arc<SignalClient>,
+        engine_inner: Arc<EngineInner>,
+        mut signal_events: SignalEvents,
+        emitter: EngineEmitter,
+    ) {
+        while let Some(signal) = signal_events.recv().await {
+            match signal {
+                SignalEvent::Open => {}
+                SignalEvent::Signal(signal) => {
+                    if let Err(err) = Self::handle_signal(
+                        signal,
+                        signal_client.clone(),
+                        engine_inner.clone(),
+                        emitter.clone(),
+                    )
+                    .await
+                    {
+                        error!("failed to handle signal: {:?}", err);
+                    }
+                }
+                SignalEvent::Close => {
+                    // Try reconnect if this isn't expected
+                }
+            }
+        }
+    }
+
+    async fn handle_rtc(
+        event: RTCEvent,
+        signal_client: Arc<SignalClient>,
+        engine_inner: Arc<EngineInner>,
+        emitter: EngineEmitter,
+    ) -> EngineResult<()> {
+        match event {
+            RTCEvent::IceCandidate {
+                ice_candidate,
+                target,
+            } => {
+                let json = serde_json::to_string(&IceCandidateJSON {
+                    sdpMid: ice_candidate.sdp_mid(),
+                    sdpMLineIndex: ice_candidate.sdp_mline_index(),
+                    candidate: ice_candidate.candidate(),
+                })?;
+
+                trace!("sending ice_candidate ({:?}) - {:?}", target, ice_candidate);
+
+                tokio::spawn(async move {
+                    signal_client
+                        .send(signal_request::Message::Trickle(TrickleRequest {
+                            candidate_init: json,
+                            target: target as i32,
+                        }))
+                        .await;
+                });
+            }
+            RTCEvent::ConnectionChange { state, target } => {
+                // Reconnect if we've been disconnected unexpectedly
+                trace!("Connection change, {:?} {:?}", state, target);
+                let subscriber_primary = engine_inner.join_response.lock().subscriber_primary;
+                let is_primary = subscriber_primary && target == SignalTarget::Subscriber;
+
+                if is_primary && state == PeerConnectionState::Disconnected {
+                    let old_state = engine_inner
+                        .pc_state
+                        .swap(PCState::Connected as u8, Ordering::SeqCst);
+                    if old_state == PCState::New as u8 {
+                        // TODO(theomonnom) Handle disconnect
+                    }
+                } else if state == PeerConnectionState::Failed {
+                    engine_inner
+                        .pc_state
+                        .store(PCState::Disconnected as u8, Ordering::SeqCst);
+                    // TODO(theomonnom) Handle disconnect
+                }
+            }
+            RTCEvent::DataChannel {
+                data_channel,
+                target,
+            } => {
+                if target == SignalTarget::Subscriber {
+                    if data_channel.label() == RELIABLE_DC_LABEL {
+                        *engine_inner.sub_reliable_dc.lock() = Some(data_channel);
+                    } else {
+                        *engine_inner.sub_lossy_dc.lock() = Some(data_channel);
+                    }
+                }
+            }
+            RTCEvent::Offer { offer, target } => {
+                if target == SignalTarget::Publisher {
+                    // Send the publisher offer to the server
+                    tokio::spawn(async move {
+                        signal_client
+                            .send(signal_request::Message::Offer(proto::SessionDescription {
+                                r#type: "offer".to_string(),
+                                sdp: offer.to_string(),
+                            }))
+                            .await;
+                    });
+                }
+            }
+            RTCEvent::AddTrack {
+                rtp_receiver,
+                streams,
+                target,
+            } => {
+                if target == SignalTarget::Subscriber {
+                    let _ = emitter
+                        .send(EngineEvent::AddTrack {
+                            rtp_receiver,
+                            streams,
+                        })
+                        .await;
+                }
+            }
+            RTCEvent::Data { data, binary } => {
+                if !binary {
+                    Err(EngineError::Internal(
+                        "text messages aren't supported".to_string(),
+                    ))?;
+                }
+
+                let data = DataPacket::decode(&*data)?;
+                match data.value.unwrap() {
+                    Value::User(user) => {
+                        // TODO(theomonnom) Send event
+                    }
+                    Value::Speaker(_) => {
+                        // TODO(theomonnonm)
+                    }
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    async fn handle_signal(
+        event: signal_response::Message,
+        signal_client: Arc<SignalClient>,
+        engine_inner: Arc<EngineInner>,
+        emitter: EngineEmitter,
+    ) -> EngineResult<()> {
+        match event {
+            signal_response::Message::Answer(answer) => {
+                trace!("received answer from the publisher: {:?}", answer);
+
+                let sdp = SessionDescription::from(answer.r#type.parse().unwrap(), &answer.sdp)?;
+                engine_inner
+                    .publisher_pc
+                    .lock()
+                    .await
+                    .set_remote_description(sdp)
+                    .await?;
+            }
+            signal_response::Message::Offer(offer) => {
+                // Handle the subscriber offer & send an answer to livekit-server
+                // We always get an offer from the server when connecting
+                trace!("received offer for the subscriber: {:?}", offer);
+                let sdp = SessionDescription::from(offer.r#type.parse().unwrap(), &offer.sdp)?;
+
+                engine_inner
+                    .subscriber_pc
+                    .lock()
+                    .await
+                    .set_remote_description(sdp)
+                    .await?;
+                let answer = engine_inner
+                    .subscriber_pc
+                    .lock()
+                    .await
+                    .peer_connection()
+                    .create_answer(RTCOfferAnswerOptions::default())
+                    .await?;
+                engine_inner
+                    .subscriber_pc
+                    .lock()
+                    .await
+                    .peer_connection()
+                    .set_local_description(answer.clone())
+                    .await?;
+
+                tokio::spawn(async move {
+                    signal_client
+                        .send(signal_request::Message::Answer(proto::SessionDescription {
+                            r#type: "answer".to_string(),
+                            sdp: answer.to_string(),
+                        }))
+                        .await;
+                });
+            }
+            signal_response::Message::Trickle(trickle) => {
+                // Add the IceCandidate received from the livekit-server
+                let json: IceCandidateJSON = serde_json::from_str(&trickle.candidate_init)?;
+                let ice = IceCandidate::from(&json.sdpMid, json.sdpMLineIndex, &json.candidate)?;
+
+                trace!(
+                    "received ice_candidate {:?} - {:?}",
+                    SignalTarget::from_i32(trickle.target).unwrap(),
+                    ice
+                );
+
+                if trickle.target == SignalTarget::Publisher as i32 {
+                    engine_inner
+                        .publisher_pc
+                        .lock()
+                        .await
+                        .add_ice_candidate(ice)
+                        .await?;
+                } else {
+                    engine_inner
+                        .subscriber_pc
+                        .lock()
+                        .await
+                        .add_ice_candidate(ice)
+                        .await?;
+                }
+            }
+            signal_response::Message::Update(update) => {
+                let _ = emitter.send(EngineEvent::ParticipantUpdate(update)).await;
+            }
+            _ => {}
+        }
+
+        Ok(())
+    }
+
+    async fn ensure_publisher_connected(&self, kind: data_packet::Kind) -> EngineResult<()> {
+        if !self.join_response().subscriber_primary {
+            return Ok(());
+        }
+
+        let publisher = &self.engine_inner.publisher_pc;
+        {
+            let mut publisher = publisher.lock().await;
+            if !publisher.is_connected()
+                && publisher.peer_connection().ice_connection_state()
+                    != IceConnectionState::IceConnectionChecking
+            {
+                let _ = self.negotiate_publisher().await;
+            }
+        }
+
+        let dc = self.data_channel(kind);
+        if dc.lock().state() == DataState::Open {
+            return Ok(());
+        }
+
+        // Wait until the PeerConnection is connected
+        let wait_connected = async move {
+            while publisher.lock().await.is_connected() && dc.lock().state() == DataState::Open {
+                sleep(Duration::from_millis(50)).await;
+            }
+        };
+
+        tokio::select! {
+            _ = wait_connected => Ok(()),
+            _ = sleep(MAX_ICE_CONNECT_TIMEOUT) => {
+                let err = EngineError::Connection("could not establish publisher connection: timeout".to_string());
+                error!(error = ?err);
+                Err(err)
+            }
+        }
+    }
+
+    async fn negotiate_publisher(&self) -> EngineResult<()> {
+        self.engine_inner
+            .has_published
+            .store(true, Ordering::SeqCst);
+        if let Err(err) = self
+            .engine_inner
+            .publisher_pc
+            .lock()
+            .await
+            .negotiate()
+            .await
+        {
+            error!("failed to negotiate the publisher: {:?}", err);
+            Err(err)?
+        } else {
+            Ok(())
+        }
+    }
+
+    fn configure_engine(
+        lk_runtime: Arc<LKRuntime>,
+        join_response: JoinResponse,
+    ) -> EngineResult<(EngineInner, RTCEvents)> {
+        let (rtc_emitter, events) = mpsc::unbounded_channel();
+        let rtc_config = RTCConfiguration::from(join_response.clone());
+
+        let mut publisher_pc = PCTransport::new(
+            lk_runtime
+                .pc_factory
+                .create_peer_connection(rtc_config.clone())?,
+        );
+
+        let mut subscriber_pc = PCTransport::new(
+            lk_runtime
+                .pc_factory
+                .create_peer_connection(rtc_config.clone())?,
+        );
+
+        let mut lossy_dc = publisher_pc.peer_connection().create_data_channel(
+            LOSSY_DC_LABEL,
+            DataChannelInit {
+                ordered: true,
+                max_retransmits: Some(0),
+                ..DataChannelInit::default()
+            },
+        )?;
+
+        let mut reliable_dc = publisher_pc.peer_connection().create_data_channel(
+            RELIABLE_DC_LABEL,
+            DataChannelInit {
+                ordered: true,
+                ..DataChannelInit::default()
+            },
+        )?;
+
+        publisher_pc
+            .peer_connection()
+            .on_ice_candidate(rtc_events::on_ice_candidate(
+                SignalTarget::Publisher,
+                rtc_emitter.clone(),
+            ));
+        subscriber_pc
+            .peer_connection()
+            .on_ice_candidate(rtc_events::on_ice_candidate(
+                SignalTarget::Subscriber,
+                rtc_emitter.clone(),
+            ));
+
+        publisher_pc.on_offer(rtc_events::on_offer(
+            SignalTarget::Publisher,
+            rtc_emitter.clone(),
+        ));
+        subscriber_pc.on_offer(rtc_events::on_offer(
+            SignalTarget::Subscriber,
+            rtc_emitter.clone(),
+        ));
+
+        publisher_pc
+            .peer_connection()
+            .on_data_channel(rtc_events::on_data_channel(
+                SignalTarget::Publisher,
+                rtc_emitter.clone(),
+            ));
+        subscriber_pc
+            .peer_connection()
+            .on_data_channel(rtc_events::on_data_channel(
+                SignalTarget::Subscriber,
+                rtc_emitter.clone(),
+            ));
+
+        publisher_pc
+            .peer_connection()
+            .on_add_track(rtc_events::on_add_track(
+                SignalTarget::Publisher,
+                rtc_emitter.clone(),
+            ));
+        subscriber_pc
+            .peer_connection()
+            .on_add_track(rtc_events::on_add_track(
+                SignalTarget::Subscriber,
+                rtc_emitter.clone(),
+            ));
+
+        publisher_pc
+            .peer_connection()
+            .on_connection_change(rtc_events::on_connection_change(
+                SignalTarget::Publisher,
+                rtc_emitter.clone(),
+            ));
+        subscriber_pc
+            .peer_connection()
+            .on_connection_change(rtc_events::on_connection_change(
+                SignalTarget::Subscriber,
+                rtc_emitter.clone(),
+            ));
+
+        lossy_dc.on_message(rtc_events::on_message(rtc_emitter.clone()));
+        reliable_dc.on_message(rtc_events::on_message(rtc_emitter.clone()));
+
+        Ok((
+            EngineInner {
+                has_published: AtomicBool::new(false),
+                join_response: Mutex::new(join_response),
+                pc_state: AtomicU8::new(PCState::New as u8),
+                publisher_pc: AsyncMutex::new(publisher_pc),
+                subscriber_pc: AsyncMutex::new(subscriber_pc),
+                lossy_dc: Mutex::new(lossy_dc),
+                reliable_dc: Mutex::new(reliable_dc),
+                sub_lossy_dc: Mutex::new(None),
+                sub_reliable_dc: Mutex::new(None),
+            },
+            events,
+        ))
+    }
+
+    fn data_channel(&self, kind: data_packet::Kind) -> &Mutex<DataChannel> {
+        if kind == data_packet::Kind::Reliable {
+            &self.engine_inner.reliable_dc
+        } else {
+            &self.engine_inner.lossy_dc
+        }
     }
 }

crates/livekit-core/src/rtc_engine/pc_transport.rs

@@ -13,7 +13,11 @@ use livekit_webrtc::rtc_error::RTCError;
 
 const NEGOTIATION_FREQUENCY: Duration = Duration::from_millis(150);
 
-pub type OnOfferHandler = Box<dyn (FnMut(SessionDescription) -> Pin<Box<dyn Future<Output=()> + Send + 'static>>) + Send + Sync>;
+pub type OnOfferHandler = Box<
+    dyn (FnMut(SessionDescription) -> Pin<Box<dyn Future<Output = ()> + Send + 'static>>)
+        + Send
+        + Sync,
+>;
 
 pub struct PCTransport {
     peer_connection: PeerConnection,
@@ -43,7 +47,7 @@ impl PCTransport {
     pub fn is_connected(&self) -> bool {
         self.peer_connection.ice_connection_state() == IceConnectionState::IceConnectionConnected
             || self.peer_connection.ice_connection_state()
-            == IceConnectionState::IceConnectionCompleted
+                == IceConnectionState::IceConnectionCompleted
     }
 
     pub fn peer_connection(&mut self) -> &mut PeerConnection {
@@ -118,7 +122,10 @@ impl PCTransport {
                         .set_remote_description(remote_description)
                         .await?;
                 } else {
-                    event!(Level::ERROR, "trying to restart ICE when the pc doesn't have remote description");
+                    event!(
+                        Level::ERROR,
+                        "trying to restart ICE when the pc doesn't have remote description"
+                    );
                 }
             } else {
                 self.renegotiate = true;

crates/livekit-core/src/rtc_engine/rtc_events.rs

@@ -0,0 +1,103 @@
+use livekit_webrtc::data_channel::{DataChannel, OnMessageHandler};
+use livekit_webrtc::jsep::{IceCandidate, SessionDescription};
+use livekit_webrtc::media_stream::MediaStream;
+use livekit_webrtc::peer_connection::{
+    OnAddTrackHandler, OnConnectionChangeHandler, OnDataChannelHandler, OnIceCandidateHandler,
+    PeerConnectionState,
+};
+use livekit_webrtc::rtp_receiver::RtpReceiver;
+use tokio::sync::mpsc;
+
+use crate::proto::SignalTarget;
+use crate::rtc_engine::pc_transport::OnOfferHandler;
+
+pub(super) type RTCEmitter = mpsc::UnboundedSender<RTCEvent>;
+pub(super) type RTCEvents = mpsc::UnboundedReceiver<RTCEvent>;
+
+#[derive(Debug)]
+pub(super) enum RTCEvent {
+    IceCandidate {
+        ice_candidate: IceCandidate,
+        target: SignalTarget,
+    },
+    ConnectionChange {
+        state: PeerConnectionState,
+        target: SignalTarget,
+    },
+    DataChannel {
+        data_channel: DataChannel,
+        target: SignalTarget,
+    },
+    Offer {
+        offer: SessionDescription,
+        target: SignalTarget,
+    },
+    AddTrack {
+        rtp_receiver: RtpReceiver,
+        streams: Vec<MediaStream>,
+        target: SignalTarget,
+    },
+    Data {
+        data: Vec<u8>,
+        binary: bool,
+    },
+}
+
+/// Handlers used to forward event to a channel
+/// Every callback here is called on the signaling thread
+
+pub(super) fn on_connection_change(
+    target: SignalTarget,
+    emitter: RTCEmitter,
+) -> OnConnectionChangeHandler {
+    Box::new(move |state| {
+        let _ = emitter.send(RTCEvent::ConnectionChange { state, target });
+    })
+}
+
+pub(super) fn on_ice_candidate(target: SignalTarget, emitter: RTCEmitter) -> OnIceCandidateHandler {
+    Box::new(move |ice_candidate| {
+        let _ = emitter.send(RTCEvent::IceCandidate {
+            ice_candidate,
+            target,
+        });
+    })
+}
+
+pub(super) fn on_offer(target: SignalTarget, emitter: RTCEmitter) -> OnOfferHandler {
+    Box::new(move |offer| {
+        let _ = emitter.send(RTCEvent::Offer { offer, target });
+
+        Box::pin(async {})
+    })
+}
+
+pub(super) fn on_data_channel(target: SignalTarget, emitter: RTCEmitter) -> OnDataChannelHandler {
+    Box::new(move |mut data_channel| {
+        data_channel.on_message(on_message(emitter.clone()));
+
+        let _ = emitter.send(RTCEvent::DataChannel {
+            data_channel,
+            target,
+        });
+    })
+}
+
+pub(super) fn on_add_track(target: SignalTarget, emitter: RTCEmitter) -> OnAddTrackHandler {
+    Box::new(move |rtp_receiver, streams| {
+        let _ = emitter.send(RTCEvent::AddTrack {
+            rtp_receiver,
+            streams,
+            target,
+        });
+    })
+}
+
+pub(super) fn on_message(emitter: RTCEmitter) -> OnMessageHandler {
+    Box::new(move |data, binary| {
+        let _ = emitter.send(RTCEvent::Data {
+            data: data.to_vec(),
+            binary,
+        });
+    })
+}

crates/livekit-core/src/signal_client/mod.rs

@@ -1,18 +1,23 @@
-use core::num::flt2dec::Sign;
 use std::fmt::Debug;
+use std::time::Duration;
 
+use livekit_webrtc::peer_connection_factory::{
+    ContinualGatheringPolicy, ICEServer, IceTransportsType, RTCConfiguration,
+};
 use thiserror::Error;
+use tokio::sync::mpsc;
 use tokio_tungstenite::tungstenite::Error as WsError;
 
-use crate::event::{Emitter, Events};
-use crate::proto::{signal_request, signal_response};
+use crate::proto::{signal_request, signal_response, JoinResponse};
 use crate::signal_client::signal_stream::SignalStream;
 
 mod signal_stream;
 
-type SignalEmitter = Emitter<SignalEvent>;
-type SignalEvents = Events<SignalEvent>;
-type SignalResult<T> = Result<T, SignalError>;
+pub(crate) type SignalEmitter = mpsc::Sender<SignalEvent>;
+pub(crate) type SignalEvents = mpsc::Receiver<SignalEvent>;
+pub(crate) type SignalResult<T> = Result<T, SignalError>;
+
+pub const JOIN_RESPONSE_TIMEOUT: Duration = Duration::from_secs(5);
 
 #[derive(Error, Debug)]
 pub enum SignalError {
@@ -22,10 +27,12 @@ pub enum SignalError {
     UrlParse(#[from] url::ParseError),
     #[error("failed to decode messages from server")]
     ProtoParse(#[from] prost::DecodeError),
+    #[error("{0}")]
+    Timeout(String),
 }
 
 /// Events used by the RTCEngine who will handle the reconnection logic
-#[derive(Clone, Debug)]
+#[derive(Debug)]
 pub(crate) enum SignalEvent {
     Open,
     Signal(signal_response::Message),
@@ -40,6 +47,17 @@ pub(crate) struct SignalOptions {
     adaptive_stream: bool,
 }
 
+impl Default for SignalOptions {
+    fn default() -> Self {
+        Self {
+            reconnect: false,
+            auto_subscribe: true,
+            sid: "".to_string(),
+            adaptive_stream: false,
+        }
+    }
+}
+
 #[derive(Debug)]
 pub struct SignalClient {
     stream: SignalStream,
@@ -47,15 +65,16 @@ pub struct SignalClient {
 }
 
 impl SignalClient {
-    pub async fn connect(
+    pub(crate) async fn connect(
         url: &str,
         token: &str,
         options: SignalOptions,
     ) -> SignalResult<(Self, SignalEvents)> {
-        // TODO(theomonnom) Retry initial connection
-        let (emitter, receiver) = SignalEmitter::new();
-        let events = SignalEvents::new(receiver);
+        let (emitter, events) = mpsc::channel(8);
         let stream = SignalStream::connect(url, token, options, emitter.clone()).await?;
+
+        // TODO(theomonnom) Retry initial connection
+
         Ok((Self { stream, emitter }, events))
     }
 
@@ -69,3 +88,60 @@ impl SignalClient {
         // TODO(theomonnom) Close & recreate SignalStream, also send the queue if needed
     }
 }
+
+impl From<JoinResponse> for RTCConfiguration {
+    fn from(join_response: JoinResponse) -> Self {
+        Self {
+            ice_servers: {
+                let mut servers = vec![];
+                for ice_server in join_response.ice_servers.clone() {
+                    servers.push(ICEServer {
+                        urls: ice_server.urls,
+                        username: ice_server.username,
+                        password: ice_server.credential,
+                    })
+                }
+                servers
+            },
+            continual_gathering_policy: ContinualGatheringPolicy::GatherContinually,
+            ice_transport_type: IceTransportsType::All,
+        }
+    }
+}
+
+pub mod utils {
+    use crate::proto::{signal_response, JoinResponse};
+    use crate::signal_client::{SignalError, SignalEvent, SignalResult, JOIN_RESPONSE_TIMEOUT};
+    use tokio::sync::mpsc;
+    use tokio::time::timeout;
+    use tokio_tungstenite::tungstenite::Error as WsError;
+    use tracing::{event, Level};
+
+    pub(crate) async fn next_join_response(
+        receiver: &mut mpsc::Receiver<SignalEvent>,
+    ) -> SignalResult<JoinResponse> {
+        let join = async {
+            while let Some(event) = receiver.recv().await {
+                match event {
+                    SignalEvent::Signal(signal_response::Message::Join(join)) => return Ok(join),
+                    SignalEvent::Close => break,
+                    SignalEvent::Open => continue,
+                    _ => {
+                        event!(
+                            Level::WARN,
+                            "received unexpected message while waiting for JoinResponse: {:?}",
+                            event
+                        );
+                        continue;
+                    }
+                }
+            }
+
+            Err(WsError::ConnectionClosed)?
+        };
+
+        timeout(JOIN_RESPONSE_TIMEOUT, join)
+            .await
+            .map_err(|_| SignalError::Timeout("failed to receive JoinResponse".to_string()))?
+    }
+}

crates/livekit-core/src/signal_client/signal_stream.rs

@@ -1,13 +1,13 @@
-use futures_util::{SinkExt, StreamExt};
 use futures_util::stream::{SplitSink, SplitStream};
+use futures_util::{SinkExt, StreamExt};
 use prost::Message as ProstMessage;
 use tokio::net::TcpStream;
 use tokio::sync::{mpsc, oneshot};
 use tokio::task::JoinHandle;
-use tokio_tungstenite::{connect_async, MaybeTlsStream, WebSocketStream};
-use tokio_tungstenite::tungstenite::Message;
-use tokio_tungstenite::tungstenite::protocol::CloseFrame;
 use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode;
+use tokio_tungstenite::tungstenite::protocol::CloseFrame;
+use tokio_tungstenite::tungstenite::Message;
+use tokio_tungstenite::{connect_async, MaybeTlsStream, WebSocketStream};
 use tracing::{event, Level};
 
 use crate::proto::{signal_request, SignalRequest, SignalResponse};
@@ -72,7 +72,7 @@ impl SignalStream {
         event!(Level::DEBUG, "connecting to websocket: {}", lk_url);
         let (ws_stream, _) = connect_async(lk_url).await?;
         event!(Level::DEBUG, "connected to websocket");
-        emitter.event(SignalEvent::Open);
+        let _ = emitter.send(SignalEvent::Open).await;
 
         let (ws_writer, ws_reader) = ws_stream.split();
         let (internal_tx, internal_rx) = mpsc::channel::<InternalMessage>(8);
@@ -119,7 +119,7 @@ impl SignalStream {
 
     /// This task is used to send messages to the websocket
     /// It is also responsible for closing the connection
-    pub async fn handle_write(
+    async fn handle_write(
         mut internal_rx: mpsc::Receiver<InternalMessage>,
         mut ws_writer: SplitSink<WebSocket, Message>,
         emitter: SignalEmitter,
@@ -136,7 +136,7 @@ impl SignalStream {
                         SignalRequest {
                             message: Some(signal),
                         }
-                            .encode_to_vec(),
+                        .encode_to_vec(),
                     );
 
                     if let Err(err) = ws_writer.send(data).await {
@@ -163,14 +163,14 @@ impl SignalStream {
         }
 
         let _ = ws_writer.close().await;
-        emitter.event(SignalEvent::Close);
+        let _ = emitter.send(SignalEvent::Close).await;
     }
 
     /// This task is used to read incoming messages from the websocket
     /// and dispatch them through the EventEmitter.
     ///
     /// It can also send messages to [handle_write] task ( Used e.g. answer to pings )
-    pub async fn handle_read(
+    async fn handle_read(
         internal_tx: mpsc::Sender<InternalMessage>,
         mut ws_reader: SplitStream<WebSocket>,
         emitter: SignalEmitter,
@@ -181,8 +181,9 @@ impl SignalStream {
                     let res = SignalResponse::decode(data.as_slice())
                         .expect("failed to decode SignalResponse");
 
-                    event!(Level::TRACE, "received SignalResponse: {:?}", res);
-                    emitter.event(SignalEvent::Signal(res.message.unwrap()));
+                    let msg = res.message.unwrap();
+                    event!(Level::TRACE, "received SignalResponse: {:?}", msg);
+                    let _ = emitter.send(SignalEvent::Signal(msg)).await;
                 }
                 Ok(Message::Ping(data)) => {
                     let _ = internal_tx

crates/livekit-utils/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "livekit-utils"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]

crates/livekit-utils/src/enum_dispatch.rs

@@ -0,0 +1,25 @@
+#[macro_export]
+macro_rules! enum_dispatch {
+    // This arm is used to avoid nested loops with the arguments
+    // The arguments are transformed to $combined_args TokenTree
+    (@match $self:ident $fnc:ident $combined_args:tt [$($variant:ident),+]) => {
+        match $self {
+            $(
+                Self::$variant(inner) => inner.$fnc$combined_args,
+            )+
+        }
+    };
+
+    ($fnc:ident, $self:ty, [$($arg:ident: $t:ty),*], $ret:ty, [$($variant:ident),+]) => {
+        fn $fnc(self: $self, $($arg: $t),*) -> $ret {
+            enum_dispatch!(@match self $fnc ($($arg,)*) [$($variant),+])
+        }
+    };
+
+    ($variants:tt $(fnc!($fnc:ident, $self:ty, $args:tt, $ret:ty);)+) => {
+        $(
+            enum_dispatch!($fnc, $self, $args, $ret, $variants);
+        )*
+    };
+}
+

crates/livekit-utils/src/lib.rs

@@ -0,0 +1 @@
+pub mod enum_dispatch;

crates/livekit-webrtc/Cargo.toml

@@ -6,10 +6,11 @@ homepage = "https://livekit.io"
 
 [dependencies]
 libwebrtc-sys = { path = "./libwebrtc-sys" }
+livekit-utils = { path = "../livekit-utils" }
 tokio = { version = "1", features = ["full"] }
 cxx = "1.0"
 log = "0.4"
 thiserror = "1.0"
 
 [dev-dependencies]
-env_logger = "0.9"
+env_logger = "0.9"

crates/livekit-webrtc/libwebrtc-sys/.gitignore

@@ -1,2 +1 @@
-/libwebrtc
 /cmake-build-debug

crates/livekit-webrtc/libwebrtc-sys/CMakeLists.txt

@@ -1,23 +0,0 @@
-# IMPORTANT NOTE
-# This file is just used because some IDEs need to understand how to do autocompletion.
-# This file is completely ignored by the library ( See build.rs for the build system )
-
-cmake_minimum_required(VERSION 3.22)
-project(livekit-webrtc)
-
-set(CMAKE_CXX_STANDARD 17)
-
-add_definitions(-DWEBRTC_WIN)
-
-include_directories(libwebrtc/include)
-include_directories(libwebrtc/include/third_party/abseil-cpp/)
-include_directories(libwebrtc/include/third_party/libc++/)
-include_directories(include/)
-include_directories(../../../target/cxxbridge) # Can be different
-
-file(GLOB_RECURSE SRC src/*.cpp)
-add_library(livekit-webrtc ${SRC})
-
-#include_directories(/Users/theomonnom/Library/Android/sdk/ndk/25.0.8775105/toolchains/llvm/prebuilt/darwin-x86_64/sysroot/usr/include)
-#find_library(ANDROID_LIB_ANDROID android)
-#target_link_libraries(client_sdk_native PRIVATE android)

crates/livekit-webrtc/libwebrtc-sys/build.rs

@@ -55,23 +55,23 @@ fn main() {
     let target_os = "windows";
     //let target_arch = "arm64";
 
-    let libwebrtc_dir = path::PathBuf::from("libwebrtc");
+    let libwebrtc_dir = path::PathBuf::from("libwebrtc/src");
 
     // Just required for the bridge build to succeed.
     let includes = &[
         path::PathBuf::from("./include"),
-        libwebrtc_dir.join("include/"),
-        libwebrtc_dir.join("include/third_party/abseil-cpp/"),
-        libwebrtc_dir.join("include/third_party/libc++/"),
+        libwebrtc_dir.clone(),
+        libwebrtc_dir.join("third_party/abseil-cpp/"),
+        libwebrtc_dir.join("third_party/libc++/"),
         // For mac & ios
-        libwebrtc_dir.join("include/sdk/objc"),
-        libwebrtc_dir.join("include/sdk/objc/base"),
+        libwebrtc_dir.join("sdk/objc"),
+        libwebrtc_dir.join("sdk/objc/base"),
     ];
 
     let mut builder = cxx_build::bridges(&[
         "src/peer_connection.rs",
         "src/peer_connection_factory.rs",
-        "src/media_stream_interface.rs",
+        "src/media_stream.rs",
         "src/data_channel.rs",
         "src/jsep.rs",
         "src/candidate.rs",
@@ -79,11 +79,14 @@ fn main() {
         "src/rtp_transceiver.rs",
         "src/rtc_error.rs",
         "src/webrtc.rs",
+        "src/video_frame.rs",
+        "src/video_frame_buffer.rs",
+        "src/yuv_helper.rs",
     ]);
 
     builder.file("src/peer_connection.cpp");
     builder.file("src/peer_connection_factory.cpp");
-    builder.file("src/media_stream_interface.cpp");
+    builder.file("src/media_stream.cpp");
     builder.file("src/data_channel.cpp");
     builder.file("src/jsep.cpp");
     builder.file("src/candidate.cpp");
@@ -98,7 +101,12 @@ fn main() {
 
     println!(
         "cargo:rustc-link-search=native={}",
-        libwebrtc_dir.canonicalize().unwrap().to_str().unwrap()
+        libwebrtc_dir
+            .join("out/Default/obj")
+            .canonicalize()
+            .unwrap()
+            .to_str()
+            .unwrap()
     );
 
     match target_os {
@@ -122,6 +130,7 @@ fn main() {
                 .flag("/std:c++17")
                 .flag("/EHsc")
                 .define("WEBRTC_WIN", None)
+                //.define("WEBRTC_ENABLE_SYMBOL_EXPORT", None) Not necessary when using WebRTC as a static library
                 .define("NOMINMAX", None);
         }
         "macos" => {
@@ -203,7 +212,7 @@ fn main() {
             let jni_regex = Regex::new(r"(Java_org_webrtc.*)").unwrap();
             let content = &String::from_utf8_lossy(&readelf_output.stdout);
             let mut jni_symbols = Vec::new();
-            jni_regex.captures_iter(&content).for_each(|cap| {
+            jni_regex.captures_iter(content).for_each(|cap| {
                 jni_symbols.push(cap.get(1).unwrap().as_str());
             });
 
@@ -215,7 +224,7 @@ fn main() {
 
             write!(vs_file, "JNI_WEBRTC {{\n\tglobal: ").unwrap();
             write!(vs_file, "JNI_OnLoad; ").unwrap();
-            for x in &jni_symbols {
+            for x in jni_symbols {
                 println!("cargo:rustc-link-arg=-Wl,--undefined={}", x);
                 write!(vs_file, "{}; ", x).unwrap();
             }

crates/livekit-webrtc/libwebrtc-sys/compile_flags.txt

@@ -0,0 +1,8 @@
+-xc++
+-std=c++17
+-Iinclude
+-Ilibwebrtc/src
+-Ilibwebrtc/src/third_party/abseil-cpp
+-Ilibwebrtc/src/third_party/libc++
+-I../../../target/cxxbridge
+-DWEBRTC_WIN

crates/livekit-webrtc/libwebrtc-sys/include/livekit/media_stream.h

@@ -0,0 +1,123 @@
+//
+// Created by Théo Monnom on 31/08/2022.
+//
+
+#ifndef CLIENT_SDK_NATIVE_MEDIA_STREAM_INTERFACE_H
+#define CLIENT_SDK_NATIVE_MEDIA_STREAM_INTERFACE_H
+
+#include <memory>
+
+#include "api/media_stream_interface.h"
+#include "livekit/rust_types.h"
+#include "rust/cxx.h"
+
+namespace livekit {
+
+class NativeVideoFrameSink;
+
+class MediaStream {
+ public:
+  explicit MediaStream(rtc::scoped_refptr<webrtc::MediaStreamInterface> stream);
+
+  rust::String id() const;
+
+ private:
+  rtc::scoped_refptr<webrtc::MediaStreamInterface> media_stream_;
+};
+
+static std::unique_ptr<MediaStream> _unique_media_stream() {
+  return nullptr;  // Ignore
+}
+
+class MediaStreamTrack {
+ protected:
+  explicit MediaStreamTrack(
+      rtc::scoped_refptr<webrtc::MediaStreamTrackInterface> track);
+
+ public:
+  static std::unique_ptr<MediaStreamTrack> from(
+      rtc::scoped_refptr<webrtc::MediaStreamTrackInterface> track);
+
+  rust::String kind() const;
+  rust::String id() const;
+
+  bool enabled() const;
+  bool set_enabled(bool enable);
+
+  TrackState state() const;
+
+ protected:
+  rtc::scoped_refptr<webrtc::MediaStreamTrackInterface> track_;
+};
+
+static std::unique_ptr<MediaStreamTrack> _unique_media_stream_track() {
+  return nullptr;  // Ignore
+}
+
+class AudioTrack : public MediaStreamTrack {
+ public:
+  explicit AudioTrack(rtc::scoped_refptr<webrtc::AudioTrackInterface> track);
+};
+
+static std::unique_ptr<AudioTrack> _unique_audio_track() {
+  return nullptr;  // Ignore
+}
+
+class VideoTrack : public MediaStreamTrack {
+ public:
+  explicit VideoTrack(rtc::scoped_refptr<webrtc::VideoTrackInterface> track);
+
+  void add_sink(NativeVideoFrameSink& sink);
+  void remove_sink(NativeVideoFrameSink& sink);
+
+  void set_should_receive(bool should_receive);
+  bool should_receive() const;
+  ContentHint content_hint() const;
+  void set_content_hint(ContentHint hint);
+
+ private:
+  webrtc::VideoTrackInterface* track() const {
+    return static_cast<webrtc::VideoTrackInterface*>(track_.get());
+  }
+};
+
+static std::unique_ptr<VideoTrack> _unique_video_track() {
+  return nullptr;  // Ignore
+}
+
+class NativeVideoFrameSink
+    : public rtc::VideoSinkInterface<webrtc::VideoFrame> {
+ public:
+  explicit NativeVideoFrameSink(rust::Box<VideoFrameSinkWrapper> observer);
+
+  void OnFrame(const webrtc::VideoFrame& frame) override;
+  void OnDiscardedFrame() override;
+  void OnConstraintsChanged(
+      const webrtc::VideoTrackSourceConstraints& constraints) override;
+
+ private:
+  rust::Box<VideoFrameSinkWrapper> observer_;
+};
+
+std::unique_ptr<NativeVideoFrameSink> create_native_video_frame_sink(
+    rust::Box<VideoFrameSinkWrapper> observer);
+
+static const MediaStreamTrack* video_to_media(const VideoTrack* track) {
+  return track;
+}
+
+static const MediaStreamTrack* audio_to_media(const AudioTrack* track) {
+  return track;
+}
+
+static const VideoTrack* media_to_video(const MediaStreamTrack* track) {
+  return static_cast<const VideoTrack*>(track);
+}
+
+static const AudioTrack* media_to_audio(const MediaStreamTrack* track) {
+  return static_cast<const AudioTrack*>(track); 
+}
+
+}  // namespace livekit
+
+#endif  // CLIENT_SDK_NATIVE_MEDIA_STREAM_INTERFACE_H

crates/livekit-webrtc/libwebrtc-sys/include/livekit/media_stream_interface.h

@@ -1,28 +0,0 @@
-//
-// Created by Théo Monnom on 31/08/2022.
-//
-
-#ifndef CLIENT_SDK_NATIVE_MEDIA_STREAM_INTERFACE_H
-#define CLIENT_SDK_NATIVE_MEDIA_STREAM_INTERFACE_H
-
-#include <memory>
-
-#include "api/media_stream_interface.h"
-
-namespace livekit {
-
-class MediaStreamInterface {
- public:
-  explicit MediaStreamInterface(
-      rtc::scoped_refptr<webrtc::MediaStreamInterface> stream);
-
- private:
-  rtc::scoped_refptr<webrtc::MediaStreamInterface> media_stream_;
-};
-
-static std::unique_ptr<MediaStreamInterface> _unique_media_stream() {
-  return nullptr;  // Ignore
-}
-}  // namespace livekit
-
-#endif  // CLIENT_SDK_NATIVE_MEDIA_STREAM_INTERFACE_H

crates/livekit-webrtc/libwebrtc-sys/include/livekit/rtp_receiver.h

@@ -8,6 +8,7 @@
 #include <memory>
 
 #include "api/rtp_receiver_interface.h"
+#include "livekit/media_stream.h"
 
 namespace livekit {
 
@@ -16,6 +17,8 @@ class RtpReceiver {
   explicit RtpReceiver(
       rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver);
 
+  std::unique_ptr<MediaStreamTrack> track() const;
+
  private:
   rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver_;
 };

crates/livekit-webrtc/libwebrtc-sys/include/livekit/rust_types.h

@@ -15,6 +15,7 @@ struct SetLocalSdpObserverWrapper;
 struct SetRemoteSdpObserverWrapper;
 struct DataChannelObserverWrapper;
 struct AddIceCandidateObserverWrapper;
+struct VideoFrameSinkWrapper;
 
 // Shared types
 enum class PeerConnectionState;
@@ -23,6 +24,10 @@ enum class IceConnectionState;
 enum class IceGatheringState;
 enum class SdpType;
 enum class DataState;
+enum class TrackState;
+enum class ContentHint;
+enum class VideoRotation;
+enum class VideoFrameBufferType;
 struct SdpParseError;
 struct RTCOfferAnswerOptions;
 struct RTCError;

crates/livekit-webrtc/libwebrtc-sys/include/livekit/video_frame.h

@@ -0,0 +1,48 @@
+//
+// Created by theom on 14/11/2022.
+//
+
+#ifndef LIVEKIT_WEBRTC_VIDEO_FRAME_H
+#define LIVEKIT_WEBRTC_VIDEO_FRAME_H
+
+#include "api/video/video_frame.h"
+#include "livekit/rust_types.h"
+#include "livekit/video_frame_buffer.h"
+
+namespace livekit {
+
+class VideoFrame {
+ public:
+  explicit VideoFrame(const webrtc::VideoFrame& frame)
+      : frame_(std::move(frame)) {}
+
+  int width() const { return frame_.width(); }
+  int height() const { return frame_.height(); }
+  uint32_t size() const { return frame_.size(); }
+  uint16_t id() const { return frame_.id(); }
+  int64_t timestamp_us() const { return frame_.timestamp_us(); }
+  int64_t ntp_time_ms() const { return frame_.ntp_time_ms(); }
+  uint32_t transport_frame_id() const { return frame_.transport_frame_id(); }
+  uint32_t timestamp() const { return frame_.timestamp(); }
+
+  VideoRotation rotation() const {
+    return static_cast<VideoRotation>(frame_.rotation());
+  }
+
+  // TODO(theomonnom) This shouldn't create a new shared_ptr at each call
+  std::unique_ptr<VideoFrameBuffer> video_frame_buffer() const {
+    return std::make_unique<VideoFrameBuffer>(frame_.video_frame_buffer());
+  }
+
+ private:
+  webrtc::VideoFrame frame_;
+};
+
+static std::unique_ptr<VideoFrame> _unique_video_frame() {
+  return nullptr;  // Ignore
+}
+
+
+}  // namespace livekit
+
+#endif  // LIVEKIT_WEBRTC_VIDEO_FRAME_H

crates/livekit-webrtc/libwebrtc-sys/include/livekit/video_frame_buffer.h

@@ -0,0 +1,103 @@
+//
+// Created by theom on 14/11/2022.
+//
+
+#ifndef LIVEKIT_WEBRTC_VIDEO_FRAME_BUFFER_H
+#define LIVEKIT_WEBRTC_VIDEO_FRAME_BUFFER_H
+
+#include <memory>
+
+#include "api/video/video_frame_buffer.h"
+#include "rust_types.h"
+
+namespace livekit {
+
+class PlanarYuvBuffer;
+class PlanarYuv8Buffer;
+class I420Buffer;
+
+class VideoFrameBuffer {
+ public:
+  explicit VideoFrameBuffer(rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer)
+      : buffer_(std::move(buffer)) {}
+
+  VideoFrameBufferType buffer_type() const {
+    return static_cast<VideoFrameBufferType>(buffer_->type());
+  }
+
+  int width() const { return buffer_->width(); }
+  int height() const { return buffer_->height(); }
+
+  std::unique_ptr<I420Buffer> to_i420() {
+    return std::make_unique<I420Buffer>(buffer_->ToI420());
+  }
+
+  std::unique_ptr<I420Buffer> get_i420() {
+    // const_cast is valid here because we take the ownership on the rust side
+    return std::make_unique<I420Buffer>(
+        rtc::scoped_refptr<webrtc::I420BufferInterface>(
+            const_cast<webrtc::I420BufferInterface*>(buffer_->GetI420())));
+  }
+
+ protected:
+  rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer_;
+};
+
+class PlanarYuvBuffer : public VideoFrameBuffer {
+ public:
+  explicit PlanarYuvBuffer(rtc::scoped_refptr<webrtc::PlanarYuvBuffer> buffer)
+      : VideoFrameBuffer(buffer) {}
+
+  int chroma_width() const { return buffer()->ChromaWidth(); }
+  int chroma_height() const { return buffer()->ChromaHeight(); }
+
+  int stride_y() const { return buffer()->StrideY(); }
+  int stride_u() const { return buffer()->StrideU(); }
+  int stride_v() const { return buffer()->StrideV(); }
+
+ private:
+  webrtc::PlanarYuvBuffer* buffer() const {
+    return static_cast<webrtc::PlanarYuvBuffer*>(buffer_.get());
+  }
+};
+
+class PlanarYuv8Buffer : public PlanarYuvBuffer {
+ public:
+  explicit PlanarYuv8Buffer(rtc::scoped_refptr<webrtc::PlanarYuv8Buffer> buffer)
+      : PlanarYuvBuffer(buffer) {}
+
+  const uint8_t* data_y() const { return buffer()->DataY(); }
+  const uint8_t* data_u() const { return buffer()->DataU(); }
+  const uint8_t* data_v() const { return buffer()->DataV(); }
+
+ private:
+  webrtc::PlanarYuv8Buffer* buffer() const {
+    return static_cast<webrtc::PlanarYuv8Buffer*>(buffer_.get());
+  }
+};
+
+class I420Buffer : public PlanarYuv8Buffer {
+ public:
+  explicit I420Buffer(rtc::scoped_refptr<webrtc::I420BufferInterface> buffer)
+      : PlanarYuv8Buffer(buffer) {}
+};
+
+static const VideoFrameBuffer* yuv_to_vfb(const PlanarYuvBuffer* yuv) {
+  return yuv;
+}
+
+static const PlanarYuvBuffer* yuv8_to_yuv(const PlanarYuv8Buffer* yuv8) {
+  return yuv8;
+}
+
+static const PlanarYuv8Buffer* i420_to_yuv8(const I420Buffer* i420) {
+  return i420;
+}
+
+static std::unique_ptr<VideoFrameBuffer> _unique_video_frame_buffer() {
+  return nullptr;  // Ignore
+}
+
+}  // namespace livekit
+
+#endif  // LIVEKIT_WEBRTC_VIDEO_FRAME_BUFFER_H

crates/livekit-webrtc/libwebrtc-sys/include/livekit/webrtc.h

@@ -32,6 +32,8 @@ class RTCRuntime {
   std::unique_ptr<rtc::Thread> signaling_thread_;
 #ifdef WEBRTC_WIN
   rtc::WinsockInitializer winsock_;
+  rtc::PhysicalSocketServer ss_;
+  rtc::AutoSocketServerThread main_thread_ {&ss_};
 #endif
 };
 

crates/livekit-webrtc/libwebrtc-sys/include/livekit/yuv_helper.h

@@ -0,0 +1,30 @@
+//
+// Created by Théo Monnom on 01/12/2022.
+//
+
+#ifndef CLIENT_SDK_NATIVE_YUV_HELPER_H
+#define CLIENT_SDK_NATIVE_YUV_HELPER_H
+
+#include <memory>
+
+#include "api/video/yuv_helper.h"
+
+namespace livekit {
+
+static void i420_to_abgr(const uint8_t* src_y,
+                       int src_stride_y,
+                       const uint8_t* src_u,
+                       int src_stride_u,
+                       const uint8_t* src_v,
+                       int src_stride_v,
+                       uint8_t* dst_rgba,
+                       int dst_stride_abgr,
+                       int width,
+                       int height) {
+  webrtc::I420ToABGR(src_y, src_stride_y, src_u, src_stride_u, src_v,
+                     src_stride_v, dst_rgba, dst_stride_abgr, width, height);
+}
+
+}  // namespace livekit
+
+#endif  // CLIENT_SDK_NATIVE_YUV_HELPER_H

crates/livekit-webrtc/libwebrtc-sys/libwebrtc/.gclient

@@ -0,0 +1,11 @@
+solutions = [
+  { 
+    "name"        : 'src',
+    "url"         : 'https://github.com/webrtc-sdk/webrtc.git',
+    "deps_file"   : 'DEPS',
+    "managed"     : False,
+    "custom_deps" : {
+    },
+    "custom_vars": {},
+  },
+]

crates/livekit-webrtc/libwebrtc-sys/libwebrtc/.gitignore

@@ -0,0 +1,3 @@
+.cipd
+src
+.gclient_*

crates/livekit-webrtc/libwebrtc-sys/libwebrtc/compile.py

@@ -0,0 +1,19 @@
+import subprocess
+
+GN_ARGS = [
+    "is_debug=false",
+    "treat_warnings_as_errors=false",
+    'target_os="win"',
+    'target_cpu="x64"',
+    "rtc_include_tests=false",
+    "rtc_use_h264=false",
+    "is_component_build=false",
+    "rtc_build_examples=false",
+    "use_rtti=true",
+    "rtc_build_tools=false",
+    "use_custom_libcxx=false",
+    "strip_debug_info=true",
+    "symbol_level=0"
+]
+
+subprocess.call(["gn", "gen", "out/Default", "--args=" + ' '.join(GN_ARGS)], shell=True)

crates/livekit-webrtc/libwebrtc-sys/src/candidate.cpp

@@ -7,4 +7,4 @@
 namespace livekit {
 Candidate::Candidate(const cricket::Candidate& candidate)
     : candidate_(candidate) {}
-}  // namespace livekit
+}  // namespace livekit

crates/livekit-webrtc/libwebrtc-sys/src/data_channel.cpp

@@ -87,4 +87,4 @@ std::unique_ptr<NativeDataChannelObserver> create_native_data_channel_observer(
     rust::Box<DataChannelObserverWrapper> observer) {
   return std::make_unique<NativeDataChannelObserver>(std::move(observer));
 }
-}  // namespace livekit
+}  // namespace livekit

crates/livekit-webrtc/libwebrtc-sys/src/jsep.cpp

@@ -158,4 +158,4 @@ create_native_set_remote_sdp_observer(
               std::move(observer))});
 }
 
-}  // namespace livekit
+}  // namespace livekit

crates/livekit-webrtc/libwebrtc-sys/src/jsep.rs

@@ -68,11 +68,18 @@ pub mod ffi {
             observer: Box<SetRemoteSdpObserverWrapper>,
         ) -> UniquePtr<NativeSetRemoteSdpObserverHandle>;
 
-        fn create_ice_candidate(sdp_mid: String, sdp_mline_index: i32, sdp: String) -> Result<UniquePtr<IceCandidate>>;
-        fn create_session_description(sdp_type: SdpType, sdp: String) -> Result<UniquePtr<SessionDescription>>;
+        fn create_ice_candidate(
+            sdp_mid: String,
+            sdp_mline_index: i32,
+            sdp: String,
+        ) -> Result<UniquePtr<IceCandidate>>;
+        fn create_session_description(
+            sdp_type: SdpType,
+            sdp: String,
+        ) -> Result<UniquePtr<SessionDescription>>;
 
         fn _unique_ice_candidate() -> UniquePtr<IceCandidate>; // Ignore
-    fn _unique_session_description() -> UniquePtr<SessionDescription>; // Ignore
+        fn _unique_session_description() -> UniquePtr<SessionDescription>; // Ignore
     }
 }
 
@@ -80,7 +87,11 @@ impl Error for ffi::SdpParseError {}
 
 impl Display for ffi::SdpParseError {
     fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
-        write!(f, "SdpParseError occurred {}: {}", self.line, self.description)
+        write!(
+            f,
+            "SdpParseError occurred {}: {}",
+            self.line, self.description
+        )
     }
 }
 
@@ -101,10 +112,7 @@ impl ffi::SdpParseError {
         let line = String::from(&value[8..line_length]);
         let description = String::from(&value[line_length..]);
 
-        Self {
-            line,
-            description,
-        }
+        Self { line, description }
     }
 }
 

crates/livekit-webrtc/libwebrtc-sys/src/lib.rs

@@ -1,10 +1,17 @@
 pub mod candidate;
 pub mod data_channel;
 pub mod jsep;
-pub mod media_stream_interface;
+pub mod media_stream;
 pub mod peer_connection;
 pub mod peer_connection_factory;
 pub mod rtc_error;
 pub mod rtp_receiver;
 pub mod rtp_transceiver;
+pub mod video_frame;
+pub mod video_frame_buffer;
 pub mod webrtc;
+pub mod yuv_helper;
+
+pub const MEDIA_TYPE_VIDEO: &str = "video";
+pub const MEDIA_TYPE_AUDIO: &str = "audio";
+pub const MEDIA_TYPE_DATA: &str = "data";

crates/livekit-webrtc/libwebrtc-sys/src/media_stream.cpp

@@ -0,0 +1,113 @@
+//
+// Created by Théo Monnom on 31/08/2022.
+//
+
+#include "livekit/media_stream.h"
+
+#include "api/media_stream_interface.h"
+#include "libwebrtc-sys/src/media_stream.rs.h"
+
+namespace livekit {
+
+MediaStreamTrack::MediaStreamTrack(
+    rtc::scoped_refptr<webrtc::MediaStreamTrackInterface> track)
+    : track_(std::move(track)) {}
+
+std::unique_ptr<MediaStreamTrack> MediaStreamTrack::from(
+    rtc::scoped_refptr<webrtc::MediaStreamTrackInterface> track) {
+  if (track->kind() == webrtc::MediaStreamTrackInterface::kVideoKind) {
+    return std::make_unique<VideoTrack>(
+        rtc::scoped_refptr<webrtc::VideoTrackInterface>(
+            static_cast<webrtc::VideoTrackInterface*>(track.get())));
+  } else {
+    return std::make_unique<AudioTrack>(
+        rtc::scoped_refptr<webrtc::AudioTrackInterface>(
+            static_cast<webrtc::AudioTrackInterface*>(track.get())));
+  }
+}
+
+rust::String MediaStreamTrack::kind() const {
+  return track_->kind();
+}
+
+rust::String MediaStreamTrack::id() const {
+  return track_->id();
+}
+
+bool MediaStreamTrack::enabled() const {
+  return track_->enabled();
+}
+
+bool MediaStreamTrack::set_enabled(bool enable) {
+  return track_->set_enabled(enable);
+}
+
+TrackState MediaStreamTrack::state() const {
+  return static_cast<TrackState>(track_->state());
+}
+
+MediaStream::MediaStream(
+    rtc::scoped_refptr<webrtc::MediaStreamInterface> stream)
+    : media_stream_(std::move(stream)) {}
+
+rust::String MediaStream::id() const {
+  return media_stream_->id();
+}
+
+AudioTrack::AudioTrack(rtc::scoped_refptr<webrtc::AudioTrackInterface> track)
+    : MediaStreamTrack(std::move(track)) {}
+
+VideoTrack::VideoTrack(rtc::scoped_refptr<webrtc::VideoTrackInterface> track)
+    : MediaStreamTrack(std::move(track)) {}
+
+void VideoTrack::add_sink(NativeVideoFrameSink& sink) {
+  track()->AddOrUpdateSink(&sink, rtc::VideoSinkWants());
+}
+
+void VideoTrack::remove_sink(NativeVideoFrameSink& sink) {
+  track()->RemoveSink(&sink);
+}
+
+void VideoTrack::set_should_receive(bool should_receive) {
+  track()->set_should_receive(should_receive);
+}
+
+bool VideoTrack::should_receive() const {
+  return track()->should_receive();
+}
+
+ContentHint VideoTrack::content_hint() const {
+  return static_cast<ContentHint>(track()->content_hint());
+}
+
+void VideoTrack::set_content_hint(ContentHint hint) {
+  track()->set_content_hint(
+      static_cast<webrtc::VideoTrackInterface::ContentHint>(hint));
+}
+
+NativeVideoFrameSink::NativeVideoFrameSink(
+    rust::Box<VideoFrameSinkWrapper> observer)
+    : observer_(std::move(observer)) {}
+
+void NativeVideoFrameSink::OnFrame(const webrtc::VideoFrame& frame) {
+  observer_->on_frame(std::make_unique<VideoFrame>(frame));
+}
+
+void NativeVideoFrameSink::OnDiscardedFrame() {
+  observer_->on_discarded_frame();
+}
+
+void NativeVideoFrameSink::OnConstraintsChanged(
+    const webrtc::VideoTrackSourceConstraints& constraints) {
+  VideoTrackSourceConstraints cst;
+  cst.min_fps = constraints.min_fps.value_or(-1);
+  cst.max_fps = constraints.max_fps.value_or(-1);
+  observer_->on_constraints_changed(cst);
+}
+
+std::unique_ptr<NativeVideoFrameSink> create_native_video_frame_sink(
+    rust::Box<VideoFrameSinkWrapper> observer) {
+  return std::make_unique<NativeVideoFrameSink>(std::move(observer));
+}
+
+}  // namespace livekit

crates/livekit-webrtc/libwebrtc-sys/src/media_stream.rs

@@ -0,0 +1,129 @@
+use cxx::UniquePtr;
+
+use crate::video_frame::ffi::VideoFrame;
+
+#[cxx::bridge(namespace = "livekit")]
+pub mod ffi {
+
+    #[derive(Debug)]
+    #[repr(i32)]
+    pub enum TrackState {
+        Live,
+        Ended,
+    }
+
+    #[derive(Debug)]
+    #[repr(i32)]
+    pub enum ContentHint {
+        None,
+        Fluid,
+        Detailed,
+        Text,
+    }
+
+    // -1 = optional
+    pub struct VideoTrackSourceConstraints {
+        pub min_fps: f64,
+        pub max_fps: f64,
+    }
+
+    unsafe extern "C++" {
+        include!("livekit/media_stream.h");
+        include!("livekit/video_frame.h");
+
+        type NativeVideoFrameSink;
+        type MediaStreamTrack;
+        type MediaStream;
+        type AudioTrack;
+        type VideoTrack;
+        type VideoFrame = crate::video_frame::ffi::VideoFrame;
+
+        fn id(self: &MediaStream) -> String;
+
+        fn kind(self: &MediaStreamTrack) -> String;
+        fn id(self: &MediaStreamTrack) -> String;
+        fn enabled(self: &MediaStreamTrack) -> bool;
+        fn set_enabled(self: Pin<&mut MediaStreamTrack>, enable: bool) -> bool;
+        fn state(self: &MediaStreamTrack) -> TrackState;
+
+        unsafe fn add_sink(self: Pin<&mut VideoTrack>, sink: Pin<&mut NativeVideoFrameSink>);
+        unsafe fn remove_sink(self: Pin<&mut VideoTrack>, sink: Pin<&mut NativeVideoFrameSink>);
+
+        fn set_should_receive(self: Pin<&mut VideoTrack>, should_receive: bool);
+        fn should_receive(self: &VideoTrack) -> bool;
+        fn content_hint(self: &VideoTrack) -> ContentHint;
+        fn set_content_hint(self: Pin<&mut VideoTrack>, hint: ContentHint);
+
+        fn create_native_video_frame_sink(
+            observer: Box<VideoFrameSinkWrapper>,
+        ) -> UniquePtr<NativeVideoFrameSink>;
+
+        unsafe fn video_to_media(track: *const VideoTrack) -> *const MediaStreamTrack;
+        unsafe fn audio_to_media(track: *const AudioTrack) -> *const MediaStreamTrack;
+        unsafe fn media_to_video(track: *const MediaStreamTrack) -> *const VideoTrack;
+        unsafe fn media_to_audio(track: *const MediaStreamTrack) -> *const AudioTrack;
+
+        fn _unique_media_stream_track() -> UniquePtr<MediaStreamTrack>; // Ignore
+        fn _unique_media_stream() -> UniquePtr<MediaStream>; // Ignore
+        fn _unique_audio_track() -> UniquePtr<AudioTrack>; // Ignore
+        fn _unique_video_track() -> UniquePtr<VideoTrack>; // Ignore
+    }
+
+    extern "Rust" {
+        type VideoFrameSinkWrapper;
+
+        fn on_frame(self: &VideoFrameSinkWrapper, frame: UniquePtr<VideoFrame>);
+        fn on_discarded_frame(self: &VideoFrameSinkWrapper);
+        fn on_constraints_changed(
+            self: &VideoFrameSinkWrapper,
+            constraints: VideoTrackSourceConstraints,
+        );
+    }
+}
+
+unsafe impl Sync for ffi::MediaStreamTrack {}
+unsafe impl Send for ffi::MediaStreamTrack {}
+unsafe impl Sync for ffi::MediaStream {}
+unsafe impl Send for ffi::MediaStream {}
+unsafe impl Send for ffi::AudioTrack {}
+unsafe impl Sync for ffi::AudioTrack {}
+unsafe impl Send for ffi::VideoTrack {}
+unsafe impl Sync for ffi::VideoTrack {}
+unsafe impl Send for ffi::NativeVideoFrameSink {}
+unsafe impl Sync for ffi::NativeVideoFrameSink {}
+
+pub trait VideoFrameSink: Send + Sync {
+    fn on_frame(&self, frame: UniquePtr<VideoFrame>);
+    fn on_discarded_frame(&self);
+    fn on_constraints_changed(&self, constraints: ffi::VideoTrackSourceConstraints);
+}
+
+pub struct VideoFrameSinkWrapper {
+    observer: *mut dyn VideoFrameSink,
+}
+
+impl VideoFrameSinkWrapper {
+    /// # Safety
+    /// VideoFrameSink must lives as long as VideoSinkInterfaceWrapper does
+    pub unsafe fn new(observer: *mut dyn VideoFrameSink) -> Self {
+        Self { observer }
+    }
+
+    fn on_frame(&self, frame: UniquePtr<VideoFrame>) {
+        unsafe {
+            (*self.observer).on_frame(frame);
+        }
+    }
+
+    fn on_discarded_frame(&self) {
+        unsafe {
+            (*self.observer).on_discarded_frame();
+        }
+    }
+
+    fn on_constraints_changed(&self, constraints: ffi::VideoTrackSourceConstraints) {
+        unsafe {
+            (*self.observer).on_constraints_changed(constraints);
+        }
+    }
+}

crates/livekit-webrtc/libwebrtc-sys/src/media_stream_interface.cpp

@@ -1,12 +0,0 @@
-//
-// Created by Théo Monnom on 31/08/2022.
-//
-
-#include "livekit/media_stream_interface.h"
-
-namespace livekit {
-
-MediaStreamInterface::MediaStreamInterface(
-    rtc::scoped_refptr<webrtc::MediaStreamInterface> stream)
-    : media_stream_(std::move(stream)) {}
-}  // namespace livekit

crates/livekit-webrtc/libwebrtc-sys/src/media_stream_interface.rs

@@ -1,10 +0,0 @@
-#[cxx::bridge(namespace = "livekit")]
-pub mod ffi {
-    unsafe extern "C++" {
-        include!("livekit/media_stream_interface.h");
-
-        type MediaStreamInterface;
-
-        fn _unique_media_stream() -> UniquePtr<MediaStreamInterface>; // Ignore
-    }
-}

crates/livekit-webrtc/libwebrtc-sys/src/peer_connection.cpp

@@ -3,6 +3,7 @@
 //
 
 #include "livekit/peer_connection.h"
+#include "livekit/media_stream.h"
 
 #include "libwebrtc-sys/src/peer_connection.rs.h"
 #include "livekit/rtc_error.h"
@@ -142,12 +143,12 @@ void NativePeerConnectionObserver::OnSignalingChange(
 
 void NativePeerConnectionObserver::OnAddStream(
     rtc::scoped_refptr<webrtc::MediaStreamInterface> stream) {
-  observer_->on_add_stream(std::make_unique<MediaStreamInterface>(stream));
+  observer_->on_add_stream(std::make_unique<MediaStream>(stream));
 }
 
 void NativePeerConnectionObserver::OnRemoveStream(
     rtc::scoped_refptr<webrtc::MediaStreamInterface> stream) {
-  observer_->on_remove_stream(std::make_unique<MediaStreamInterface>(stream));
+  observer_->on_remove_stream(std::make_unique<MediaStream>(stream));
 }
 
 void NativePeerConnectionObserver::OnDataChannel(
@@ -241,7 +242,7 @@ void NativePeerConnectionObserver::OnAddTrack(
   rust::Vec<MediaStreamPtr> vec;
 
   for (const auto& item : streams) {
-    vec.push_back(MediaStreamPtr{std::make_unique<MediaStreamInterface>(item)});
+    vec.push_back(MediaStreamPtr{std::make_unique<MediaStream>(item)});
   }
 
   observer_->on_add_track(std::make_unique<RtpReceiver>(receiver),

crates/livekit-webrtc/libwebrtc-sys/src/peer_connection.rs

@@ -6,7 +6,7 @@ use cxx::UniquePtr;
 use crate::candidate::ffi::Candidate;
 use crate::data_channel::ffi::DataChannel;
 use crate::jsep::ffi::IceCandidate;
-use crate::media_stream_interface::ffi::MediaStreamInterface;
+use crate::media_stream::ffi::MediaStream;
 use crate::rtc_error::ffi::RTCError;
 use crate::rtp_receiver::ffi::RtpReceiver;
 use crate::rtp_transceiver::ffi::RtpTransceiver;
@@ -83,7 +83,7 @@ pub mod ffi {
     // Wrapper to opaque C++ objects
     // https://github.com/dtolnay/cxx/issues/741
     struct MediaStreamPtr {
-        pub ptr: UniquePtr<MediaStreamInterface>,
+        pub ptr: UniquePtr<MediaStream>,
     }
 
     struct CandidatePtr {
@@ -96,7 +96,7 @@ pub mod ffi {
         include!("livekit/data_channel.h");
         include!("livekit/rtp_receiver.h");
         include!("livekit/rtp_transceiver.h");
-        include!("livekit/media_stream_interface.h");
+        include!("livekit/media_stream.h");
         include!("livekit/candidate.h");
         include!("libwebrtc-sys/src/rtc_error.rs.h");
 
@@ -106,7 +106,7 @@ pub mod ffi {
         type DataChannel = crate::data_channel::ffi::DataChannel;
         type RtpReceiver = crate::rtp_receiver::ffi::RtpReceiver;
         type RtpTransceiver = crate::rtp_transceiver::ffi::RtpTransceiver;
-        type MediaStreamInterface = crate::media_stream_interface::ffi::MediaStreamInterface;
+        type MediaStream = crate::media_stream::ffi::MediaStream;
         type NativeCreateSdpObserverHandle = crate::jsep::ffi::NativeCreateSdpObserverHandle;
         type NativeSetLocalSdpObserverHandle = crate::jsep::ffi::NativeSetLocalSdpObserverHandle;
         type NativeSetRemoteSdpObserverHandle = crate::jsep::ffi::NativeSetRemoteSdpObserverHandle;
@@ -194,14 +194,8 @@ pub mod ffi {
         type PeerConnectionObserverWrapper;
 
         fn on_signaling_change(self: &PeerConnectionObserverWrapper, new_state: SignalingState);
-        fn on_add_stream(
-            self: &PeerConnectionObserverWrapper,
-            stream: UniquePtr<MediaStreamInterface>,
-        );
-        fn on_remove_stream(
-            self: &PeerConnectionObserverWrapper,
-            stream: UniquePtr<MediaStreamInterface>,
-        );
+        fn on_add_stream(self: &PeerConnectionObserverWrapper, stream: UniquePtr<MediaStream>);
+        fn on_remove_stream(self: &PeerConnectionObserverWrapper, stream: UniquePtr<MediaStream>);
         fn on_data_channel(
             self: &PeerConnectionObserverWrapper,
             data_channel: UniquePtr<DataChannel>,
@@ -317,8 +311,8 @@ impl AddIceCandidateObserverWrapper {
 
 pub trait PeerConnectionObserver: Send + Sync {
     fn on_signaling_change(&self, new_state: ffi::SignalingState);
-    fn on_add_stream(&self, stream: UniquePtr<MediaStreamInterface>);
-    fn on_remove_stream(&self, stream: UniquePtr<MediaStreamInterface>);
+    fn on_add_stream(&self, stream: UniquePtr<MediaStream>);
+    fn on_remove_stream(&self, stream: UniquePtr<MediaStream>);
     fn on_data_channel(&self, data_channel: UniquePtr<DataChannel>);
     fn on_renegotiation_needed(&self);
     fn on_negotiation_needed_event(&self, event: u32);
@@ -338,11 +332,7 @@ pub trait PeerConnectionObserver: Send + Sync {
     fn on_ice_candidates_removed(&self, removed: Vec<UniquePtr<Candidate>>);
     fn on_ice_connection_receiving_change(&self, receiving: bool);
     fn on_ice_selected_candidate_pair_changed(&self, event: ffi::CandidatePairChangeEvent);
-    fn on_add_track(
-        &self,
-        receiver: UniquePtr<RtpReceiver>,
-        streams: Vec<UniquePtr<MediaStreamInterface>>,
-    );
+    fn on_add_track(&self, receiver: UniquePtr<RtpReceiver>, streams: Vec<UniquePtr<MediaStream>>);
     fn on_track(&self, transceiver: UniquePtr<RtpTransceiver>);
     fn on_remove_track(&self, receiver: UniquePtr<RtpReceiver>);
     fn on_interesting_usage(&self, usage_pattern: i32);
@@ -354,7 +344,7 @@ pub struct PeerConnectionObserverWrapper {
 }
 
 impl PeerConnectionObserverWrapper {
-    /// SAFETY
+    /// # Safety
     /// PeerConnectionObserver must lives as long as PeerConnectionObserverWrapper does
     pub unsafe fn new(observer: *mut dyn PeerConnectionObserver) -> Self {
         Self { observer }
@@ -366,13 +356,13 @@ impl PeerConnectionObserverWrapper {
         }
     }
 
-    fn on_add_stream(&self, stream: UniquePtr<MediaStreamInterface>) {
+    fn on_add_stream(&self, stream: UniquePtr<MediaStream>) {
         unsafe {
             (*self.observer).on_add_stream(stream);
         }
     }
 
-    fn on_remove_stream(&self, stream: UniquePtr<MediaStreamInterface>) {
+    fn on_remove_stream(&self, stream: UniquePtr<MediaStream>) {
         unsafe {
             (*self.observer).on_remove_stream(stream);
         }

crates/livekit-webrtc/libwebrtc-sys/src/peer_connection_factory.cpp

@@ -27,10 +27,12 @@ PeerConnectionFactory::PeerConnectionFactory(
   dependencies.network_thread = rtc_runtime_->network_thread();
   dependencies.worker_thread = rtc_runtime_->worker_thread();
   dependencies.signaling_thread = rtc_runtime_->signaling_thread();
+  dependencies.socket_factory = rtc_runtime_->network_thread()->socketserver();
   dependencies.task_queue_factory = webrtc::CreateDefaultTaskQueueFactory();
   dependencies.event_log_factory = std::make_unique<webrtc::RtcEventLogFactory>(
       dependencies.task_queue_factory.get());
   dependencies.call_factory = webrtc::CreateCallFactory();
+  dependencies.trials = std::make_unique<webrtc::FieldTrialBasedConfig>();
 
   cricket::MediaEngineDependencies media_deps;
   media_deps.task_queue_factory = dependencies.task_queue_factory.get();
@@ -38,6 +40,8 @@ PeerConnectionFactory::PeerConnectionFactory(
   media_deps.video_decoder_factory = webrtc::CreateBuiltinVideoDecoderFactory();
   media_deps.audio_encoder_factory = webrtc::CreateBuiltinAudioEncoderFactory();
   media_deps.audio_decoder_factory = webrtc::CreateBuiltinAudioDecoderFactory();
+  media_deps.audio_processing = webrtc::AudioProcessingBuilder().Create();
+  media_deps.trials = dependencies.trials.get();
 
   dependencies.media_engine = cricket::CreateMediaEngine(std::move(media_deps));
 
@@ -96,4 +100,4 @@ std::unique_ptr<NativeRTCConfiguration> create_rtc_configuration(
 
   return rtc;
 }
-}  // namespace livekit
+}  // namespace livekit

crates/livekit-webrtc/libwebrtc-sys/src/peer_connection_factory.rs

@@ -35,15 +35,17 @@ pub mod ffi {
 
         type PeerConnection = crate::peer_connection::ffi::PeerConnection;
         type NativePeerConnectionObserver =
-        crate::peer_connection::ffi::NativePeerConnectionObserver;
+            crate::peer_connection::ffi::NativePeerConnectionObserver;
         type PeerConnectionFactory;
         type NativeRTCConfiguration;
         type RTCRuntime = crate::webrtc::ffi::RTCRuntime;
 
-        fn create_peer_connection_factory(runtime: SharedPtr<RTCRuntime>) -> UniquePtr<PeerConnectionFactory>;
+        fn create_peer_connection_factory(
+            runtime: SharedPtr<RTCRuntime>,
+        ) -> UniquePtr<PeerConnectionFactory>;
         fn create_rtc_configuration(conf: RTCConfiguration) -> UniquePtr<NativeRTCConfiguration>;
 
-        /// SAFETY
+        /// # Safety
         /// The observer must live as long as the PeerConnection
         unsafe fn create_peer_connection(
             self: &PeerConnectionFactory,

crates/livekit-webrtc/libwebrtc-sys/src/rtp_receiver.cpp

@@ -8,4 +8,9 @@ namespace livekit {
 RtpReceiver::RtpReceiver(
     rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver)
     : receiver_(std::move(receiver)) {}
+
+std::unique_ptr<MediaStreamTrack> RtpReceiver::track() const {
+  return MediaStreamTrack::from(receiver_->track());
+}
+
 }  // namespace livekit

crates/livekit-webrtc/libwebrtc-sys/src/rtp_receiver.rs

@@ -2,9 +2,17 @@
 pub mod ffi {
     unsafe extern "C++" {
         include!("livekit/rtp_receiver.h");
+        include!("livekit/media_stream.h");
 
+        type MediaStreamTrack = crate::media_stream::ffi::MediaStreamTrack;
         type RtpReceiver;
 
+        fn track(self: &RtpReceiver) -> UniquePtr<MediaStreamTrack>;
+
         fn _unique_rtp_receiver() -> UniquePtr<RtpReceiver>; // Ignore
     }
 }
+
+unsafe impl Sync for ffi::RtpReceiver {}
+
+unsafe impl Send for ffi::RtpReceiver {}

crates/livekit-webrtc/libwebrtc-sys/src/video_frame.rs

@@ -0,0 +1,32 @@
+#[cxx::bridge(namespace = "livekit")]
+pub mod ffi {
+    #[derive(Debug)]
+    #[repr(i32)]
+    pub enum VideoRotation {
+        VideoRotation0 = 0,
+        VideoRotation90 = 90,
+        VideoRotation180 = 180,
+        VideoRotation270 = 270,
+    }
+
+    unsafe extern "C++" {
+        include!("livekit/video_frame.h");
+        include!("livekit/video_frame_buffer.h");
+
+        type VideoFrame;
+        type VideoFrameBuffer = crate::video_frame_buffer::ffi::VideoFrameBuffer;
+
+        fn width(self: &VideoFrame) -> i32;
+        fn height(self: &VideoFrame) -> i32;
+        fn size(self: &VideoFrame) -> u32;
+        fn id(self: &VideoFrame) -> u16;
+        fn timestamp_us(self: &VideoFrame) -> i64;
+        fn ntp_time_ms(self: &VideoFrame) -> i64;
+        fn transport_frame_id(self: &VideoFrame) -> u32;
+        fn timestamp(self: &VideoFrame) -> u32;
+        fn rotation(self: &VideoFrame) -> VideoRotation;
+        fn video_frame_buffer(self: &VideoFrame) -> UniquePtr<VideoFrameBuffer>;
+
+        fn _unique_video_frame() -> UniquePtr<VideoFrame>; // Ignore
+    }
+}

crates/livekit-webrtc/libwebrtc-sys/src/video_frame_buffer.rs

@@ -0,0 +1,48 @@
+#[cxx::bridge(namespace = "livekit")]
+pub mod ffi {
+    #[derive(Debug)]
+    #[repr(i32)]
+    pub enum VideoFrameBufferType {
+        Native,
+        I420,
+        I420A,
+        I422,
+        I444,
+        I010,
+        NV12,
+    }
+
+    unsafe extern "C++" {
+        include!("livekit/video_frame_buffer.h");
+
+        type VideoFrameBuffer;
+        type PlanarYuvBuffer;
+        type PlanarYuv8Buffer;
+        type I420Buffer;
+
+        fn buffer_type(self: &VideoFrameBuffer) -> VideoFrameBufferType;
+        fn width(self: &VideoFrameBuffer) -> i32;
+        fn height(self: &VideoFrameBuffer) -> i32;
+
+        // Require ownership
+        unsafe fn to_i420(self: Pin<&mut VideoFrameBuffer>) -> UniquePtr<I420Buffer>;
+        unsafe fn get_i420(self: Pin<&mut VideoFrameBuffer>) -> UniquePtr<I420Buffer>;
+        // TODO(theomonnom): Bridge other get_*
+
+        fn chroma_width(self: &PlanarYuvBuffer) -> i32;
+        fn chroma_height(self: &PlanarYuvBuffer) -> i32;
+        fn stride_y(self: &PlanarYuvBuffer) -> i32;
+        fn stride_u(self: &PlanarYuvBuffer) -> i32;
+        fn stride_v(self: &PlanarYuvBuffer) -> i32;
+
+        fn data_y(self: &PlanarYuv8Buffer) -> *const u8;
+        fn data_u(self: &PlanarYuv8Buffer) -> *const u8;
+        fn data_v(self: &PlanarYuv8Buffer) -> *const u8;
+
+        unsafe fn yuv_to_vfb(yuv: *const PlanarYuvBuffer) -> *const VideoFrameBuffer;
+        unsafe fn yuv8_to_yuv(yuv8: *const PlanarYuv8Buffer) -> *const PlanarYuvBuffer;
+        unsafe fn i420_to_yuv8(i420: *const I420Buffer) -> *const PlanarYuv8Buffer;
+
+        fn _unique_video_frame_buffer() -> UniquePtr<VideoFrameBuffer>;
+    }
+}

crates/livekit-webrtc/libwebrtc-sys/src/yuv_helper.rs

@@ -0,0 +1,19 @@
+#[cxx::bridge(namespace = "livekit")]
+pub mod ffi {
+    unsafe extern "C++" {
+        include!("livekit/yuv_helper.h");
+
+        unsafe fn i420_to_abgr(
+            src_y: *const u8,
+            src_stride_y: i32,
+            src_u: *const u8,
+            src_stride_u: i32,
+            src_v: *const u8,
+            src_stride_v: i32,
+            dst_abgr: *mut u8,
+            dst_stride_abgr: i32,
+            width: i32,
+            height: i32,
+        );
+    }
+}

crates/livekit-webrtc/src/data_channel.rs

@@ -18,7 +18,9 @@ pub struct DataChannel {
 
 impl Debug for DataChannel {
     fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
-        write!(f, "DataChannel[{}]", self.label())
+        f.debug_struct("DataChannel")
+            .field("label", &self.label())
+            .finish()
     }
 }
 
@@ -99,15 +101,21 @@ impl DataChannel {
     }
 }
 
+impl Drop for DataChannel {
+    fn drop(&mut self) {
+        self.cxx_handle.pin_mut().unregister_observer();
+    }
+}
+
 pub type OnStateChangeHandler = Box<dyn FnMut() + Send + Sync>;
 pub type OnMessageHandler = Box<dyn FnMut(&[u8], bool) + Send + Sync>;
-// data, is_binary
 pub type OnBufferedAmountChangeHandler = Box<dyn FnMut(u64) + Send + Sync>;
 
+#[derive(Default)]
 struct InternalDataChannelObserver {
-    on_state_change_handler: Arc<Mutex<Option<OnStateChangeHandler>>>,
-    on_message_handler: Arc<Mutex<Option<OnMessageHandler>>>,
-    on_buffered_amount_change_handler: Arc<Mutex<Option<OnBufferedAmountChangeHandler>>>,
+    on_state_change_handler: Mutex<Option<OnStateChangeHandler>>,
+    on_message_handler: Mutex<Option<OnMessageHandler>>,
+    on_buffered_amount_change_handler: Mutex<Option<OnBufferedAmountChangeHandler>>,
 }
 
 impl sys_dc::DataChannelObserver for InternalDataChannelObserver {
@@ -136,16 +144,6 @@ impl sys_dc::DataChannelObserver for InternalDataChannelObserver {
     }
 }
 
-impl Default for InternalDataChannelObserver {
-    fn default() -> Self {
-        Self {
-            on_state_change_handler: Arc::new(Default::default()),
-            on_message_handler: Arc::new(Default::default()),
-            on_buffered_amount_change_handler: Arc::new(Default::default()),
-        }
-    }
-}
-
 #[derive(Debug)]
 pub struct DataChannelInit {
     #[deprecated]

crates/livekit-webrtc/src/jsep.rs

@@ -1,4 +1,4 @@
-use std::fmt::{Debug, Display, Formatter};
+use std::fmt::{Debug, Formatter};
 
 use cxx::UniquePtr;
 
@@ -17,8 +17,16 @@ impl Debug for IceCandidate {
 }
 
 impl IceCandidate {
-    pub fn from(sdp_mid: &str, sdp_mline_index: i32, sdp: &str) -> Result<IceCandidate, SdpParseError> {
-        let res = sys_jsep::ffi::create_ice_candidate(sdp_mid.to_string(), sdp_mline_index, sdp.to_string());
+    pub fn from(
+        sdp_mid: &str,
+        sdp_mline_index: i32,
+        sdp: &str,
+    ) -> Result<IceCandidate, SdpParseError> {
+        let res = sys_jsep::ffi::create_ice_candidate(
+            sdp_mid.to_string(),
+            sdp_mline_index,
+            sdp.to_string(),
+        );
 
         match res {
             Ok(cxx_handle) => Ok(IceCandidate::new(cxx_handle)),

crates/livekit-webrtc/src/lib.rs

@@ -6,4 +6,7 @@ pub mod peer_connection_factory;
 pub mod rtc_error;
 pub mod rtp_receiver;
 pub mod rtp_transceiver;
+pub mod video_frame;
+pub mod video_frame_buffer;
 pub mod webrtc;
+pub mod yuv_helper;

crates/livekit-webrtc/src/media_stream.rs

@@ -1,2 +1,278 @@
-#[derive(Debug)]
-pub struct MediaStream {}
+use cxx::UniquePtr;
+use libwebrtc_sys::media_stream as sys_ms;
+use libwebrtc_sys::MEDIA_TYPE_VIDEO;
+use std::fmt::{Debug, Formatter};
+use std::pin::Pin;
+use std::sync::{Arc, Mutex};
+
+pub use sys_ms::ffi::ContentHint;
+pub use sys_ms::ffi::TrackState;
+
+use crate::video_frame::VideoFrame;
+use crate::video_frame_buffer::VideoFrameBuffer;
+
+pub trait MediaStreamTrackTrait {
+    fn kind(&self) -> String;
+    fn id(&self) -> String;
+    fn enabled(&self) -> bool;
+    fn set_enabled(&self, enabled: bool) -> bool;
+    fn state(&self) -> TrackState;
+}
+
+#[derive(Clone)]
+pub enum MediaStreamTrackHandle {
+    Audio(Arc<AudioTrack>),
+    Video(Arc<VideoTrack>),
+}
+
+macro_rules! shared_getter {
+    ($x:ident, $ret:ty) => {
+        fn $x(&self) -> $ret {
+            match self {
+                Self::Video(inner) => inner.$x(),
+                Self::Audio(inner) => inner.$x(),
+            }
+        }
+    };
+}
+
+impl MediaStreamTrackHandle {
+    pub(crate) fn new(cxx_handle: UniquePtr<sys_ms::ffi::MediaStreamTrack>) -> Self {
+        unsafe {
+            if cxx_handle.kind() == MEDIA_TYPE_VIDEO {
+                Self::Video(VideoTrack::new(UniquePtr::from_raw(
+                    sys_ms::ffi::media_to_video(cxx_handle.into_raw())
+                        as *mut sys_ms::ffi::VideoTrack,
+                )))
+            } else {
+                Self::Audio(AudioTrack::new(UniquePtr::from_raw(
+                    sys_ms::ffi::media_to_audio(cxx_handle.into_raw())
+                        as *mut sys_ms::ffi::AudioTrack,
+                )))
+            }
+        }
+    }
+}
+
+impl Debug for MediaStreamTrackHandle {
+    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
+        f.debug_struct("MediaStreamTrack")
+            .field("id", &self.id())
+            .field("kind", &self.kind())
+            .field("enabled", &self.enabled())
+            .field("state", &self.state())
+            .finish()
+    }
+}
+
+impl MediaStreamTrackTrait for MediaStreamTrackHandle {
+    shared_getter!(kind, String);
+    shared_getter!(id, String);
+    shared_getter!(enabled, bool);
+    shared_getter!(state, TrackState);
+
+    fn set_enabled(&self, enabled: bool) -> bool {
+        match self {
+            Self::Video(inner) => inner.set_enabled(enabled),
+            Self::Audio(inner) => inner.set_enabled(enabled),
+        }
+    }
+}
+
+pub struct AudioTrack {
+    cxx_handle: Mutex<UniquePtr<sys_ms::ffi::AudioTrack>>,
+}
+
+impl AudioTrack {
+    pub(crate) fn new(cxx_handle: UniquePtr<sys_ms::ffi::AudioTrack>) -> Arc<Self> {
+        Arc::new(Self {
+            cxx_handle: Mutex::new(cxx_handle),
+        })
+    }
+}
+
+pub struct VideoTrack {
+    cxx_handle: Mutex<UniquePtr<sys_ms::ffi::VideoTrack>>,
+    observer: Box<InternalVideoTrackSink>,
+
+    // Keep alive for c++
+    native_observer: UniquePtr<sys_ms::ffi::NativeVideoFrameSink>,
+}
+
+macro_rules! impl_media_stream_track_trait {
+    ($x:ty, $cast:ident) => {
+        impl MediaStreamTrackTrait for $x {
+            fn kind(&self) -> String {
+                unsafe { (*sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())).kind() }
+            }
+
+            fn id(&self) -> String {
+                unsafe { (*sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())).id() }
+            }
+
+            fn enabled(&self) -> bool {
+                unsafe { (*sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())).enabled() }
+            }
+
+            fn set_enabled(&self, enabled: bool) -> bool {
+                unsafe {
+                    let media = sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())
+                        as *mut sys_ms::ffi::MediaStreamTrack;
+
+                    Pin::new_unchecked(&mut *media).set_enabled(enabled)
+                }
+            }
+
+            fn state(&self) -> TrackState {
+                unsafe { (*sys_ms::ffi::$cast(&**self.cxx_handle.lock().unwrap())).state() }
+            }
+        }
+    };
+}
+
+impl_media_stream_track_trait!(VideoTrack, video_to_media);
+impl_media_stream_track_trait!(AudioTrack, audio_to_media);
+
+pub type OnFrameHandler = Box<dyn FnMut(VideoFrame, VideoFrameBuffer) + Send + Sync>;
+pub type OnDiscardedFrameHandler = Box<dyn FnMut() + Send + Sync>;
+pub type OnConstraintsChanged = Box<dyn FnMut(VideoTrackSourceConstraints) + Send + Sync>;
+
+#[derive(Default)]
+struct InternalVideoTrackSink {
+    on_frame_handler: Mutex<Option<OnFrameHandler>>,
+    on_discarded_frame_handler: Mutex<Option<OnDiscardedFrameHandler>>,
+    on_constraints_changed_handler: Mutex<Option<OnConstraintsChanged>>,
+}
+
+pub struct VideoTrackSourceConstraints {
+    pub min_fps: Option<f64>,
+    pub max_fps: Option<f64>,
+}
+
+impl From<sys_ms::ffi::VideoTrackSourceConstraints> for VideoTrackSourceConstraints {
+    fn from(cst: sys_ms::ffi::VideoTrackSourceConstraints) -> Self {
+        Self {
+            min_fps: (cst.min_fps != 1.0).then_some(cst.min_fps),
+            max_fps: (cst.max_fps != 1.0).then_some(cst.max_fps),
+        }
+    }
+}
+
+impl sys_ms::VideoFrameSink for InternalVideoTrackSink {
+    fn on_frame(&self, frame: UniquePtr<libwebrtc_sys::video_frame::ffi::VideoFrame>) {
+        if let Some(cb) = self.on_frame_handler.lock().unwrap().as_mut() {
+            let frame = VideoFrame::new(frame);
+            let video_frame_buffer = unsafe { frame.video_frame_buffer() };
+            cb(frame, video_frame_buffer);
+        }
+    }
+
+    fn on_discarded_frame(&self) {
+        if let Some(cb) = self.on_discarded_frame_handler.lock().unwrap().as_mut() {
+            cb();
+        }
+    }
+
+    fn on_constraints_changed(&self, constraints: sys_ms::ffi::VideoTrackSourceConstraints) {
+        if let Some(cb) = self.on_constraints_changed_handler.lock().unwrap().as_mut() {
+            cb(constraints.into());
+        }
+    }
+}
+
+impl VideoTrack {
+    fn new(cxx_handle: UniquePtr<sys_ms::ffi::VideoTrack>) -> Arc<Self> {
+        let mut observer = Box::new(InternalVideoTrackSink::default());
+
+        let mut track = unsafe {
+            Self {
+                cxx_handle: Mutex::new(cxx_handle),
+                native_observer: sys_ms::ffi::create_native_video_frame_sink(Box::new(
+                    sys_ms::VideoFrameSinkWrapper::new(&mut *observer),
+                )),
+                observer,
+            }
+        };
+
+        unsafe {
+            track
+                .cxx_handle
+                .lock()
+                .unwrap()
+                .pin_mut()
+                .add_sink(track.native_observer.pin_mut());
+        }
+
+        Arc::new(track)
+    }
+
+    pub fn set_should_receive(&self, should_receive: bool) {
+        self.cxx_handle
+            .lock()
+            .unwrap()
+            .pin_mut()
+            .set_should_receive(should_receive)
+    }
+
+    pub fn set_content_hint(&self, hint: ContentHint) {
+        self.cxx_handle
+            .lock()
+            .unwrap()
+            .pin_mut()
+            .set_content_hint(hint)
+    }
+
+    pub fn should_receive(&self) -> bool {
+        self.cxx_handle.lock().unwrap().should_receive()
+    }
+
+    pub fn content_hint(&self) -> ContentHint {
+        self.cxx_handle.lock().unwrap().content_hint()
+    }
+
+    pub fn on_frame(&self, handler: OnFrameHandler) {
+        *self.observer.on_frame_handler.lock().unwrap() = Some(handler);
+    }
+
+    pub fn on_discarded_frame(&self, handler: OnDiscardedFrameHandler) {
+        *self.observer.on_discarded_frame_handler.lock().unwrap() = Some(handler);
+    }
+
+    pub fn on_constraints_changed(&self, handler: OnConstraintsChanged) {
+        *self.observer.on_constraints_changed_handler.lock().unwrap() = Some(handler);
+    }
+}
+
+impl Drop for VideoTrack {
+    fn drop(&mut self) {
+        unsafe {
+            self.cxx_handle
+                .lock()
+                .unwrap()
+                .pin_mut()
+                .remove_sink(self.native_observer.pin_mut());
+        }
+    }
+}
+
+pub struct MediaStream {
+    cxx_handle: UniquePtr<sys_ms::ffi::MediaStream>,
+}
+
+impl Debug for MediaStream {
+    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
+        f.debug_struct("MediaStream")
+            .field("id", &self.id())
+            .finish()
+    }
+}
+
+impl MediaStream {
+    pub(crate) fn new(cxx_handle: UniquePtr<sys_ms::ffi::MediaStream>) -> Self {
+        Self { cxx_handle }
+    }
+
+    pub fn id(&self) -> String {
+        self.cxx_handle.id()
+    }
+}

crates/livekit-webrtc/src/peer_connection.rs

@@ -1,3 +1,4 @@
+use std::fmt::{Debug, Formatter};
 use std::mem::ManuallyDrop;
 use std::sync::{Arc, Mutex};
 
@@ -17,7 +18,7 @@ pub use libwebrtc_sys::peer_connection::ffi::SignalingState;
 
 use crate::data_channel::{DataChannel, DataChannelInit};
 use crate::jsep::{IceCandidate, SessionDescription};
-use crate::media_stream::MediaStream;
+use crate::media_stream::{MediaStream, VideoTrack, AudioTrack};
 use crate::rtc_error::RTCError;
 use crate::rtp_receiver::RtpReceiver;
 use crate::rtp_transceiver::RtpTransceiver;
@@ -31,6 +32,18 @@ pub struct PeerConnection {
     native_observer: UniquePtr<sys_pc::ffi::NativePeerConnectionObserver>,
 }
 
+impl Debug for PeerConnection {
+    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
+        f.debug_struct("PeerConnection")
+            .field("signaling_state", &self.signaling_state())
+            .field("ice_connection_state", &self.ice_connection_state())
+            .field("ice_gathering_state", &self.ice_gathering_state())
+            .field("local_description", &self.local_description())
+            .field("remote_description", &self.remote_description())
+            .finish()
+    }
+}
+
 impl PeerConnection {
     pub(crate) fn new(
         cxx_handle: UniquePtr<sys_pc::ffi::PeerConnection>,
@@ -402,10 +415,7 @@ impl sys_pc::PeerConnectionObserver for InternalObserver {
         }
     }
 
-    fn on_add_stream(
-        &self,
-        stream: UniquePtr<libwebrtc_sys::media_stream_interface::ffi::MediaStreamInterface>,
-    ) {
+    fn on_add_stream(&self, stream: UniquePtr<libwebrtc_sys::media_stream::ffi::MediaStream>) {
         trace!("on_add_stream");
         let mut handler = self.on_add_stream_handler.lock().unwrap();
         if let Some(f) = handler.as_mut() {
@@ -413,10 +423,7 @@ impl sys_pc::PeerConnectionObserver for InternalObserver {
         }
     }
 
-    fn on_remove_stream(
-        &self,
-        stream: UniquePtr<libwebrtc_sys::media_stream_interface::ffi::MediaStreamInterface>,
-    ) {
+    fn on_remove_stream(&self, stream: UniquePtr<libwebrtc_sys::media_stream::ffi::MediaStream>) {
         trace!("on_remove_stream");
         let mut handler = self.on_remove_stream_handler.lock().unwrap();
         if let Some(f) = handler.as_mut() {
@@ -452,7 +459,7 @@ impl sys_pc::PeerConnectionObserver for InternalObserver {
     }
 
     fn on_ice_connection_change(&self, new_state: IceConnectionState) {
-        trace!("on_ice_connection_change");
+        trace!("on_ice_connection_change (new_state: {:?})", new_state);
         let mut handler = self.on_ice_connection_change_handler.lock().unwrap();
         if let Some(f) = handler.as_mut() {
             f(new_state);
@@ -460,7 +467,7 @@ impl sys_pc::PeerConnectionObserver for InternalObserver {
     }
 
     fn on_standardized_ice_connection_change(&self, new_state: IceConnectionState) {
-        trace!("on_standardized_ice_connection_change");
+        trace!("on_standardized_ice_connection_change (new_state: {:?}", new_state);
         let mut handler = self
             .on_standardized_ice_connection_change_handler
             .lock()
@@ -471,7 +478,7 @@ impl sys_pc::PeerConnectionObserver for InternalObserver {
     }
 
     fn on_connection_change(&self, new_state: PeerConnectionState) {
-        trace!("on_connection_change");
+        trace!("on_connection_change (new_state: {:?})", new_state);
         let mut handler = self.on_connection_change_handler.lock().unwrap();
         if let Some(f) = handler.as_mut() {
             f(new_state);
@@ -479,7 +486,7 @@ impl sys_pc::PeerConnectionObserver for InternalObserver {
     }
 
     fn on_ice_gathering_change(&self, new_state: IceGatheringState) {
-        trace!("on_ice_gathering_change");
+        trace!("on_ice_gathering_change (new_state: {:?}", new_state);
         let mut handler = self.on_ice_gathering_change_handler.lock().unwrap();
         if let Some(f) = handler.as_mut() {
             f(new_state);
@@ -548,12 +555,13 @@ impl sys_pc::PeerConnectionObserver for InternalObserver {
     fn on_add_track(
         &self,
         receiver: UniquePtr<libwebrtc_sys::rtp_receiver::ffi::RtpReceiver>,
-        streams: Vec<UniquePtr<libwebrtc_sys::media_stream_interface::ffi::MediaStreamInterface>>,
+        streams: Vec<UniquePtr<libwebrtc_sys::media_stream::ffi::MediaStream>>,
     ) {
         trace!("on_add_track");
         let mut handler = self.on_add_track_handler.lock().unwrap();
         if let Some(f) = handler.as_mut() {
-            // TODO(theomonnom)
+            let streams = streams.into_iter().map(MediaStream::new).collect();
+            f(RtpReceiver::new(receiver), streams)
         }
     }
 

crates/livekit-webrtc/src/peer_connection_factory.rs

@@ -18,7 +18,9 @@ pub struct PeerConnectionFactory {
 impl PeerConnectionFactory {
     pub fn new(rtc_runtime: RTCRuntime) -> Self {
         Self {
-            cxx_handle: sys_factory::ffi::create_peer_connection_factory(rtc_runtime.clone().release()),
+            cxx_handle: sys_factory::ffi::create_peer_connection_factory(
+                rtc_runtime.clone().release(),
+            ),
             rtc_runtime,
         }
     }
@@ -31,8 +33,9 @@ impl PeerConnectionFactory {
 
         unsafe {
             let mut observer = Box::new(InternalObserver::default());
-            let mut native_observer = sys_pc::ffi::create_native_peer_connection_observer(self.rtc_runtime.clone().release(),
-                                                                                          Box::new(sys_pc::PeerConnectionObserverWrapper::new(&mut *observer)),
+            let mut native_observer = sys_pc::ffi::create_native_peer_connection_observer(
+                self.rtc_runtime.clone().release(),
+                Box::new(sys_pc::PeerConnectionObserverWrapper::new(&mut *observer)),
             );
 
             let res = self

crates/livekit-webrtc/src/rtc_error.rs

@@ -1,3 +1,2 @@
 // TODO(theomonnom) Wrap the RTCError ffi so we can use Option(u16)
 pub use libwebrtc_sys::rtc_error::ffi::RTCError;
-

crates/livekit-webrtc/src/rtp_receiver.rs

@@ -1,2 +1,28 @@
-#[derive(Debug)]
-pub struct RtpReceiver {}
+use crate::media_stream::{MediaStreamTrackHandle, MediaStreamTrackTrait};
+use cxx::UniquePtr;
+use libwebrtc_sys::rtp_receiver as sys_rec;
+use std::fmt::{Debug, Formatter};
+
+pub struct RtpReceiver {
+    cxx_handle: UniquePtr<sys_rec::ffi::RtpReceiver>,
+}
+
+impl Debug for RtpReceiver {
+    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
+        f.debug_struct("RtpReceiver")
+            .field("track", &self.track())
+            .finish()
+    }
+}
+
+impl RtpReceiver {
+    pub(crate) fn new(cxx_handle: UniquePtr<sys_rec::ffi::RtpReceiver>) -> Self {
+        Self {
+            cxx_handle,
+        }
+    }
+
+    pub fn track(&self) -> MediaStreamTrackHandle {
+        MediaStreamTrackHandle::new(self.cxx_handle.track())
+    }
+}

crates/livekit-webrtc/src/video_frame.rs

@@ -0,0 +1,60 @@
+use cxx::UniquePtr;
+use libwebrtc_sys::video_frame as vf_sys;
+
+pub use vf_sys::ffi::VideoRotation;
+
+use crate::video_frame_buffer::VideoFrameBuffer;
+
+pub struct VideoFrame {
+    cxx_handle: UniquePtr<vf_sys::ffi::VideoFrame>,
+}
+
+impl VideoFrame {
+    pub(crate) fn new(cxx_handle: UniquePtr<vf_sys::ffi::VideoFrame>) -> Self {
+        Self { cxx_handle }
+    }
+
+    pub fn width(&self) -> i32 {
+        self.cxx_handle.width()
+    }
+
+    pub fn height(&self) -> i32 {
+        self.cxx_handle.height()
+    }
+
+    pub fn size(&self) -> u32 {
+        self.cxx_handle.size()
+    }
+
+    pub fn id(&self) -> u16 {
+        self.cxx_handle.id()
+    }
+
+    pub fn timestamp_us(&self) -> i64 {
+        self.cxx_handle.timestamp_us()
+    }
+
+    pub fn ntp_time_ms(&self) -> i64 {
+        self.cxx_handle.ntp_time_ms()
+    }
+
+    pub fn transport_frame_id(&self) -> u32 {
+        self.cxx_handle.transport_frame_id()
+    }
+
+    pub fn timestamp(&self) -> u32 {
+        self.cxx_handle.timestamp()
+    }
+
+    pub fn rotation(&self) -> VideoRotation {
+        self.cxx_handle.rotation()
+    }
+
+    /// # Safety
+    /// Must be called only once, this function create the safe Rust
+    /// wrapper around a VideoFrameBuffer.
+    /// Only one wrapper musts exist at a time.
+    pub(crate) unsafe fn video_frame_buffer(&self) -> VideoFrameBuffer {
+        VideoFrameBuffer::new(self.cxx_handle.video_frame_buffer())
+    }
+}

crates/livekit-webrtc/src/video_frame_buffer.rs

@@ -0,0 +1,262 @@
+use cxx::UniquePtr;
+use libwebrtc_sys::video_frame_buffer as vfb_sys;
+use livekit_utils::enum_dispatch;
+use std::pin::Pin;
+use std::slice;
+use vfb_sys::ffi::VideoFrameBufferType;
+
+pub trait VideoFrameBufferTrait {
+    fn width(&self) -> i32;
+    fn height(&self) -> i32;
+    fn to_i420(self) -> I420Buffer;
+}
+
+pub trait PlanarYuvBuffer: VideoFrameBufferTrait {
+    fn chroma_width(&self) -> i32;
+    fn chroma_height(&self) -> i32;
+    fn stride_y(&self) -> i32;
+    fn stride_u(&self) -> i32;
+    fn stride_v(&self) -> i32;
+}
+
+pub trait PlanarYuv8Buffer: PlanarYuvBuffer {
+    fn data_y(&self) -> &[u8];
+    fn data_u(&self) -> &[u8];
+    fn data_v(&self) -> &[u8];
+}
+
+pub enum VideoFrameBuffer {
+    Native(NativeBuffer),
+    I420(I420Buffer),
+    I420A(I420ABuffer),
+    I422(I422Buffer),
+    I444(I444Buffer),
+    I010(I010Buffer),
+    NV12(NV12Buffer),
+}
+
+impl VideoFrameBuffer {
+    pub(crate) fn new(mut cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>) -> Self {
+        unsafe {
+            match cxx_handle.buffer_type() {
+                VideoFrameBufferType::Native => Self::Native(NativeBuffer::new(cxx_handle)),
+                VideoFrameBufferType::I420 => {
+                    Self::I420(I420Buffer::new(cxx_handle.pin_mut().get_i420()))
+                }
+                VideoFrameBufferType::I420A => Self::I420A(I420ABuffer::new(cxx_handle)),
+                VideoFrameBufferType::I422 => Self::I422(I422Buffer::new(cxx_handle)),
+                VideoFrameBufferType::I444 => Self::I444(I444Buffer::new(cxx_handle)),
+                VideoFrameBufferType::I010 => Self::I010(I010Buffer::new(cxx_handle)),
+                VideoFrameBufferType::NV12 => Self::NV12(NV12Buffer::new(cxx_handle)),
+                _ => unreachable!(), // VideoFrameBufferType is represented as i32
+            }
+        }
+    }
+}
+
+impl VideoFrameBufferTrait for VideoFrameBuffer {
+    enum_dispatch!(
+        [Native, I420, I420A, I422, I444, I010, NV12]
+        fnc!(width, &Self, [], i32);
+        fnc!(height, &Self, [], i32);
+        fnc!(to_i420, Self, [], I420Buffer);
+    );
+}
+
+macro_rules! recursive_cast {
+    ($ptr:expr $(, $fnc:ident)*) => {
+        {
+            let ptr = $ptr;
+            $(
+                let ptr = unsafe { vfb_sys::ffi::$fnc(ptr) };
+            )*
+            ptr
+        }
+    };
+}
+
+macro_rules! impl_video_frame_buffer {
+    ($x:ty $(, $cast:ident)*) => {
+
+        // Allow unused_unsafe when we don't do any cast ( e.g. NativeBuffer )
+        #[allow(unused_unsafe)]
+        impl VideoFrameBufferTrait for $x {
+            fn width(&self) -> i32 {
+                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
+                unsafe {
+                    (*ptr).width()
+                }
+            }
+
+            fn height(&self) -> i32 {
+                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
+                unsafe {
+                    (*ptr).height()
+                }
+            }
+
+            // Require ownership because libwebrtc uses the same pointers
+            fn to_i420(self) -> I420Buffer {
+                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*)
+                    as *const vfb_sys::ffi::VideoFrameBuffer
+                    as *mut vfb_sys::ffi::VideoFrameBuffer;
+
+                unsafe {
+                    I420Buffer::new(Pin::new_unchecked(&mut *ptr).to_i420())
+                }
+            }
+        }
+    };
+}
+
+macro_rules! impl_yuv_buffer {
+    ($x:ty $(, $cast:ident)*) => {
+        impl PlanarYuvBuffer for $x {
+            fn chroma_width(&self) -> i32 {
+                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
+                unsafe {
+                    (*ptr).chroma_width()
+                }
+            }
+
+            fn chroma_height(&self) -> i32 {
+                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
+                unsafe {
+                    (*ptr).chroma_height()
+                }
+            }
+
+            fn stride_y(&self) -> i32 {
+                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
+                unsafe {
+                    (*ptr).stride_y()
+                }
+            }
+
+            fn stride_u(&self) -> i32 {
+                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
+                unsafe {
+                    (*ptr).stride_u()
+                }
+            }
+
+            fn stride_v(&self) -> i32 {
+                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
+                unsafe {
+                    (*ptr).stride_v()
+                }
+            }
+        }
+    };
+}
+
+macro_rules! impl_yuv8_buffer {
+    ($x:ty $(, $cast:ident)*) => {
+        impl PlanarYuv8Buffer for $x {
+            fn data_y(&self) -> &[u8] {
+                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
+                unsafe {
+                    slice::from_raw_parts((*ptr).data_y(), (self.width() * self.height()) as usize)
+                }
+            }
+
+            fn data_u(&self) -> &[u8] {
+                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
+                unsafe {
+                    let chroma_height = (self.height() + 1) / 2;
+                    slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * chroma_height) as usize)
+                }
+            }
+
+            fn data_v(&self) -> &[u8] {
+                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
+                unsafe {
+                    let chroma_height = (self.height() + 1) / 2;
+                    slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * chroma_height) as usize)
+                }
+            }
+        }
+    };
+}
+
+pub struct NativeBuffer {
+    cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>,
+}
+
+pub struct I420Buffer {
+    cxx_handle: UniquePtr<vfb_sys::ffi::I420Buffer>,
+}
+
+pub struct I420ABuffer {
+    cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>,
+}
+
+pub struct I422Buffer {
+    cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>,
+}
+
+pub struct I444Buffer {
+    cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>,
+}
+
+pub struct I010Buffer {
+    cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>,
+}
+
+pub struct NV12Buffer {
+    cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>,
+}
+
+impl_video_frame_buffer!(NativeBuffer);
+impl_video_frame_buffer!(I420Buffer, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
+impl_video_frame_buffer!(I420ABuffer);
+impl_video_frame_buffer!(I422Buffer);
+impl_video_frame_buffer!(I444Buffer);
+impl_video_frame_buffer!(I010Buffer);
+impl_video_frame_buffer!(NV12Buffer);
+
+impl_yuv_buffer!(I420Buffer, i420_to_yuv8, yuv8_to_yuv);
+
+impl_yuv8_buffer!(I420Buffer, i420_to_yuv8);
+
+impl NativeBuffer {
+    fn new(cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>) -> Self {
+        Self { cxx_handle }
+    }
+}
+
+impl I420Buffer {
+    fn new(cxx_handle: UniquePtr<vfb_sys::ffi::I420Buffer>) -> Self {
+        Self { cxx_handle }
+    }
+}
+
+impl I420ABuffer {
+    fn new(cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>) -> Self {
+        Self { cxx_handle }
+    }
+}
+
+impl I422Buffer {
+    fn new(cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>) -> Self {
+        Self { cxx_handle }
+    }
+}
+
+impl I444Buffer {
+    fn new(cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>) -> Self {
+        Self { cxx_handle }
+    }
+}
+
+impl I010Buffer {
+    fn new(cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>) -> Self {
+        Self { cxx_handle }
+    }
+}
+
+impl NV12Buffer {
+    fn new(cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>) -> Self {
+        Self { cxx_handle }
+    }
+}

crates/livekit-webrtc/src/yuv_helper.rs

@@ -0,0 +1,43 @@
+use std::convert::TryInto;
+
+use libwebrtc_sys::yuv_helper as yuv_sys;
+
+pub fn i420_to_abgr(
+    src_y: &[u8],
+    src_stride_y: i32,
+    src_u: &[u8],
+    src_stride_u: i32,
+    src_v: &[u8],
+    src_stride_v: i32,
+    dst_abgr: &mut [u8],
+    dst_stride_abgr: i32,
+    width: i32,
+    height: i32,
+) {
+    // Assert minimum capacity for safety
+    let chroma_height = (height + 1) / 2; // the buffer should be padded?
+    let min_y: usize = (src_stride_y * height).try_into().unwrap();
+    let min_u: usize = (src_stride_u * chroma_height).try_into().unwrap();
+    let min_v: usize = (src_stride_v * chroma_height).try_into().unwrap();
+    let min_abgr: usize = (dst_stride_abgr * height).try_into().unwrap();
+
+    assert!(src_y.len() >= min_y);
+    assert!(src_u.len() >= min_u);
+    assert!(src_v.len() >= min_v);
+    assert!(dst_abgr.len() >= min_abgr);
+
+    unsafe {
+        yuv_sys::ffi::i420_to_abgr(
+            src_y.as_ptr(),
+            src_stride_y,
+            src_u.as_ptr(),
+            src_stride_u,
+            src_v.as_ptr(),
+            src_stride_v,
+            dst_abgr.as_mut_ptr(),
+            dst_stride_abgr,
+            width,
+            height,
+        );
+    }
+}

examples/Cargo.toml

@@ -1,3 +1,4 @@
 [workspace]
 members = ["*"]
-exclude = ["target"]
+exclude = ["target"]
+resolver = "2"

examples/simple_room/Cargo.toml

@@ -7,4 +7,12 @@ edition = "2021"
 tokio = { version = "1", features = ["full"] }
 tracing = "0.1"
 tracing-subscriber = "0.3"
-livekit = { path = "../.." }
+livekit = { path = "../.." }
+futures = "0.3"
+wgpu = "0.14.0"
+winit = "0.27.5"
+
+egui = { git = "https://github.com/emilk/egui" }
+egui-wgpu = { git = "https://github.com/emilk/egui", features = ["winit"] }
+egui-winit = { git = "https://github.com/emilk/egui" }
+egui_demo_lib = { git = "https://github.com/emilk/egui" }

examples/simple_room/src/main.rs

@@ -1,22 +1,192 @@
-use std::time::Duration;
+use std::convert::TryInto;
+use std::ops::DerefMut;
+use std::{num::NonZeroU32, time::Duration};
+
+use egui_wgpu::WgpuConfiguration;
+use livekit::webrtc::media_stream::VideoTrack;
+use livekit::webrtc::video_frame_buffer::{
+    PlanarYuv8Buffer, PlanarYuvBuffer, VideoFrameBufferTrait,
+};
+use livekit::webrtc::yuv_helper;
+use std::sync::{Arc, Mutex};
+use video_renderer::VideoRenderer;
+use wgpu::{Device, Queue};
+
 use tokio::time::sleep;
-use livekit::proto::data_packet;
-use livekit::room;
+
+use livekit::room::track::remote_track::RemoteTrackHandle;
+use livekit::room::{Room, RoomError};
 
 const URL: &str = "ws://localhost:7880";
 const TOKEN : &str = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjIzODQ4MDY0NzMsImlzcyI6IkFQSXpLYkFTaUNWYWtnSiIsIm5hbWUiOiJuYXRpdmUiLCJuYmYiOjE2NjQ4MDY0NzMsInN1YiI6Im5hdGl2ZSIsInZpZGVvIjp7InJvb21DcmVhdGUiOnRydWUsInJvb21Kb2luIjp0cnVlfX0.BgVdBnq3XFD3_BQHoe1azqjifYysubgFl6Qlzu9IQGI";
 
 // eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjIzODQ4MDY3MzAsImlzcyI6IkFQSXpLYkFTaUNWYWtnSiIsIm5hbWUiOiJ3ZWIiLCJuYmYiOjE2NjQ4MDY3MzAsInN1YiI6IndlYiIsInZpZGVvIjp7InJvb21DcmVhdGUiOnRydWUsInJvb21Kb2luIjp0cnVlfX0.VbDoULjX1CVGZu2sPy3SvWYlVZUBXxQVPmdB9BnmlN4
 
-#[tokio::main]
-async fn main() -> Result<(), room::RoomError> {
-    tracing_subscriber::fmt::init();
+mod video_renderer;
 
-    let mut room = room::connect(URL, TOKEN).await?;
-    room.local_participant()
-        .publish_data(b"some data", data_packet::Kind::Reliable)
-        .await?;
+use winit::{
+    event::*,
+    event_loop::{ControlFlow, EventLoop},
+    window::{Window, WindowBuilder, WindowId},
+};
 
-    sleep(Duration::from_secs(120)).await;
-    Ok(())
+struct AppState {
+    room: Room,
+    demo: egui_demo_lib::DemoWindows,
+    egui_context: egui::Context,
+    egui_state: egui_winit::State,
+    egui_painter: egui_wgpu::winit::Painter,
+    window: winit::window::Window,
+}
+
+impl AppState {
+    fn on_event<T>(&mut self, event: Event<'_, T>, control_flow: &mut ControlFlow) {
+        match event {
+            Event::WindowEvent { window_id, event } => {
+                if let Some(flow) = self.on_window_event(window_id, event) {
+                    *control_flow = flow;
+                }
+            }
+            Event::RedrawRequested(window_id) if window_id == self.window.id() => {
+                self.render();
+            }
+            Event::RedrawEventsCleared => {
+                self.window.request_redraw();
+            }
+            _ => {}
+        };
+    }
+
+    fn on_window_event(
+        &mut self,
+        _window_id: WindowId,
+        event: WindowEvent<'_>,
+    ) -> Option<ControlFlow> {
+        if self
+            .egui_state
+            .on_event(&self.egui_context, &event)
+            .consumed
+        {
+            return None;
+        }
+
+        match event {
+            WindowEvent::CloseRequested => Some(ControlFlow::Exit),
+            WindowEvent::Resized(inner_size) => {
+                self.egui_painter
+                    .on_window_resized(inner_size.width, inner_size.height);
+                None
+            }
+            WindowEvent::ScaleFactorChanged { new_inner_size, .. } => {
+                self.egui_painter
+                    .on_window_resized(new_inner_size.width, new_inner_size.height);
+                None
+            }
+            _ => None,
+        }
+    }
+
+    fn render(&mut self) {
+        let raw_inputs = self.egui_state.take_egui_input(&self.window);
+        let full_output = self.egui_context.run(raw_inputs, |ctx| {
+            //self.ui(ctx);
+        });
+        let clipped_primitives = self.egui_context.tessellate(full_output.shapes);
+
+        self.egui_painter.paint_and_update_textures(
+            egui_winit::native_pixels_per_point(&self.window),
+            egui::Rgba::BLACK,
+            &clipped_primitives,
+            &full_output.textures_delta,
+        );
+
+        self.egui_state.handle_platform_output(
+            &self.window,
+            &self.egui_context,
+            full_output.platform_output,
+        );
+    }
+}
+
+struct App {
+    rt: tokio::runtime::Runtime,
+}
+
+impl App {
+    pub fn new(rt: tokio::runtime::Runtime) -> Self {
+        Self { rt }
+    }
+
+    pub fn run(&mut self) {
+        self.rt.block_on(async {
+            let event_loop = EventLoop::new();
+            let window = WindowBuilder::new().build(&event_loop).unwrap();
+
+            let egui_context = egui::Context::default();
+            let egui_state = egui_winit::State::new(&event_loop);
+            let mut egui_painter =
+                egui_wgpu::winit::Painter::new(WgpuConfiguration::default(), 1, 32);
+            unsafe {
+                egui_painter.set_window(Some(&window));
+            }
+
+            let mut inner = AppState {
+                room: Room::new(),
+                demo: egui_demo_lib::DemoWindows::default(),
+                egui_context,
+                egui_state,
+                egui_painter,
+                window,
+            };
+
+            inner
+                .room
+                .events()
+                .on_participant_connected(|_event| async move {});
+
+            inner.room.events().on_track_subscribed({
+                let test = Arc::new(Mutex::new(None));
+
+                let egui_render = inner.egui_painter.render_state().clone().unwrap();
+
+                move |event| {
+                    let test = test.clone();
+                    let egui_render = egui_render.clone();
+
+                    async move {
+                        let track = event.publication.track().unwrap();
+                        if let RemoteTrackHandle::Video(video_track) = track {
+                            *test.lock().unwrap() =
+                                Some(VideoRenderer::new(egui_render, video_track.rtc_track()))
+                        }
+                    }
+                }
+            });
+
+            inner.room.connect(URL, TOKEN).await.unwrap();
+
+            tokio::spawn(async {
+                loop {
+                    println!("Test");
+                    tokio::time::sleep(Duration::from_secs(5)).await;
+                }
+            });
+
+            tokio::task::block_in_place(move || loop {
+                event_loop.run(move |event, _, control_flow| {
+                    inner.on_event(event, control_flow);
+                });
+            });
+        });
+    }
+}
+
+fn main() {
+    let rt = tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()
+        .unwrap();
+
+    let mut app = App::new(rt);
+    app.run();
 }

examples/simple_room/src/video_renderer.rs

@@ -0,0 +1,172 @@
+use livekit::webrtc::media_stream::VideoTrack;
+use livekit::webrtc::video_frame_buffer::PlanarYuv8Buffer;
+use livekit::webrtc::video_frame_buffer::PlanarYuvBuffer;
+use livekit::webrtc::video_frame_buffer::VideoFrameBufferTrait;
+use livekit::webrtc::yuv_helper;
+use std::convert::TryInto;
+use std::num::NonZeroU32;
+use std::{
+    ops::DerefMut,
+    sync::{Arc, Mutex},
+};
+
+pub struct VideoRenderer {
+    internal: Arc<Mutex<RendererInternal>>,
+    rtc_track: Arc<VideoTrack>,
+}
+
+struct RendererInternal {
+    render_state: egui_wgpu::RenderState,
+    width: u32,
+    height: u32,
+    rgba_data: Vec<u8>,
+    texture: Option<wgpu::Texture>,
+    texture_view: Option<wgpu::TextureView>,
+    egui_texture: Option<egui::TextureId>,
+}
+
+impl RendererInternal {
+    fn ensure_texture_size(&mut self, width: u32, height: u32) {
+        if self.width == width && self.height == height {
+            return;
+        }
+
+        self.width = width;
+        self.height = height;
+        self.rgba_data.resize((width * height * 4) as usize, 0);
+
+        self.texture = Some(
+            self.render_state
+                .device
+                .create_texture(&wgpu::TextureDescriptor {
+                    label: Some("lk-videotexture"),
+                    usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
+                    dimension: wgpu::TextureDimension::D2,
+                    size: wgpu::Extent3d {
+                        width,
+                        height,
+                        ..Default::default()
+                    },
+                    sample_count: 1,
+                    mip_level_count: 1,
+                    format: wgpu::TextureFormat::Rgba8UnormSrgb,
+                }),
+        );
+
+        self.texture_view = Some(self.texture.as_mut().unwrap().create_view(
+            &wgpu::TextureViewDescriptor {
+                label: Some("lk-videotexture-view"),
+                format: Some(wgpu::TextureFormat::Rgba8UnormSrgb),
+                dimension: Some(wgpu::TextureViewDimension::D2),
+                mip_level_count: NonZeroU32::new(1),
+                array_layer_count: NonZeroU32::new(1),
+                ..Default::default()
+            },
+        ));
+
+        if let Some(texture_id) = self.egui_texture {
+            // Update the existing texture
+            self.render_state
+                .renderer
+                .write()
+                .update_egui_texture_from_wgpu_texture(
+                    &*self.render_state.device,
+                    self.texture_view.as_ref().unwrap(),
+                    wgpu::FilterMode::Linear,
+                    texture_id,
+                );
+        } else {
+            self.egui_texture = Some(self.render_state.renderer.write().register_native_texture(
+                &*self.render_state.device,
+                self.texture_view.as_ref().unwrap(),
+                wgpu::FilterMode::Linear,
+            ));
+        }
+    }
+}
+
+impl VideoRenderer {
+    pub fn new(render_state: egui_wgpu::RenderState, rtc_track: Arc<VideoTrack>) -> Self {
+        let internal = Arc::new(Mutex::new(RendererInternal {
+            render_state,
+            width: 0,
+            height: 0,
+            rgba_data: Vec::default(),
+            texture: None,
+            texture_view: None,
+            egui_texture: None,
+        }));
+
+        rtc_track.on_frame({
+            let internal = internal.clone();
+
+            Box::new(move |_frame, buffer| {
+                let mut internal = internal.lock().unwrap();
+                let buffer = buffer.to_i420();
+
+                let width: u32 = buffer.width().try_into().unwrap();
+                let height: u32 = buffer.height().try_into().unwrap();
+
+                internal.ensure_texture_size(width, height);
+
+                let rgba_ptr = internal.rgba_data.deref_mut();
+                let rgba_stride = buffer.width() * 4;
+
+                yuv_helper::i420_to_abgr(
+                    buffer.data_y(),
+                    buffer.stride_y(),
+                    buffer.data_u(),
+                    buffer.stride_u(),
+                    buffer.data_v(),
+                    buffer.stride_v(),
+                    rgba_ptr,
+                    rgba_stride,
+                    buffer.width(),
+                    buffer.height(),
+                );
+
+                let copy_desc = wgpu::ImageCopyTexture {
+                    texture: internal.texture.as_ref().unwrap(),
+                    mip_level: 0,
+                    origin: wgpu::Origin3d::default(),
+                    aspect: wgpu::TextureAspect::default(),
+                };
+
+                let copy_layout = wgpu::ImageDataLayout {
+                    bytes_per_row: Some(NonZeroU32::new(width * 4).unwrap()),
+                    ..Default::default()
+                };
+
+                let copy_size = wgpu::Extent3d {
+                    width,
+                    height,
+                    ..Default::default()
+                };
+
+                internal.render_state.queue.write_texture(
+                    copy_desc,
+                    &internal.rgba_data,
+                    copy_layout,
+                    copy_size,
+                );
+
+                println!("wrote");
+            })
+        });
+
+        Self {
+            rtc_track,
+            internal,
+        }
+    }
+
+    pub fn texture_id(&self) -> Option<egui::TextureId> {
+        self.internal.lock().unwrap().egui_texture.clone()
+    }
+}
+
+impl Drop for VideoRenderer {
+    fn drop(&mut self) {
+        self.rtc_track.on_frame(Box::new(|_, _| {}));
+    }
+}

src/lib.rs

@@ -1,2 +1,6 @@
 // export everything inside livekit-core
-pub use livekit_core::*;
+pub use livekit_core::*;
+
+pub mod webrtc {
+    pub use livekit_webrtc::*;
+}