feat: video publishing (#42)

- Prepare webrtc abstraction ( for future wasm support )
- Added track publish support for videos
  - Added LogoTrack example to simple_room demo
- Lot of cleanup
- There are compiler warnings I'll solve on our v1 release

Cargo.lock

@@ -418,9 +418,9 @@ dependencies = [
 
 [[package]]
 name = "futures-channel"
-version = "0.3.25"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed"
+checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5"
 dependencies = [
  "futures-core",
  "futures-sink",
@@ -428,9 +428,9 @@ dependencies = [
 
 [[package]]
 name = "futures-core"
-version = "0.3.25"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac"
+checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608"
 
 [[package]]
 name = "futures-executor"
@@ -462,9 +462,9 @@ dependencies = [
 
 [[package]]
 name = "futures-sink"
-version = "0.3.25"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9"
+checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364"
 
 [[package]]
 name = "futures-task"
@@ -779,7 +779,6 @@ checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4"
 name = "livekit"
 version = "0.1.1"
 dependencies = [
- "futures",
  "futures-util",
  "lazy_static",
  "livekit-utils",
@@ -792,6 +791,7 @@ dependencies = [
  "serde_json",
  "thiserror",
  "tokio",
+ "tokio-stream",
  "tokio-tungstenite",
  "tracing",
  "url",
@@ -801,6 +801,7 @@ dependencies = [
 name = "livekit-ffi"
 version = "0.1.1"
 dependencies = [
+ "futures-util",
  "lazy_static",
  "livekit",
  "parking_lot",
@@ -815,6 +816,7 @@ dependencies = [
 name = "livekit-utils"
 version = "0.1.1"
 dependencies = [
+ "futures-util",
  "parking_lot",
  "tokio",
 ]
@@ -825,10 +827,15 @@ version = "0.1.1"
 dependencies = [
  "cxx",
  "env_logger 0.9.3",
+ "futures",
+ "js-sys",
  "livekit-utils",
  "log",
  "thiserror",
  "tokio",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
  "webrtc-sys",
 ]
 
@@ -1354,17 +1361,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "sha-1"
-version = "0.10.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c"
-dependencies = [
- "cfg-if",
- "cpufeatures",
- "digest",
-]
-
 [[package]]
 name = "sha1"
 version = "0.10.5"
@@ -1536,9 +1532,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
 
 [[package]]
 name = "tokio"
-version = "1.23.0"
+version = "1.26.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eab6d665857cc6ca78d6e80303a02cea7a7851e85dfbd77cbdc09bd129f1ef46"
+checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64"
 dependencies = [
  "autocfg",
  "bytes",
@@ -1551,7 +1547,7 @@ dependencies = [
  "signal-hook-registry",
  "socket2",
  "tokio-macros",
- "windows-sys 0.42.0",
+ "windows-sys 0.45.0",
 ]
 
 [[package]]
@@ -1576,10 +1572,21 @@ dependencies = [
 ]
 
 [[package]]
-name = "tokio-tungstenite"
+name = "tokio-stream"
-version = "0.17.2"
+version = "0.1.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f714dd15bead90401d77e04243611caec13726c2408afd5b31901dfcdcb3b181"
+checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313"
+dependencies = [
+ "futures-core",
+ "pin-project-lite",
+ "tokio",
+]
+
+[[package]]
+name = "tokio-tungstenite"
+version = "0.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "54319c93411147bced34cb5609a80e0a8e44c5999c93903a81cd866630ec0bfd"
 dependencies = [
  "futures-util",
  "log",
@@ -1649,9 +1656,9 @@ checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
 
 [[package]]
 name = "tungstenite"
-version = "0.17.3"
+version = "0.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e27992fd6a8c29ee7eef28fc78349aa244134e10ad447ce3b9f0ac0ed0fa4ce0"
+checksum = "30ee6ab729cd4cf0fd55218530c4522ed30b7b6081752839b68fcec8d0960788"
 dependencies = [
  "base64 0.13.1",
  "byteorder",
@@ -1661,7 +1668,7 @@ dependencies = [
  "log",
  "native-tls",
  "rand",
- "sha-1",
+ "sha1",
  "thiserror",
  "url",
  "utf-8",
@@ -1899,12 +1906,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7"
 dependencies = [
  "windows_aarch64_gnullvm",
- "windows_aarch64_msvc 0.42.2",
+ "windows_aarch64_msvc 0.42.1",
- "windows_i686_gnu 0.42.2",
+ "windows_i686_gnu 0.42.1",
- "windows_i686_msvc 0.42.2",
+ "windows_i686_msvc 0.42.1",
- "windows_x86_64_gnu 0.42.2",
+ "windows_x86_64_gnu 0.42.1",
  "windows_x86_64_gnullvm",
- "windows_x86_64_msvc 0.42.2",
+ "windows_x86_64_msvc 0.42.1",
 ]
 
 [[package]]
@@ -1918,24 +1925,24 @@ dependencies = [
 
 [[package]]
 name = "windows-targets"
-version = "0.42.2"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"
+checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7"
 dependencies = [
  "windows_aarch64_gnullvm",
- "windows_aarch64_msvc 0.42.2",
+ "windows_aarch64_msvc 0.42.1",
- "windows_i686_gnu 0.42.2",
+ "windows_i686_gnu 0.42.1",
- "windows_i686_msvc 0.42.2",
+ "windows_i686_msvc 0.42.1",
- "windows_x86_64_gnu 0.42.2",
+ "windows_x86_64_gnu 0.42.1",
  "windows_x86_64_gnullvm",
- "windows_x86_64_msvc 0.42.2",
+ "windows_x86_64_msvc 0.42.1",
 ]
 
 [[package]]
 name = "windows_aarch64_gnullvm"
-version = "0.42.2"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
+checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608"
 
 [[package]]
 name = "windows_aarch64_msvc"
@@ -1945,9 +1952,9 @@ checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"
 
 [[package]]
 name = "windows_aarch64_msvc"
-version = "0.42.2"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
+checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7"
 
 [[package]]
 name = "windows_i686_gnu"
@@ -1957,9 +1964,9 @@ checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"
 
 [[package]]
 name = "windows_i686_gnu"
-version = "0.42.2"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
+checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640"
 
 [[package]]
 name = "windows_i686_msvc"
@@ -1969,9 +1976,9 @@ checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"
 
 [[package]]
 name = "windows_i686_msvc"
-version = "0.42.2"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
+checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605"
 
 [[package]]
 name = "windows_x86_64_gnu"
@@ -1981,15 +1988,15 @@ checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"
 
 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.42.2"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
+checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45"
 
 [[package]]
 name = "windows_x86_64_gnullvm"
-version = "0.42.2"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
+checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463"
 
 [[package]]
 name = "windows_x86_64_msvc"
@@ -1999,9 +2006,9 @@ checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
 
 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.42.2"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
+checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd"
 
 [[package]]
 name = "winreg"

examples/simple_room/Cargo.toml

@@ -16,3 +16,5 @@ parking_lot = "0.12.1"
 egui = "0.20.1"
 egui-wgpu = { version = "0.20.0", features = ["winit"] }
 egui-winit = "0.20.1"
+
+image = "0.24.5"

examples/simple_room/src/app.rs

@@ -1,9 +1,16 @@
 use crate::events::UiCmd;
+use crate::logo_track::LogoTrack;
 use crate::video_renderer::VideoRenderer;
 use crate::{events::AsyncCmd, video_grid::VideoGrid};
 use egui::{Rounding, Stroke};
 use egui_wgpu::WgpuConfiguration;
+use image::ImageFormat;
+use livekit::options::{TrackPublishOptions, VideoCaptureOptions};
 use livekit::prelude::*;
+use livekit::webrtc::native::yuv_helper;
+use livekit::webrtc::video_frame::native::I420BufferExt;
+use livekit::webrtc::video_frame::{I420Buffer, VideoFrame, VideoRotation};
+use livekit::webrtc::video_source::native::NativeVideoSource;
 use livekit::SimulateScenario;
 use parking_lot::Mutex;
 use std::collections::HashMap;
@@ -11,6 +18,7 @@ use std::sync::{
     atomic::{AtomicBool, Ordering},
     Arc,
 };
+use std::time::Duration;
 use tokio::sync::{mpsc, oneshot};
 
 // Useful default constants for developing
@@ -25,16 +33,22 @@ use winit::{
     window::{WindowBuilder, WindowId},
 };
 
+struct Session {
+    room: Room,
+    logo_track: LogoTrack,
+    close_tx: oneshot::Sender<()>,
+    handle: tokio::task::JoinHandle<()>,
+}
+
 struct AppState {
-    room: Mutex<Option<Room>>,
+    session: Mutex<Option<Session>>,
-    close_tx: Mutex<Option<oneshot::Sender<()>>>,
     connecting: AtomicBool,
 }
 
 struct App {
     state: Arc<AppState>,
-
     video_renderers: HashMap<(ParticipantSid, TrackSid), VideoRenderer>,
+
     egui_context: egui::Context,
     egui_state: egui_winit::State,
     egui_painter: egui_wgpu::winit::Painter,
@@ -69,8 +83,7 @@ pub fn run(rt: tokio::runtime::Runtime) {
         let (ui_cmd_tx, ui_cmd_rx) = mpsc::unbounded_channel::<UiCmd>();
 
         let state = Arc::new(AppState {
-            room: Default::default(),
+            session: Default::default(),
-            close_tx: Default::default(),
             connecting: AtomicBool::new(false),
         });
 
@@ -94,45 +107,53 @@ pub fn run(rt: tokio::runtime::Runtime) {
             while let Some(event) = async_cmd_rx.recv().await {
                 match event {
                     AsyncCmd::RoomConnect { url, token } => {
-                        if let Some(close_tx) = state.close_tx.lock().take() {
-                            let _ = state.room.lock().take().unwrap().close().await;
-                            let _ = close_tx.send(());
-                        }
-
                         state.connecting.store(true, Ordering::SeqCst);
 
                         let res = Room::connect(&url, &token).await;
-                        match res {
+                        if let Ok((room, room_events)) = res {
-                            Ok((room, room_events)) => {
                             let (close_tx, close_rx) = oneshot::channel();
-                                state.room.lock().replace(room);
+                            let logo_track = LogoTrack::new(room.session());
-                                state.close_tx.lock().replace(close_tx);
+                            let handle = tokio::spawn(room_task(
-
-                                tokio::spawn(room_task(
                                 state.clone(),
                                 room_events,
                                 close_rx,
                                 ui_cmd_tx.clone(),
                             ));
 
+                            *state.session.lock() = Some(Session {
+                                room,
+                                logo_track,
+                                close_tx,
+                                handle,
+                            });
+
                             let _ = ui_cmd_tx.send(UiCmd::ConnectResult { result: Ok(()) });
-                            }
+                        } else if let Err(err) = res {
-                            Err(err) => {
                             let _ = ui_cmd_tx.send(UiCmd::ConnectResult { result: Err(err) });
                         }
-                        }
 
                         state.connecting.store(false, Ordering::SeqCst);
                     }
                     AsyncCmd::RoomDisconnect => {
-                        if let Some(close_tx) = state.close_tx.lock().take() {
+                        if let Some(session) = state.session.lock().take() {
-                            let _ = state.room.lock().take().unwrap().close().await;
+                            let _ = session.room.close().await;
-                            let _ = close_tx.send(());
+                            let _ = session.close_tx.send(());
+                            let _ = session.handle.await;
                         }
                     }
                     AsyncCmd::SimulateScenario { scenario } => {
-                        if let Some(room) = state.room.lock().as_ref() {
+                        if let Some(session) = state.session.lock().as_ref() {
-                            let _ = room.session().simulate_scenario(scenario).await;
+                            let _ = session.room.session().simulate_scenario(scenario).await;
+                        }
+                    }
+                    AsyncCmd::ToggleLogo => {
+                        if let Some(session) = state.session.lock().as_mut() {
+                            let logo_track = &mut session.logo_track;
+                            if !logo_track.is_published() {
+                                logo_track.publish().await.unwrap();
+                            } else {
+                                logo_track.unpublish().await.unwrap();
+                            }
                         }
                     }
                 }
@@ -140,7 +161,7 @@ pub fn run(rt: tokio::runtime::Runtime) {
         });
 
         tokio::task::block_in_place(move || loop {
-            // UI/Main Thread
+            // ui/main thread
             event_loop.run(move |event, _, control_flow| {
                 app.update(event, control_flow);
             });
@@ -183,7 +204,7 @@ impl App {
                             track, participant, ..
                         } => {
                             match track.clone() {
-                                RemoteTrackHandle::Video(video_track) => {
+                                RemoteTrack::Video(video_track) => {
                                     // Create a new VideoRenderer
                                     let video_renderer = VideoRenderer::new(
                                         self.egui_painter.render_state().clone().unwrap(),
@@ -192,7 +213,7 @@ impl App {
                                     self.video_renderers
                                         .insert((participant.sid(), track.sid()), video_renderer);
                                 }
-                                RemoteTrackHandle::Audio(_) => {
+                                RemoteTrack::Audio(_) => {
                                     // The demo doesn't support Audio rendering at the moment.
                                 }
                             };
@@ -300,6 +321,12 @@ impl App {
                         });
                     }
                 });
+
+                ui.menu_button("Publish", |ui| {
+                    if ui.button("CustomTrack - LK Logo").clicked() {
+                        let _ = self.cmd_tx.send(AsyncCmd::ToggleLogo);
+                    }
+                });
             });
         });
 
@@ -321,9 +348,9 @@ impl App {
 
                 ui.horizontal(|ui| {
                     let connecting = self.state.connecting.load(Ordering::SeqCst);
+                    let session = self.state.session.lock();
 
-                    let room = self.state.room.lock();
+                    ui.add_enabled_ui(!connecting && session.is_none(), |ui| {
-                    ui.add_enabled_ui(!connecting && room.is_none(), |ui| {
                         if ui.button("Connect").clicked() {
                             self.connection_failure = None;
                             let _ = self.cmd_tx.send(AsyncCmd::RoomConnect {
@@ -337,7 +364,7 @@ impl App {
                         ui.spinner();
                     }
 
-                    if room.is_some() {
+                    if session.is_some() {
                         if ui.button("Disconnect").clicked() {
                             let _ = self.cmd_tx.send(AsyncCmd::RoomDisconnect);
                         }
@@ -352,17 +379,16 @@ impl App {
 
                 {
                     // Room Info
-                    let room = self.state.room.lock();
+                    if let Some(session) = self.state.session.lock().as_ref() {
-                    if let Some(room) = room.as_ref() {
+                        ui.label(format!("Name: {}", session.room.session().name()));
-                        ui.label(format!("Name: {}", room.session().name()));
+                        ui.label(format!("SID: {}", session.room.session().sid()));
-                        ui.label(format!("SID: {}", room.session().sid()));
                         ui.label(format!(
                             "ConnectionState: {:?}",
-                            room.session().connection_state()
+                            session.room.session().connection_state()
                         ));
                         ui.label(format!(
                             "ParticipantCount: {:?}",
-                            room.session().participants().len() + 1
+                            session.room.session().participants().len() + 1
                         ));
                     }
                 }
@@ -408,8 +434,11 @@ impl App {
                                         );
                                     }
 
-                                    let name = self.state.room.lock().as_ref().and_then(|room| {
+                                    let name =
-                                        room.session()
+                                        self.state.session.lock().as_ref().and_then(|session| {
+                                            session
+                                                .room
+                                                .session()
                                                 .participants()
                                                 .get(participant_sid)
                                                 .map(|p| p.name())

examples/simple_room/src/events.rs

@@ -6,6 +6,7 @@ pub enum AsyncCmd {
     RoomConnect { url: String, token: String },
     RoomDisconnect,
     SimulateScenario { scenario: SimulateScenario },
+    ToggleLogo, // Unpublish/Publish a logo track
 }
 
 #[derive(Debug)]

examples/simple_room/src/logo_track.rs

@@ -0,0 +1,206 @@
+use image::ImageFormat;
+use image::RgbaImage;
+use livekit::options::{TrackPublishOptions, VideoCaptureOptions};
+use livekit::prelude::*;
+use livekit::webrtc::{
+    native::yuv_helper,
+    video_frame::native::I420BufferExt,
+    video_frame::{I420Buffer, VideoFrame, VideoRotation},
+    video_source::native::NativeVideoSource,
+};
+use parking_lot::Mutex;
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::oneshot;
+use tokio::task::JoinHandle;
+
+// The logo must not be bigger than the framebuffer
+const PIXEL_SIZE: usize = 4;
+const FRAME_RATE: u64 = 30;
+const MOVE_SPEED: i32 = 16;
+const FB_WIDTH: usize = 1280;
+const FB_HEIGHT: usize = 720;
+
+#[derive(Clone)]
+struct FrameData {
+    image: Arc<RgbaImage>,
+    framebuffer: Arc<Mutex<Vec<u8>>>,
+    video_frame: Arc<Mutex<VideoFrame<I420Buffer>>>,
+    pos: (u32, u32),
+    direction: (i32, i32),
+}
+
+struct TrackHandle {
+    close_tx: oneshot::Sender<()>,
+    track: LocalVideoTrack,
+    task: JoinHandle<()>,
+}
+
+pub struct LogoTrack {
+    rtc_source: NativeVideoSource,
+    session: RoomSession,
+    handle: Option<TrackHandle>,
+}
+
+impl LogoTrack {
+    pub fn new(session: RoomSession) -> Self {
+        Self {
+            rtc_source: NativeVideoSource::default(),
+            session,
+            handle: None,
+        }
+    }
+
+    pub fn is_published(&self) -> bool {
+        self.handle.is_some()
+    }
+
+    pub async fn publish(&mut self) -> Result<(), RoomError> {
+        self.unpublish().await;
+
+        let (close_tx, close_rx) = oneshot::channel();
+        let track = LocalVideoTrack::create_video_track(
+            "livekit_logo",
+            VideoCaptureOptions::default(),
+            self.rtc_source.clone(),
+        );
+
+        let task = tokio::spawn(Self::track_task(close_rx, self.rtc_source.clone()));
+
+        self.session
+            .local_participant()
+            .publish_track(
+                LocalTrack::Video(track.clone()),
+                TrackPublishOptions {
+                    source: TrackSource::Camera,
+                    ..Default::default()
+                },
+            )
+            .await?;
+
+        let handle = TrackHandle {
+            close_tx,
+            task,
+            track,
+        };
+
+        self.handle = Some(handle);
+        Ok(())
+    }
+
+    pub async fn unpublish(&mut self) -> Result<(), RoomError> {
+        if let Some(handle) = self.handle.take() {
+            let _ = handle.close_tx.send(());
+            let _ = handle.task.await;
+
+            self.session
+                .local_participant()
+                .unpublish_track(handle.track.sid(), true)
+                .await?;
+        }
+        Ok(())
+    }
+
+    async fn track_task(mut close_rx: oneshot::Receiver<()>, rtc_source: NativeVideoSource) {
+        let mut interval = tokio::time::interval(Duration::from_millis(1000 / FRAME_RATE));
+
+        let image = tokio::task::spawn_blocking(|| {
+            image::load_from_memory_with_format(include_bytes!("moving-logo.png"), ImageFormat::Png)
+                .unwrap()
+                .to_rgba8()
+        })
+        .await
+        .unwrap();
+
+        let mut data = FrameData {
+            image: Arc::new(image),
+            framebuffer: Arc::new(Mutex::new(vec![0u8; (FB_WIDTH * FB_HEIGHT * 4) as usize])),
+            video_frame: Arc::new(Mutex::new(VideoFrame {
+                rotation: VideoRotation::VideoRotation0,
+                buffer: I420Buffer::new(FB_WIDTH as u32, FB_HEIGHT as u32),
+                timestamp: 0,
+            })),
+            pos: (0, 0),
+            direction: (1, 1),
+        };
+
+        loop {
+            tokio::select! {
+                _ = &mut close_rx => {
+                    break;
+                }
+                _ = interval.tick() => {}
+            }
+
+            data.pos.0 = (data.pos.0 as i32 + data.direction.0 * MOVE_SPEED) as u32;
+            data.pos.1 = (data.pos.1 as i32 + data.direction.1 * MOVE_SPEED) as u32;
+
+            if data.pos.0 >= (FB_WIDTH - data.image.width() as usize) as u32 {
+                data.direction.0 = -1;
+            } else if data.pos.0 <= 0 {
+                data.direction.0 = 1;
+            }
+
+            if data.pos.1 >= (FB_HEIGHT - data.image.height() as usize) as u32 {
+                data.direction.1 = -1;
+            } else if data.pos.1 <= 0 {
+                data.direction.1 = 1;
+            }
+
+            tokio::task::spawn_blocking({
+                let data = data.clone();
+                let source = rtc_source.clone();
+                move || {
+                    let image = data.image.as_raw();
+                    let mut framebuffer = data.framebuffer.lock();
+                    let mut video_frame = data.video_frame.lock();
+                    let i420_buffer = &mut video_frame.buffer;
+
+                    let stride_y = i420_buffer.stride_y();
+                    let stride_u = i420_buffer.stride_u();
+                    let stride_v = i420_buffer.stride_v();
+                    let (data_y, data_u, data_v) = i420_buffer.data_mut();
+
+                    framebuffer.fill(0);
+                    for i in 0..data.image.height() as usize {
+                        let x = data.pos.0 as usize;
+                        let y = data.pos.1 as usize;
+                        let frame_width = data.image.width() as usize;
+                        let logo_stride = frame_width * PIXEL_SIZE;
+                        let row_start = (x + ((i + y) * FB_WIDTH)) * PIXEL_SIZE;
+                        let row_end = row_start + logo_stride;
+
+                        framebuffer[row_start..row_end].copy_from_slice(
+                            &image[i * logo_stride..i * logo_stride + logo_stride],
+                        );
+                    }
+
+                    yuv_helper::abgr_to_i420(
+                        &framebuffer,
+                        (FB_WIDTH * PIXEL_SIZE) as i32,
+                        data_y,
+                        stride_y,
+                        data_u,
+                        stride_u,
+                        data_v,
+                        stride_v,
+                        FB_WIDTH as i32,
+                        FB_HEIGHT as i32,
+                    )
+                    .unwrap();
+
+                    source.capture_frame(&*video_frame);
+                }
+            })
+            .await;
+        }
+    }
+}
+
+impl Drop for LogoTrack {
+    fn drop(&mut self) {
+        if let Some(handle) = self.handle.take() {
+            let _ = handle.close_tx.send(());
+        }
+    }
+}

examples/simple_room/src/main.rs

@@ -1,5 +1,6 @@
 mod app;
 mod events;
+mod logo_track;
 mod video_grid;
 mod video_renderer;
 

examples/simple_room/src/video_renderer.rs

@@ -1,18 +1,17 @@
-use livekit::prelude::*;
+use futures::StreamExt;
-use livekit::webrtc::video_frame_buffer::*;
+use livekit::webrtc::native::yuv_helper;
-use livekit::webrtc::yuv_helper;
+use livekit::webrtc::prelude::*;
-use std::convert::TryInto;
+use livekit::webrtc::video_stream::native::NativeVideoStream;
 use std::num::NonZeroU32;
 use std::{
     ops::DerefMut,
     sync::{Arc, Mutex},
 };
 use tracing::debug_span;
-use tracing::{error, warn};
 
 pub struct VideoRenderer {
     internal: Arc<Mutex<RendererInternal>>,
-    rtc_track: Arc<VideoTrack>,
+    rtc_track: RtcVideoTrack,
 }
 
 struct RendererInternal {
@@ -86,7 +85,7 @@ impl RendererInternal {
 }
 
 impl VideoRenderer {
-    pub fn new(render_state: egui_wgpu::RenderState, rtc_track: Arc<VideoTrack>) -> Self {
+    pub fn new(render_state: egui_wgpu::RenderState, rtc_track: RtcVideoTrack) -> Self {
         let internal = Arc::new(Mutex::new(RendererInternal {
             render_state,
             width: 0,
@@ -97,15 +96,20 @@ impl VideoRenderer {
             egui_texture: None,
         }));
 
-        rtc_track.on_frame({
+        let mut video_sink = NativeVideoStream::new(rtc_track.clone());
-            let internal = internal.clone();
 
-            Box::new(move |_frame, buffer| {
+        tokio::spawn({
+            let internal = internal.clone();
+            async move {
+                while let Some(frame) = video_sink.next().await {
+                    let internal = internal.clone();
+                    // Process the frame
+                    let _ = tokio::task::spawn_blocking(move || {
                         let span = debug_span!("texture_upload");
                         let _enter = span.enter();
 
                         let mut internal = internal.lock().unwrap();
-                let buffer = buffer.to_i420();
+                        let buffer = frame.buffer.to_i420();
 
                         let width: u32 = buffer.width().try_into().unwrap();
                         let height: u32 = buffer.height().try_into().unwrap();
@@ -115,18 +119,21 @@ impl VideoRenderer {
                         let rgba_ptr = internal.rgba_data.deref_mut();
                         let rgba_stride = buffer.width() * 4;
 
+                        let (data_y, data_u, data_v) = buffer.data();
+
                         yuv_helper::i420_to_abgr(
-                    buffer.data_y(),
+                            data_y,
                             buffer.stride_y(),
-                    buffer.data_u(),
+                            data_u,
                             buffer.stride_u(),
-                    buffer.data_v(),
+                            data_v,
                             buffer.stride_v(),
                             rgba_ptr,
                             rgba_stride,
                             buffer.width(),
                             buffer.height(),
-                );
+                        )
+                        .unwrap();
 
                         let copy_desc = wgpu::ImageCopyTexture {
                             texture: internal.texture.as_ref().unwrap(),
@@ -153,6 +160,9 @@ impl VideoRenderer {
                             copy_size,
                         );
                     })
+                    .await;
+                }
+            }
         });
 
         Self {
@@ -165,9 +175,3 @@ impl VideoRenderer {
         self.internal.lock().unwrap().egui_texture.clone()
     }
 }
-
-impl Drop for VideoRenderer {
-    fn drop(&mut self) {
-        self.rtc_track.on_frame(Box::new(|_, _| {}));
-    }
-}

livekit-ffi/Cargo.toml

@@ -14,10 +14,10 @@ prost = "0.11.0"
 prost-types = "0.11.1"
 lazy_static = "1.4.0"
 thiserror = "1.0.38"
+futures-util = "0.3.23"
 
 [build-dependencies]
 prost-build = { version = "0.11.1" }
 
-
 [lib]
 crate-type = ["cdylib", "staticlib"]

livekit-ffi/protocol/ffi.proto

@@ -279,19 +279,12 @@ message TrackEvent {
 
 message FrameReceived {
   VideoFrameInfo frame = 1;
-  VideoFrameBufferInfo frame_buffer = 2;
+  VideoFrameBufferInfo buffer = 2;
 }
 
 message VideoFrameInfo {
-  int32 width = 1;
+  int64 timestamp = 1;
-  int32 height = 2;
+  VideoRotation rotation = 2;
-  uint32 size = 3;
-  uint32 id = 4; // uint16
-  int64 timestamp_us = 5;
-  int64 ntp_time_ms = 6;
-  uint32 transport_frame_id = 7;
-  uint32 timestamp = 8;
-  VideoRotation rotation = 9;
 }
 
 message VideoFrameBufferInfo {
@@ -312,11 +305,13 @@ message PlanarYuvBufferInfo {
   int32 stride_y = 3;
   int32 stride_u = 4;
   int32 stride_v = 5;
+  int32 stride_a = 6;
 
   // *const u8 or *const u16
-  uint64 data_y_ptr = 6;
+  uint64 data_y_ptr = 7;
-  uint64 data_u_ptr = 7;
+  uint64 data_u_ptr = 8;
-  uint64 data_v_ptr = 8;
+  uint64 data_v_ptr = 9;
+  uint64 data_a_ptr = 10; // nullptr = no alpha
 }
 
 message BiplanarYuvBufferInfo {
@@ -341,6 +336,7 @@ enum VideoFrameBufferType {
   I444 = 4;
   I010 = 5;
   NV12 = 6;
+  WEBGL = 7;
 }
 
 /// Participant Events

livekit-ffi/src/server/conversion/mod.rs

@@ -1,12 +1,12 @@
 use crate::{proto, server::FFIHandleId};
-use livekit::{
+use livekit::prelude::*;
-    prelude::*,
+use livekit::webrtc::prelude::*;
-    webrtc::video_frame_buffer::{
+use std::any::Any;
-        BiplanarYuv8Buffer, BiplanarYuvBuffer, I010Buffer, I420ABuffer, I420Buffer, I422Buffer,
+
-        I444Buffer, NV12Buffer, PlanarYuv16BBuffer, PlanarYuv8Buffer, PlanarYuvBuffer,
+pub mod participant;
-    },
+pub mod publication;
-};
+pub mod room;
-use std::sync::Arc;
+pub mod video_frame;
 
 impl From<FFIHandleId> for proto::FfiHandleId {
     fn from(id: FFIHandleId) -> Self {
@@ -30,8 +30,8 @@ macro_rules! impl_participant_into {
     };
 }
 
-impl_participant_into!(&Arc<LocalParticipant>);
+impl_participant_into!(&LocalParticipant);
-impl_participant_into!(&Arc<RemoteParticipant>);
+impl_participant_into!(&RemoteParticipant);
 impl_participant_into!(&Participant);
 
 impl From<TrackSource> for proto::TrackSource {
@@ -92,14 +92,13 @@ impl_track_into!(&LocalAudioTrack);
 impl_track_into!(&LocalVideoTrack);
 impl_track_into!(&RemoteAudioTrack);
 impl_track_into!(&RemoteVideoTrack);
-impl_track_into!(&TrackHandle);
+impl_track_into!(&Track);
-impl_track_into!(&LocalTrackHandle);
+impl_track_into!(&LocalTrack);
-impl_track_into!(&RemoteTrackHandle);
+impl_track_into!(&RemoteTrack);
 
 impl From<TrackKind> for proto::TrackKind {
     fn from(kind: TrackKind) -> Self {
         match kind {
-            TrackKind::Unknown => proto::TrackKind::KindUnknown,
             TrackKind::Audio => proto::TrackKind::KindAudio,
             TrackKind::Video => proto::TrackKind::KindVideo,
         }
@@ -109,7 +108,6 @@ impl From<TrackKind> for proto::TrackKind {
 impl From<StreamState> for proto::StreamState {
     fn from(state: StreamState) -> Self {
         match state {
-            StreamState::Unknown => Self::StateUnknown,
             StreamState::Active => Self::StateActive,
             StreamState::Paused => Self::StatePaused,
         }
@@ -182,136 +180,6 @@ impl proto::RoomEvent {
     }
 }
 
-impl From<VideoRotation> for proto::VideoRotation {
-    fn from(rotation: VideoRotation) -> proto::VideoRotation {
-        match rotation {
-            VideoRotation::VideoRotation0 => Self::VideoRotation0,
-            VideoRotation::VideoRotation90 => Self::VideoRotation90,
-            VideoRotation::VideoRotation180 => Self::VideoRotation180,
-            VideoRotation::VideoRotation270 => Self::VideoRotation270,
-        }
-    }
-}
-
-impl From<VideoFrame> for proto::VideoFrameInfo {
-    fn from(frame: VideoFrame) -> Self {
-        Self {
-            width: frame.width(),
-            height: frame.height(),
-            size: frame.size(),
-            id: frame.id() as u32,
-            timestamp_us: frame.timestamp_us(),
-            ntp_time_ms: frame.ntp_time_ms(),
-            transport_frame_id: frame.transport_frame_id(),
-            timestamp: frame.timestamp(),
-            rotation: proto::VideoRotation::from(frame.rotation()).into(),
-        }
-    }
-}
-
-impl From<VideoFrameBufferType> for proto::VideoFrameBufferType {
-    fn from(buffer_type: VideoFrameBufferType) -> Self {
-        match buffer_type {
-            VideoFrameBufferType::Native => Self::Native,
-            VideoFrameBufferType::I420 => Self::I420,
-            VideoFrameBufferType::I420A => Self::I420a,
-            VideoFrameBufferType::I422 => Self::I422,
-            VideoFrameBufferType::I444 => Self::I444,
-            VideoFrameBufferType::I010 => Self::I010,
-            VideoFrameBufferType::NV12 => Self::Nv12,
-        }
-    }
-}
-
-macro_rules! impl_yuv_into {
-    ($b:ty) => {
-        impl From<$b> for proto::PlanarYuvBufferInfo {
-            fn from(buffer: $b) -> Self {
-                Self {
-                    chroma_width: buffer.chroma_width(),
-                    chroma_height: buffer.chroma_height(),
-                    stride_y: buffer.stride_y(),
-                    stride_u: buffer.stride_u(),
-                    stride_v: buffer.stride_v(),
-                    data_y_ptr: buffer.data_y().as_ptr() as u64,
-                    data_u_ptr: buffer.data_u().as_ptr() as u64,
-                    data_v_ptr: buffer.data_v().as_ptr() as u64,
-                }
-            }
-        }
-    };
-}
-
-impl_yuv_into!(&I420Buffer);
-impl_yuv_into!(&I420ABuffer);
-impl_yuv_into!(&I422Buffer);
-impl_yuv_into!(&I444Buffer);
-impl_yuv_into!(&I010Buffer);
-
-macro_rules! impl_biyuv_into {
-    ($b:ty) => {
-        impl From<$b> for proto::BiplanarYuvBufferInfo {
-            fn from(buffer: $b) -> Self {
-                Self {
-                    chroma_width: buffer.chroma_width(),
-                    chroma_height: buffer.chroma_height(),
-                    stride_y: buffer.stride_y(),
-                    stride_uv: buffer.stride_uv(),
-                    data_y_ptr: buffer.data_y().as_ptr() as u64,
-                    data_uv_ptr: buffer.data_uv().as_ptr() as u64,
-                }
-            }
-        }
-    };
-}
-
-impl_biyuv_into!(&NV12Buffer);
-
-impl proto::VideoFrameBufferInfo {
-    pub fn from(handle_id: FFIHandleId, buffer: &VideoFrameBuffer) -> Self {
-        Self {
-            handle: Some(handle_id.into()),
-            buffer_type: proto::VideoFrameBufferType::from(buffer.buffer_type()).into(),
-            width: buffer.width(),
-            height: buffer.height(),
-            buffer: Some(match &buffer {
-                VideoFrameBuffer::Native(_) => {
-                    proto::video_frame_buffer_info::Buffer::Native(proto::NativeBufferInfo {})
-                }
-                VideoFrameBuffer::I420(i420) => {
-                    proto::video_frame_buffer_info::Buffer::Yuv(i420.into())
-                }
-                VideoFrameBuffer::I420A(i420a) => {
-                    proto::video_frame_buffer_info::Buffer::Yuv(i420a.into())
-                }
-                VideoFrameBuffer::I422(i422) => {
-                    proto::video_frame_buffer_info::Buffer::Yuv(i422.into())
-                }
-                VideoFrameBuffer::I444(i444) => {
-                    proto::video_frame_buffer_info::Buffer::Yuv(i444.into())
-                }
-                VideoFrameBuffer::I010(i010) => {
-                    proto::video_frame_buffer_info::Buffer::Yuv(i010.into())
-                }
-                VideoFrameBuffer::NV12(nv12) => {
-                    proto::video_frame_buffer_info::Buffer::BiYuv(nv12.into())
-                }
-            }),
-        }
-    }
-}
-
-impl From<proto::VideoFormatType> for VideoFormatType {
-    fn from(format: proto::VideoFormatType) -> Self {
-        match format {
-            proto::VideoFormatType::FormatArgb => Self::ARGB,
-            proto::VideoFormatType::FormatBgra => Self::BGRA,
-            proto::VideoFormatType::FormatAbgr => Self::ABGR,
-            proto::VideoFormatType::FormatRgba => Self::RGBA,
-        }
-    }
-}
-
 impl From<&RoomSession> for proto::RoomInfo {
     fn from(session: &RoomSession) -> Self {
         Self {

livekit-ffi/src/server/conversion/video_frame.rs

@@ -0,0 +1,203 @@
+use crate::{proto, server::FFIHandleId};
+use livekit::webrtc::prelude::*;
+use livekit::webrtc::video_frame;
+use std::any::Any;
+
+macro_rules! impl_yuv_into {
+    (@fields, $buffer:ident, $data_y:ident, $data_u:ident, $data_v: ident) => {
+        Self {
+            chroma_width: $buffer.chroma_width(),
+            chroma_height: $buffer.chroma_height(),
+            stride_y: $buffer.stride_y(),
+            stride_u: $buffer.stride_u(),
+            stride_v: $buffer.stride_v(),
+            data_y_ptr: $data_y.as_ptr() as u64,
+            data_u_ptr: $data_u.as_ptr() as u64,
+            data_v_ptr: $data_v.as_ptr() as u64,
+            ..Default::default()
+        }
+    };
+    ($buffer:ty, ALPHA) => {
+        impl From<$buffer> for proto::PlanarYuvBufferInfo {
+            fn from(buffer: $buffer) -> Self {
+                let (data_y, data_u, data_v, data_a) = buffer.data();
+                let mut proto = impl_yuv_into!(@fields, buffer, data_y, data_u, data_v);
+                proto.stride_a = buffer.stride_a();
+                proto.data_a_ptr = data_a.map(|data_a| data_a.as_ptr() as u64).unwrap_or(0);
+                proto
+            }
+        }
+    };
+    ($buffer:ty) => {
+        impl From<$buffer> for proto::PlanarYuvBufferInfo {
+            fn from(buffer: $buffer) -> Self {
+                let (data_y, data_u, data_v) = buffer.data();
+                impl_yuv_into!(@fields, buffer, data_y, data_u, data_v)
+            }
+        }
+    };
+}
+
+macro_rules! impl_biyuv_into {
+    ($b:ty) => {
+        impl From<$b> for proto::BiplanarYuvBufferInfo {
+            fn from(buffer: $b) -> Self {
+                let (data_y, data_uv) = buffer.data();
+                Self {
+                    chroma_width: buffer.chroma_width(),
+                    chroma_height: buffer.chroma_height(),
+                    stride_y: buffer.stride_y(),
+                    stride_uv: buffer.stride_uv(),
+                    data_y_ptr: data_y.as_ptr() as u64,
+                    data_uv_ptr: data_uv.as_ptr() as u64,
+                }
+            }
+        }
+    };
+}
+
+impl_yuv_into!(&I420Buffer);
+impl_yuv_into!(&I420ABuffer, ALPHA);
+impl_yuv_into!(&I422Buffer);
+impl_yuv_into!(&I444Buffer);
+impl_yuv_into!(&I010Buffer);
+impl_biyuv_into!(&NV12Buffer);
+
+impl proto::VideoFrameInfo {
+    pub fn from<T>(frame: &VideoFrame<T>) -> Self
+    where
+        T: VideoFrameBuffer,
+    {
+        Self {
+            timestamp: frame.timestamp,
+            rotation: proto::VideoRotation::from(frame.rotation).into(),
+        }
+    }
+}
+
+impl proto::VideoFrameBufferInfo {
+    pub fn from(handle: FFIHandleId, buffer: &dyn VideoFrameBuffer) -> Self {
+        match &buffer.buffer_type() {
+            #[cfg(not(target_arch = "wasm32"))]
+            VideoFrameBufferType::Native => Self::from_native(handle, buffer.as_native().unwrap()),
+            VideoFrameBufferType::I420 => Self::from_i420(handle, buffer.as_i420().unwrap()),
+            VideoFrameBufferType::I420A => Self::from_i420a(handle, buffer.as_i420a().unwrap()),
+            VideoFrameBufferType::I422 => Self::from_i422(handle, buffer.as_i422().unwrap()),
+            VideoFrameBufferType::I444 => Self::from_i444(handle, buffer.as_i444().unwrap()),
+            VideoFrameBufferType::I010 => Self::from_i010(handle, buffer.as_i010().unwrap()),
+            VideoFrameBufferType::NV12 => Self::from_nv12(handle, buffer.as_nv12().unwrap()),
+            _ => panic!("unsupported buffer type on this platform"),
+        }
+    }
+
+    #[cfg(not(target_arch = "wasm32"))]
+    pub fn from_native(handle_id: FFIHandleId, buffer: &video_frame::native::NativeBuffer) -> Self {
+        Self {
+            handle: Some(handle_id.into()),
+            buffer_type: proto::VideoFrameBufferType::Native.into(),
+            width: buffer.width(),
+            height: buffer.height(),
+            buffer: Some(proto::video_frame_buffer_info::Buffer::Native(
+                proto::NativeBufferInfo {},
+            )),
+        }
+    }
+
+    pub fn from_i420(handle_id: FFIHandleId, buffer: &I420Buffer) -> Self {
+        Self {
+            handle: Some(handle_id.into()),
+            buffer_type: proto::VideoFrameBufferType::I420.into(),
+            width: buffer.width(),
+            height: buffer.height(),
+            buffer: Some(proto::video_frame_buffer_info::Buffer::Yuv(buffer.into())),
+        }
+    }
+
+    pub fn from_i420a(handle_id: FFIHandleId, buffer: &I420ABuffer) -> Self {
+        Self {
+            handle: Some(handle_id.into()),
+            buffer_type: proto::VideoFrameBufferType::I420a.into(),
+            width: buffer.width(),
+            height: buffer.height(),
+            buffer: Some(proto::video_frame_buffer_info::Buffer::Yuv(buffer.into())),
+        }
+    }
+
+    pub fn from_i422(handle_id: FFIHandleId, buffer: &I422Buffer) -> Self {
+        Self {
+            handle: Some(handle_id.into()),
+            buffer_type: proto::VideoFrameBufferType::I422.into(),
+            width: buffer.width(),
+            height: buffer.height(),
+            buffer: Some(proto::video_frame_buffer_info::Buffer::Yuv(buffer.into())),
+        }
+    }
+
+    pub fn from_i444(handle_id: FFIHandleId, buffer: &I444Buffer) -> Self {
+        Self {
+            handle: Some(handle_id.into()),
+            buffer_type: proto::VideoFrameBufferType::I444.into(),
+            width: buffer.width(),
+            height: buffer.height(),
+            buffer: Some(proto::video_frame_buffer_info::Buffer::Yuv(buffer.into())),
+        }
+    }
+
+    pub fn from_i010(handle_id: FFIHandleId, buffer: &I010Buffer) -> Self {
+        Self {
+            handle: Some(handle_id.into()),
+            buffer_type: proto::VideoFrameBufferType::I010.into(),
+            width: buffer.width(),
+            height: buffer.height(),
+            buffer: Some(proto::video_frame_buffer_info::Buffer::Yuv(buffer.into())),
+        }
+    }
+
+    pub fn from_nv12(handle_id: FFIHandleId, buffer: &NV12Buffer) -> Self {
+        Self {
+            handle: Some(handle_id.into()),
+            buffer_type: proto::VideoFrameBufferType::Nv12.into(),
+            width: buffer.width(),
+            height: buffer.height(),
+            buffer: Some(proto::video_frame_buffer_info::Buffer::BiYuv(buffer.into())),
+        }
+    }
+}
+
+impl From<proto::VideoFormatType> for VideoFormatType {
+    fn from(format: proto::VideoFormatType) -> Self {
+        match format {
+            proto::VideoFormatType::FormatArgb => Self::ARGB,
+            proto::VideoFormatType::FormatBgra => Self::BGRA,
+            proto::VideoFormatType::FormatAbgr => Self::ABGR,
+            proto::VideoFormatType::FormatRgba => Self::RGBA,
+        }
+    }
+}
+
+impl From<VideoRotation> for proto::VideoRotation {
+    fn from(rotation: VideoRotation) -> proto::VideoRotation {
+        match rotation {
+            VideoRotation::VideoRotation0 => Self::VideoRotation0,
+            VideoRotation::VideoRotation90 => Self::VideoRotation90,
+            VideoRotation::VideoRotation180 => Self::VideoRotation180,
+            VideoRotation::VideoRotation270 => Self::VideoRotation270,
+        }
+    }
+}
+
+impl From<VideoFrameBufferType> for proto::VideoFrameBufferType {
+    fn from(buffer_type: VideoFrameBufferType) -> Self {
+        match buffer_type {
+            VideoFrameBufferType::Native => Self::Native,
+            VideoFrameBufferType::I420 => Self::I420,
+            VideoFrameBufferType::I420A => Self::I420a,
+            VideoFrameBufferType::I422 => Self::I422,
+            VideoFrameBufferType::I444 => Self::I444,
+            VideoFrameBufferType::I010 => Self::I010,
+            VideoFrameBufferType::NV12 => Self::Nv12,
+            VideoFrameBufferType::WebGl => Self::Webgl,
+            _ => panic!("unsupported buffer type on FFI server"),
+        }
+    }
+}

livekit-ffi/src/server/mod.rs

@@ -1,6 +1,7 @@
 use crate::proto;
 use lazy_static::lazy_static;
 use livekit::prelude::*;
+use livekit::webrtc::video_frame::{native::VideoFrameBufferExt, BoxVideoFrame, VideoFrameBuffer};
 use parking_lot::{Mutex, RwLock};
 use prost::Message;
 use std::any::Any;
@@ -166,11 +167,11 @@ impl FFIServer {
                 let buffer = self.release_handle(to_i420.buffer.unwrap().id as FFIHandleId);
 
                 if let Some(buffer) = buffer {
-                    if let Ok(buffer) = buffer.downcast::<VideoFrameBuffer>() {
+                    if let Ok(buffer) = buffer.downcast::<Box<dyn VideoFrameBuffer>>() {
                         let handle_id = self.next_handle_id();
-                        let i420 = VideoFrameBuffer::I420(buffer.to_i420());
+                        let buffer = buffer.to_i420();
-                        buffer_info = Some(proto::VideoFrameBufferInfo::from(handle_id, &i420));
+                        buffer_info = Some(proto::VideoFrameBufferInfo::from(handle_id, &buffer));
-                        self.insert_handle(handle_id, Box::new(i420));
+                        self.insert_handle(handle_id, Box::new(buffer));
                     }
                 }
 
@@ -188,7 +189,7 @@ impl FFIServer {
                 let buffer = ffi_owned.get(&(to_argb.buffer.unwrap().id as FFIHandleId));
 
                 if let Some(buffer) = buffer {
-                    if let Some(buffer) = buffer.downcast_ref::<VideoFrameBuffer>() {
+                    if let Some(buffer) = buffer.downcast_ref::<Box<dyn VideoFrameBuffer>>() {
                         let dst_buf = unsafe {
                             slice::from_raw_parts_mut(
                                 to_argb.dst_ptr as *mut u8,

livekit-ffi/src/server/room.rs

@@ -1,6 +1,8 @@
-use crate::proto::{self};
+use crate::proto;
 use crate::server::FFIServer;
+use futures_util::stream::StreamExt;
 use livekit::prelude::*;
+use livekit::webrtc::video_stream::native::NativeVideoStream;
 use tokio::sync::{mpsc, oneshot};
 
 pub async fn create_room(
@@ -67,9 +69,9 @@ async fn room_task(
                         publication: _,
                         participant: _,
                     } => {
-                        if let RemoteTrackHandle::Video(video_track) = track {
+                        if let RemoteTrack::Video(video_track) = track {
-                            let rtc_track = video_track.rtc_track();
+                            let video_stream = NativeVideoStream::new(video_track.rtc_track());
-                            rtc_track.on_frame(on_video_frame(server, video_track.sid()));
+                            tokio::spawn(video_frame_task(server, video_track.sid(), video_stream));
                         }
                     }
                     _ => {}
@@ -87,19 +89,20 @@ async fn room_task(
 async fn participant_task(participant: Participant) {
     let mut participant_events = participant.register_observer();
     while let Some(event) = participant_events.recv().await {
-        // TODO convert event to proto
+        // TODO(theomonnom): convert event to proto
     }
 }
 
-fn on_video_frame(server: &'static FFIServer, track_sid: TrackSid) -> OnFrameHandler {
+async fn video_frame_task(
-    // TODO(theomonnom): Should I use VideoSinkInfo here? (It'll help to have a more verbose
+    server: &'static FFIServer,
-    // lifetime)
+    track_sid: TrackSid,
-
+    mut stream: NativeVideoStream,
-    Box::new(move |frame, buffer| {
+) {
-        // Frame received, create a new FFIHandle from the video buffer.
+    while let Some(frame) = stream.next().await {
         let handle_id = server.next_handle_id();
-        let proto_buffer = proto::VideoFrameBufferInfo::from(handle_id, &buffer);
+        let frame_info = proto::VideoFrameInfo::from(&frame);
-        server.insert_handle(handle_id, Box::new(buffer));
+        let buffer_info = proto::VideoFrameBufferInfo::from(handle_id, &frame.buffer);
+        server.insert_handle(handle_id, Box::new(frame.buffer));
 
         // Send the received frame to the FFI language.
         let _ = server.send_event(
@@ -107,12 +110,12 @@ fn on_video_frame(server: &'static FFIServer, track_sid: TrackSid) -> OnFrameHan
                 track_sid: track_sid.to_string(),
                 message: Some(proto::track_event::Message::FrameReceived(
                     proto::FrameReceived {
-                        frame: Some(frame.into()),
+                        frame: Some(frame_info),
-                        frame_buffer: Some(proto_buffer),
+                        buffer: Some(buffer_info),
                     },
                 )),
             }),
             None,
         );
-    })
+    }
 }

livekit-utils/Cargo.toml

@@ -6,8 +6,7 @@ license = "Apache-2.0"
 description = "Shared utilities for livekit"
 repository = "https://github.com/livekit/client-sdk-rust" 
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
 [dependencies]
-parking_lot = "0.12.1"
+tokio = { version = "1.26.0", features = ["full"] }
-tokio = { version = "1", features = ["full"] }
+futures-util = "0.3"
+parking_lot = "0.12"

livekit-utils/src/enum_dispatch.rs

@@ -1,11 +1,9 @@
-
+// TODO(theomonnom): Async methods
-// TODO(theomonnom): Match the complete function signature like:
-//  - pub(crate) fn update_info(&self, info: ParticipantInfo) -> ();
 #[macro_export]
 macro_rules! enum_dispatch {
     // This arm is used to avoid nested loops with the arguments
-    // The arguments are transformed to $combined_args TokenTree
+    // The arguments are transformed to $combined_args tt
-    (@match $self:ident $fnc:ident $combined_args:tt [$($variant:ident),+]) => {
+    (@match [$($variant:ident),+]: $fnc:ident, $self:ident, $combined_args:tt) => {
         match $self {
             $(
                 Self::$variant(inner) => inner.$fnc$combined_args,
@@ -13,15 +11,17 @@ macro_rules! enum_dispatch {
         }
     };
 
-    ($vis:vis$(,)? $fnc:ident, $self:ty, [$($arg:ident: $t:ty),*], $ret:ty, [$($variant:ident),+]) => {
+    // Create the function and extract self fron the $args tt (little hack)
-        $vis fn $fnc(self: $self, $($arg: $t),*) -> $ret {
+    (@fnc [$($variant:ident),+]: $vis:vis fn $fnc:ident($self:ident: $sty:ty $(, $arg:ident: $t:ty)*) -> $ret:ty) => {
-            enum_dispatch!(@match self $fnc ($($arg,)*) [$($variant),+])
+        #[inline]
+        $vis fn $fnc($self: $sty, $($arg: $t),*) -> $ret {
+            enum_dispatch!(@match [$($variant),+]: $fnc, $self, ($($arg,)*))
         }
     };
 
-    ($variants:tt $(fnc!($vis:vis$(,)? $fnc:ident, $self:ty, $args:tt, $ret:ty);)+) => {
+    ($variants:tt; $($vis:vis fn $fnc:ident$args:tt -> $ret:ty;)+) => {
         $(
-            enum_dispatch!($vis, $fnc, $self, $args, $ret, $variants);
+            enum_dispatch!(@fnc $variants: $vis fn $fnc$args -> $ret);
-        )*
+        )+
     };
 }

livekit-utils/src/observer.rs

@@ -1,14 +1,16 @@
-// Really basic implementation of the observer pattern using mpsc channels.
+use futures_util::sink::Sink;
-// Currently unbounded channels
+use futures_util::task::{Context, Poll};
-
+use parking_lot::Mutex;
+use std::pin::Pin;
+use std::sync::Arc;
 use tokio::sync::mpsc;
 
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 pub struct Dispatcher<T>
 where
     T: Clone,
 {
-    senders: Vec<mpsc::UnboundedSender<T>>,
+    senders: Arc<Mutex<Vec<mpsc::UnboundedSender<T>>>>,
 }
 
 impl<T> Default for Dispatcher<T>
@@ -26,14 +28,43 @@ impl<T> Dispatcher<T>
 where
     T: Clone,
 {
-    pub fn register(&mut self) -> mpsc::UnboundedReceiver<T> {
+    pub fn register(&self) -> mpsc::UnboundedReceiver<T> {
         let (tx, rx) = mpsc::unbounded_channel();
-        self.senders.push(tx);
+        self.senders.lock().push(tx);
         rx
     }
 
-    pub fn dispatch(&mut self, msg: &T) {
+    pub fn dispatch(&self, msg: &T) {
         self.senders
+            .lock()
             .retain(|sender| sender.send(msg.clone()).is_ok());
     }
+
+    pub fn clear(&self) {
+        self.senders.lock().clear();
+    }
+}
+
+impl<T> Sink<T> for Dispatcher<T>
+where
+    T: Clone,
+{
+    type Error = ();
+
+    fn poll_ready(self: Pin<&mut Self>, _: &mut Context) -> Poll<Result<(), Self::Error>> {
+        Poll::Ready(Ok(()))
+    }
+
+    fn start_send(self: Pin<&mut Self>, item: T) -> Result<(), Self::Error> {
+        self.dispatch(&item);
+        Ok(())
+    }
+
+    fn poll_flush(self: Pin<&mut Self>, _: &mut Context) -> Poll<Result<(), Self::Error>> {
+        Poll::Ready(Ok(()))
+    }
+
+    fn poll_close(self: Pin<&mut Self>, _: &mut Context) -> Poll<Result<(), Self::Error>> {
+        Poll::Ready(Ok(()))
+    }
 }

livekit-webrtc/Cargo.toml

@@ -8,12 +8,35 @@ description = "Livekit safe bindings to libwebrtc"
 repository = "https://github.com/livekit/client-sdk-rust" 
 
 [dependencies]
-webrtc-sys = { path = "../webrtc-sys", version = "0.1.1" }
 livekit-utils = { path = "../livekit-utils", version = "0.1.1" }
-tokio = { version = "1", features = ["full"] }
-cxx = "1.0"
 log = "0.4"
 thiserror = "1.0"
 
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+webrtc-sys = { path = "../webrtc-sys", version = "0.1.1" }
+futures = { version = "0.3" }
+tokio = { version = "1.26.0", features = ["full"] }
+cxx = "1.0"
+
+[target.'cfg(target_arch = "wasm32")'.dependencies]
+wasm-bindgen = "0.2.84"
+js-sys = "0.3"
+wasm-bindgen-futures = "0.4.34"
+web-sys = { version = "0.3.22", features=[
+  "MessageEvent",
+  "RtcPeerConnection",
+  "RtcSignalingState",
+  "RtcSdpType",
+  "RtcSessionDescriptionInit",
+  "RtcPeerConnectionIceEvent",
+  "RtcIceCandidate",
+  "RtcDataChannel",
+  "RtcDataChannelEvent",
+  "RtcDataChannelState",
+  "EventTarget",
+  "WebGlRenderingContext",
+  "WebGlTexture",
+] }
+
 [dev-dependencies]
 env_logger = "0.9"

livekit-webrtc/src/data_channel.rs

@@ -1,152 +1,9 @@
-use std::error::Error;
+use crate::{imp::data_channel as dc_imp, rtp_parameters::Priority};
-use std::fmt::{Debug, Display, Formatter};
+use std::{fmt::Debug, str::Utf8Error};
-use std::sync::Mutex;
+use thiserror::Error;
 
-use cxx::UniquePtr;
+#[derive(Clone, Debug)]
-use log::trace;
-
-pub use sys_dc::ffi::{DataState, Priority};
-use webrtc_sys::data_channel as sys_dc;
-
-pub struct DataChannel {
-    cxx_handle: UniquePtr<sys_dc::ffi::DataChannel>,
-    observer: Box<InternalDataChannelObserver>,
-
-    // Keep alive for C++
-    native_observer: UniquePtr<sys_dc::ffi::NativeDataChannelObserver>,
-}
-
-impl Debug for DataChannel {
-    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
-        f.debug_struct("DataChannel")
-            .field("label", &self.label())
-            .finish()
-    }
-}
-
-#[derive(Debug)]
-pub struct DataSendError;
-
-impl Display for DataSendError {
-    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
-        write!(f, "failed to send data to the DataChannel")
-    }
-}
-
-impl Error for DataSendError {}
-
-impl DataChannel {
-    pub(crate) fn new(cxx_handle: UniquePtr<sys_dc::ffi::DataChannel>) -> Self {
-        let mut observer = Box::new(InternalDataChannelObserver::default());
-
-        let mut dc = unsafe {
-            Self {
-                cxx_handle,
-                native_observer: sys_dc::ffi::create_native_data_channel_observer(Box::new(
-                    sys_dc::DataChannelObserverWrapper::new(&mut *observer),
-                )),
-                observer,
-            }
-        };
-
-        unsafe {
-            dc.cxx_handle
-                .pin_mut()
-                .register_observer(dc.native_observer.pin_mut());
-        }
-
-        dc
-    }
-
-    pub fn send(&self, data: &[u8], binary: bool) -> Result<(), DataSendError> {
-        let buffer = sys_dc::ffi::DataBuffer {
-            ptr: data.as_ptr(),
-            len: data.len(),
-            binary,
-        };
-
-        self.cxx_handle
-            .send(&buffer)
-            .then_some(())
-            .ok_or(DataSendError {})
-    }
-
-    pub fn label(&self) -> String {
-        self.cxx_handle.label()
-    }
-
-    pub fn state(&self) -> DataState {
-        self.cxx_handle.state()
-    }
-
-    pub fn close(&self) {
-        self.cxx_handle.close();
-    }
-
-    pub fn on_state_change(&mut self, handler: OnStateChangeHandler) {
-        *self.observer.on_state_change_handler.lock().unwrap() = Some(handler);
-    }
-
-    pub fn on_message(&mut self, handler: OnMessageHandler) {
-        *self.observer.on_message_handler.lock().unwrap() = Some(handler);
-    }
-
-    pub fn on_buffered_amount_change(&mut self, handler: OnBufferedAmountChangeHandler) {
-        *self
-            .observer
-            .on_buffered_amount_change_handler
-            .lock()
-            .unwrap() = Some(handler);
-    }
-}
-
-impl Drop for DataChannel {
-    fn drop(&mut self) {
-        self.cxx_handle.pin_mut().unregister_observer();
-    }
-}
-
-pub type OnStateChangeHandler = Box<dyn FnMut() + Send + Sync>;
-pub type OnMessageHandler = Box<dyn FnMut(&[u8], bool) + Send + Sync>;
-pub type OnBufferedAmountChangeHandler = Box<dyn FnMut(u64) + Send + Sync>;
-
-#[derive(Default)]
-struct InternalDataChannelObserver {
-    on_state_change_handler: Mutex<Option<OnStateChangeHandler>>,
-    on_message_handler: Mutex<Option<OnMessageHandler>>,
-    on_buffered_amount_change_handler: Mutex<Option<OnBufferedAmountChangeHandler>>,
-}
-
-impl sys_dc::DataChannelObserver for InternalDataChannelObserver {
-    fn on_state_change(&self) {
-        trace!("DataChannel: on_state_change");
-        let mut handler = self.on_state_change_handler.lock().unwrap();
-        if let Some(f) = handler.as_mut() {
-            f();
-        }
-    }
-
-    fn on_message(&self, data: &[u8], is_binary: bool) {
-        trace!("DataChannel: on_message");
-        let mut handler = self.on_message_handler.lock().unwrap();
-        if let Some(f) = handler.as_mut() {
-            f(data, is_binary);
-        }
-    }
-
-    fn on_buffered_amount_change(&self, sent_data_size: u64) {
-        trace!("DataChannel: on_buffered_amount_change");
-        let mut handler = self.on_buffered_amount_change_handler.lock().unwrap();
-        if let Some(f) = handler.as_mut() {
-            f(sent_data_size);
-        }
-    }
-}
-
-#[derive(Debug)]
 pub struct DataChannelInit {
-    #[deprecated]
-    pub reliable: bool,
     pub ordered: bool,
     pub max_retransmit_time: Option<i32>,
     pub max_retransmits: Option<i32>,
@@ -159,11 +16,10 @@ pub struct DataChannelInit {
 impl Default for DataChannelInit {
     fn default() -> Self {
         Self {
-            reliable: false,
             ordered: true,
             max_retransmit_time: None,
             max_retransmits: None,
-            protocol: "".to_string(),
+            protocol: String::new(),
             negotiated: false,
             id: -1,
             priority: None,
@@ -171,20 +27,72 @@ impl Default for DataChannelInit {
     }
 }
 
-impl From<DataChannelInit> for sys_dc::ffi::DataChannelInit {
+#[derive(Debug, Error)]
-    fn from(init: DataChannelInit) -> Self {
+pub enum DataChannelError {
-        Self {
+    #[error("failed to send data, dc not open? send buffer is full ?")]
-            reliable: init.reliable,
+    Send,
-            ordered: init.ordered,
+    #[error("only utf8 strings can be sent")]
-            has_max_retransmit_time: init.max_retransmit_time.is_some(),
+    Utf8(#[from] Utf8Error),
-            max_retransmit_time: init.max_retransmit_time.unwrap_or_default(),
+}
-            has_max_retransmits: init.max_retransmits.is_some(),
+
-            max_retransmits: init.max_retransmits.unwrap_or_default(),
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-            protocol: init.protocol,
+pub enum DataState {
-            negotiated: init.negotiated,
+    Connecting,
-            id: init.id,
+    Open,
-            has_priority: init.priority.is_some(),
+    Closing,
-            priority: init.priority.unwrap_or(Priority::Low),
+    Closed,
+}
+
+#[derive(Debug)]
+pub struct DataBuffer<'a> {
+    pub data: &'a [u8],
+    pub binary: bool,
+}
+
+pub type OnStateChange = Box<dyn FnMut(DataState) + Send + Sync>;
+pub type OnMessage = Box<dyn FnMut(DataBuffer) + Send + Sync>;
+pub type OnBufferedAmountChange = Box<dyn FnMut(u64) + Send + Sync>;
+
+#[derive(Clone)]
+pub struct DataChannel {
+    pub(crate) handle: dc_imp::DataChannel,
+}
+
+impl DataChannel {
+    pub fn send(&self, data: &[u8], binary: bool) -> Result<(), DataChannelError> {
+        self.handle.send(data, binary)
     }
+
+    pub fn label(&self) -> String {
+        self.handle.label()
+    }
+
+    pub fn state(&self) -> DataState {
+        self.handle.state()
+    }
+
+    pub fn close(&self) {
+        self.handle.close()
+    }
+
+    pub fn on_state_change(&self, callback: Option<OnStateChange>) {
+        self.handle.on_state_change(callback)
+    }
+
+    pub fn on_message(&self, callback: Option<OnMessage>) {
+        self.handle.on_message(callback)
+    }
+
+    pub fn on_buffered_amount_change(&self, callback: Option<OnBufferedAmountChange>) {
+        self.handle.on_buffered_amount_change(callback)
+    }
+}
+
+impl Debug for DataChannel {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("DataChannel")
+            .field("label", &self.label())
+            .field("state", &self.state())
+            .finish()
     }
 }

livekit-webrtc/src/ice_candidate.rs

@@ -0,0 +1,40 @@
+use crate::{imp::ice_candidate as imp_ic, session_description::SdpParseError};
+use std::fmt::Debug;
+
+pub struct IceCandidate {
+    pub(crate) handle: imp_ic::IceCandidate,
+}
+
+impl IceCandidate {
+    pub fn parse(
+        sdp_mid: &str,
+        sdp_mline_index: i32,
+        sdp: &str,
+    ) -> Result<IceCandidate, SdpParseError> {
+        imp_ic::IceCandidate::parse(sdp_mid, sdp_mline_index, sdp)
+    }
+
+    pub fn sdp_mid(&self) -> String {
+        self.handle.sdp_mid()
+    }
+
+    pub fn sdp_mline_index(&self) -> i32 {
+        self.handle.sdp_mline_index()
+    }
+
+    pub fn candidate(&self) -> String {
+        self.handle.candidate()
+    }
+}
+
+impl ToString for IceCandidate {
+    fn to_string(&self) -> String {
+        self.handle.to_string()
+    }
+}
+
+impl Debug for IceCandidate {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("IceCandidate").finish()
+    }
+}

livekit-webrtc/src/jsep.rs

@@ -1,103 +0,0 @@
-use std::fmt::{Debug, Formatter};
-
-use cxx::{SharedPtr, UniquePtr};
-
-pub use sys_jsep::ffi::{SdpParseError, SdpType};
-use webrtc_sys::jsep as sys_jsep;
-
-#[derive(Clone)]
-pub struct IceCandidate {
-    cxx_handle: SharedPtr<sys_jsep::ffi::IceCandidate>,
-}
-
-impl Debug for IceCandidate {
-    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
-        write!(f, "IceCandidate[{}]", self.to_string())
-    }
-}
-
-impl IceCandidate {
-    pub fn from(
-        sdp_mid: &str,
-        sdp_mline_index: i32,
-        sdp: &str,
-    ) -> Result<IceCandidate, SdpParseError> {
-        let res = sys_jsep::ffi::create_ice_candidate(
-            sdp_mid.to_string(),
-            sdp_mline_index,
-            sdp.to_string(),
-        );
-
-        match res {
-            Ok(cxx_handle) => Ok(IceCandidate::new(cxx_handle)),
-            Err(e) => Err(unsafe { SdpParseError::from(e.what()) }),
-        }
-    }
-
-    pub(crate) fn new(cxx_handle: SharedPtr<sys_jsep::ffi::IceCandidate>) -> Self {
-        Self { cxx_handle }
-    }
-
-    pub(crate) fn release(self) -> SharedPtr<sys_jsep::ffi::IceCandidate> {
-        self.cxx_handle
-    }
-
-    pub fn sdp_mid(&self) -> String {
-        self.cxx_handle.sdp_mid()
-    }
-
-    pub fn sdp_mline_index(&self) -> i32 {
-        self.cxx_handle.sdp_mline_index()
-    }
-
-    pub fn candidate(&self) -> String {
-        self.cxx_handle.candidate()
-    }
-}
-
-impl ToString for IceCandidate {
-    fn to_string(&self) -> String {
-        self.cxx_handle.stringify()
-    }
-}
-
-pub struct SessionDescription {
-    cxx_handle: UniquePtr<sys_jsep::ffi::SessionDescription>,
-}
-
-impl Debug for SessionDescription {
-    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
-        write!(f, "SessionDescription[{}]", self.to_string())
-    }
-}
-
-impl SessionDescription {
-    pub fn from(sdp_type: SdpType, description: &str) -> Result<SessionDescription, SdpParseError> {
-        let res = sys_jsep::ffi::create_session_description(sdp_type, description.to_string());
-
-        match res {
-            Ok(cxx_handle) => Ok(SessionDescription::new(cxx_handle)),
-            Err(e) => Err(unsafe { SdpParseError::from(e.what()) }),
-        }
-    }
-
-    pub(crate) fn new(cxx_handle: UniquePtr<sys_jsep::ffi::SessionDescription>) -> Self {
-        Self { cxx_handle }
-    }
-
-    pub(crate) fn release(self) -> UniquePtr<sys_jsep::ffi::SessionDescription> {
-        self.cxx_handle
-    }
-}
-
-impl ToString for SessionDescription {
-    fn to_string(&self) -> String {
-        self.cxx_handle.stringify()
-    }
-}
-
-impl Clone for SessionDescription {
-    fn clone(&self) -> Self {
-        SessionDescription::new(self.cxx_handle.clone())
-    }
-}

livekit-webrtc/src/lib.rs

@@ -1,15 +1,48 @@
+use thiserror::Error;
+
+#[cfg_attr(target_arch = "wasm32", path = "web/mod.rs")]
+#[cfg_attr(not(target_arch = "wasm32"), path = "native/mod.rs")]
+mod imp;
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub enum MediaType {
+    Audio,
+    Video,
+    Data,
+    Unsupported,
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub enum RtcErrorType {
+    Internal,
+    InvalidSdp,
+    InvalidState,
+}
+
+#[derive(Error, Debug)]
+#[error("an RtcError occured: {error_type:?} - {message}")]
+pub struct RtcError {
+    pub error_type: RtcErrorType,
+    pub message: String,
+}
+
 pub mod data_channel;
-pub mod jsep;
+pub mod ice_candidate;
 pub mod media_stream;
 pub mod peer_connection;
 pub mod peer_connection_factory;
 pub mod prelude;
-pub mod rtc_error;
 pub mod rtp_parameters;
 pub mod rtp_receiver;
 pub mod rtp_sender;
 pub mod rtp_transceiver;
+pub mod session_description;
 pub mod video_frame;
-pub mod video_frame_buffer;
+pub mod video_source;
-pub mod webrtc;
+pub mod video_stream;
-pub mod yuv_helper;
+
+#[cfg(not(target_arch = "wasm32"))]
+pub mod native {
+    pub use crate::imp::yuv_helper;
+    pub use webrtc_sys::webrtc::ffi::create_random_uuid;
+}

livekit-webrtc/src/media_stream.rs

@@ -1,254 +1,139 @@
-use cxx::{SharedPtr, UniquePtr};
+use crate::imp::media_stream as imp_ms;
 use livekit_utils::enum_dispatch;
-use std::fmt::{Debug, Formatter};
+use std::fmt::Debug;
-use std::sync::{Arc, Mutex};
-use webrtc_sys::media_stream as sys_ms;
-use webrtc_sys::MEDIA_TYPE_VIDEO;
 
-pub use sys_ms::ffi::ContentHint;
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-pub use sys_ms::ffi::TrackState;
+pub enum RtcTrackState {
-
+    Live,
-use crate::video_frame::VideoFrame;
+    Ended,
-use crate::video_frame_buffer::VideoFrameBuffer;
-
-pub trait MediaStreamTrackTrait {
-    fn kind(&self) -> String;
-    fn id(&self) -> String;
-    fn enabled(&self) -> bool;
-    fn set_enabled(&self, enabled: bool) -> bool;
-    fn state(&self) -> TrackState;
 }
 
 #[derive(Clone)]
-pub enum MediaStreamTrackHandle {
+pub struct MediaStream {
-    Audio(Arc<AudioTrack>),
+    pub(crate) handle: imp_ms::MediaStream,
-    Video(Arc<VideoTrack>),
 }
 
-impl MediaStreamTrackHandle {
+impl MediaStream {
-    pub(crate) fn new(cxx_handle: SharedPtr<sys_ms::ffi::MediaStreamTrack>) -> Self {
+    pub fn id(&self) -> String {
-        if cxx_handle.kind() == MEDIA_TYPE_VIDEO {
+        self.handle.id()
-            Self::Video(VideoTrack::new(cxx_handle))
-        } else {
-            Self::Audio(AudioTrack::new(cxx_handle))
-        }
     }
 
-    // TODO(theomonnom): enum_dispatch with visibility support?
+    pub fn audio_tracks(&self) -> Vec<RtcAudioTrack> {
-    pub(crate) fn cxx_handle(&self) -> SharedPtr<sys_ms::ffi::MediaStreamTrack> {
+        self.handle.audio_tracks()
-        match self {
-            Self::Video(video) => video.cxx_handle(),
-            Self::Audio(audio) => audio.cxx_handle(),
     }
+
+    pub fn video_tracks(&self) -> Vec<RtcVideoTrack> {
+        self.handle.video_tracks()
     }
 }
 
-impl Debug for MediaStreamTrackHandle {
+impl Debug for MediaStream {
-    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("MediaStreamTrack")
+        f.debug_struct("MediaStream")
+            .field("id", &self.id())
+            .field("audio_tracks", &self.audio_tracks())
+            .field("video_tracks", &self.video_tracks())
+            .finish()
+    }
+}
+
+#[derive(Clone)]
+pub struct RtcVideoTrack {
+    pub(crate) handle: imp_ms::RtcVideoTrack,
+}
+
+#[derive(Clone)]
+pub struct RtcAudioTrack {
+    pub(crate) handle: imp_ms::RtcAudioTrack,
+}
+
+#[derive(Debug, Clone)]
+pub enum MediaStreamTrack {
+    Video(RtcVideoTrack),
+    Audio(RtcAudioTrack),
+}
+
+#[cfg(not(target_arch = "wasm32"))]
+impl MediaStreamTrack {
+    enum_dispatch!(
+        [Video, Audio];
+        pub(crate) fn sys_handle(self: &Self) -> cxx::SharedPtr<webrtc_sys::media_stream::ffi::MediaStreamTrack>;
+    );
+}
+
+impl MediaStreamTrack {
+    enum_dispatch!(
+        [Video, Audio];
+        pub fn id(self: &Self) -> String;
+        pub fn enabled(self: &Self) -> bool;
+        pub fn set_enabled(self: &Self, enabled: bool) -> bool;
+        pub fn state(self: &Self) -> RtcTrackState;
+    );
+}
+
+macro_rules! media_stream_track {
+    () => {
+        pub fn id(&self) -> String {
+            self.handle.id()
+        }
+
+        pub fn enabled(&self) -> bool {
+            self.handle.enabled()
+        }
+
+        pub fn set_enabled(&self, enabled: bool) -> bool {
+            self.handle.set_enabled(enabled)
+        }
+
+        pub fn state(&self) -> RtcTrackState {
+            self.handle.state().into()
+        }
+
+        #[cfg(not(target_arch = "wasm32"))]
+        pub(crate) fn sys_handle(
+            &self,
+        ) -> cxx::SharedPtr<webrtc_sys::media_stream::ffi::MediaStreamTrack> {
+            self.handle.sys_handle()
+        }
+    };
+}
+
+impl RtcVideoTrack {
+    media_stream_track!();
+}
+
+impl RtcAudioTrack {
+    media_stream_track!();
+}
+
+impl Debug for RtcAudioTrack {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("RtcAudioTrack")
             .field("id", &self.id())
-            .field("kind", &self.kind())
             .field("enabled", &self.enabled())
             .field("state", &self.state())
             .finish()
     }
 }
 
-impl MediaStreamTrackTrait for MediaStreamTrackHandle {
+impl Debug for RtcVideoTrack {
-    enum_dispatch!(
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        [Audio, Video]
+        f.debug_struct("RtcVideoTrack")
-        fnc!(kind, &Self, [], String);
-        fnc!(id, &Self, [], String);
-        fnc!(enabled, &Self, [], bool);
-        fnc!(state, &Self, [], TrackState);
-        fnc!(set_enabled, &Self, [enabled: bool], bool);
-    );
-}
-
-pub struct AudioTrack {
-    cxx_handle: SharedPtr<sys_ms::ffi::MediaStreamTrack>,
-}
-
-impl AudioTrack {
-    fn new(cxx_handle: SharedPtr<sys_ms::ffi::MediaStreamTrack>) -> Arc<Self> {
-        Arc::new(Self { cxx_handle })
-    }
-
-    pub(crate) fn cxx_handle(&self) -> SharedPtr<sys_ms::ffi::MediaStreamTrack> {
-        self.cxx_handle.clone()
-    }
-}
-
-pub struct VideoTrack {
-    cxx_handle: SharedPtr<sys_ms::ffi::MediaStreamTrack>,
-    observer: Box<InternalVideoTrackSink>,
-
-    // Keep alive for c++
-    native_observer: UniquePtr<sys_ms::ffi::NativeVideoFrameSink>,
-}
-
-impl VideoTrack {
-    fn new(cxx_handle: SharedPtr<sys_ms::ffi::MediaStreamTrack>) -> Arc<Self> {
-        let mut observer = Box::new(InternalVideoTrackSink::default());
-
-        let mut track = unsafe {
-            Self {
-                cxx_handle,
-                native_observer: {
-                    sys_ms::ffi::create_native_video_frame_sink(Box::new(
-                        sys_ms::VideoFrameSinkWrapper::new(&mut *observer),
-                    ))
-                },
-                observer,
-            }
-        };
-
-        unsafe {
-            (*track.video_handle()).add_sink(track.native_observer.pin_mut());
-        }
-
-        Arc::new(track)
-    }
-
-    pub(crate) fn cxx_handle(&self) -> SharedPtr<sys_ms::ffi::MediaStreamTrack> {
-        self.cxx_handle.clone()
-    }
-
-    fn video_handle(&self) -> *const sys_ms::ffi::VideoTrack {
-        unsafe { sys_ms::ffi::media_to_video(&*self.cxx_handle) }
-    }
-
-    pub fn set_should_receive(&self, should_receive: bool) {
-        unsafe { (*self.video_handle()).set_should_receive(should_receive) }
-    }
-
-    pub fn set_content_hint(&self, hint: ContentHint) {
-        unsafe { (*self.video_handle()).set_content_hint(hint) }
-    }
-
-    pub fn should_receive(&self) -> bool {
-        unsafe { (*self.video_handle()).should_receive() }
-    }
-
-    pub fn content_hint(&self) -> ContentHint {
-        unsafe { (*self.video_handle()).content_hint() }
-    }
-
-    pub fn on_frame(&self, handler: OnFrameHandler) {
-        *self.observer.on_frame_handler.lock().unwrap() = Some(handler);
-    }
-
-    pub fn on_discarded_frame(&self, handler: OnDiscardedFrameHandler) {
-        *self.observer.on_discarded_frame_handler.lock().unwrap() = Some(handler);
-    }
-
-    pub fn on_constraints_changed(&self, handler: OnConstraintsChangedHandler) {
-        *self.observer.on_constraints_changed_handler.lock().unwrap() = Some(handler);
-    }
-}
-
-impl Drop for VideoTrack {
-    fn drop(&mut self) {
-        unsafe {
-            (*self.video_handle()).remove_sink(self.native_observer.pin_mut());
-        }
-    }
-}
-
-macro_rules! impl_media_stream_track_trait {
-    ($x:ty) => {
-        impl MediaStreamTrackTrait for $x {
-            fn kind(&self) -> String {
-                self.cxx_handle.kind()
-            }
-
-            fn id(&self) -> String {
-                self.cxx_handle.id()
-            }
-
-            fn enabled(&self) -> bool {
-                self.cxx_handle.enabled()
-            }
-
-            fn set_enabled(&self, enabled: bool) -> bool {
-                self.cxx_handle.set_enabled(enabled)
-            }
-
-            fn state(&self) -> TrackState {
-                self.cxx_handle.state()
-            }
-        }
-    };
-}
-
-impl_media_stream_track_trait!(VideoTrack);
-impl_media_stream_track_trait!(AudioTrack);
-
-pub type OnFrameHandler = Box<dyn FnMut(VideoFrame, VideoFrameBuffer) + Send + Sync>;
-pub type OnDiscardedFrameHandler = Box<dyn FnMut() + Send + Sync>;
-pub type OnConstraintsChangedHandler = Box<dyn FnMut(VideoTrackSourceConstraints) + Send + Sync>;
-
-#[derive(Default)]
-struct InternalVideoTrackSink {
-    on_frame_handler: Mutex<Option<OnFrameHandler>>,
-    on_discarded_frame_handler: Mutex<Option<OnDiscardedFrameHandler>>,
-    on_constraints_changed_handler: Mutex<Option<OnConstraintsChangedHandler>>,
-}
-
-pub struct VideoTrackSourceConstraints {
-    pub min_fps: Option<f64>,
-    pub max_fps: Option<f64>,
-}
-
-impl From<sys_ms::ffi::VideoTrackSourceConstraints> for VideoTrackSourceConstraints {
-    fn from(cst: sys_ms::ffi::VideoTrackSourceConstraints) -> Self {
-        Self {
-            min_fps: (cst.min_fps != 1.0).then_some(cst.min_fps),
-            max_fps: (cst.max_fps != 1.0).then_some(cst.max_fps),
-        }
-    }
-}
-
-impl sys_ms::VideoFrameSink for InternalVideoTrackSink {
-    fn on_frame(&self, frame: UniquePtr<webrtc_sys::video_frame::ffi::VideoFrame>) {
-        if let Some(cb) = self.on_frame_handler.lock().unwrap().as_mut() {
-            let frame = VideoFrame::new(frame);
-            let video_frame_buffer = unsafe { frame.video_frame_buffer() };
-            cb(frame, video_frame_buffer);
-        }
-    }
-
-    fn on_discarded_frame(&self) {
-        if let Some(cb) = self.on_discarded_frame_handler.lock().unwrap().as_mut() {
-            cb();
-        }
-    }
-
-    fn on_constraints_changed(&self, constraints: sys_ms::ffi::VideoTrackSourceConstraints) {
-        if let Some(cb) = self.on_constraints_changed_handler.lock().unwrap().as_mut() {
-            cb(constraints.into());
-        }
-    }
-}
-
-pub struct MediaStream {
-    cxx_handle: SharedPtr<sys_ms::ffi::MediaStream>,
-}
-
-impl Debug for MediaStream {
-    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
-        f.debug_struct("MediaStream")
             .field("id", &self.id())
+            .field("enabled", &self.enabled())
+            .field("state", &self.state())
             .finish()
     }
 }
 
-impl MediaStream {
+impl From<RtcAudioTrack> for MediaStreamTrack {
-    pub(crate) fn new(cxx_handle: SharedPtr<sys_ms::ffi::MediaStream>) -> Self {
+    fn from(track: RtcAudioTrack) -> Self {
-        Self { cxx_handle }
+        Self::Audio(track)
     }
+}
 
-    pub fn id(&self) -> String {
+impl From<RtcVideoTrack> for MediaStreamTrack {
-        self.cxx_handle.id()
+    fn from(track: RtcVideoTrack) -> Self {
+        Self::Video(track)
     }
 }

livekit-webrtc/src/native/data_channel.rs

@@ -0,0 +1,137 @@
+use crate::data_channel::{
+    DataBuffer, DataChannelError, DataChannelInit, DataState, OnBufferedAmountChange, OnMessage,
+    OnStateChange,
+};
+use cxx::SharedPtr;
+use std::str;
+use std::sync::{Arc, Mutex};
+use webrtc_sys::data_channel as sys_dc;
+
+impl From<sys_dc::ffi::DataState> for DataState {
+    fn from(value: sys_dc::ffi::DataState) -> Self {
+        match value {
+            sys_dc::ffi::DataState::Connecting => Self::Connecting,
+            sys_dc::ffi::DataState::Open => Self::Open,
+            sys_dc::ffi::DataState::Closing => Self::Closing,
+            sys_dc::ffi::DataState::Closed => Self::Closed,
+            _ => panic!("unknown data channel state"),
+        }
+    }
+}
+
+impl From<DataChannelInit> for sys_dc::ffi::DataChannelInit {
+    fn from(value: DataChannelInit) -> Self {
+        Self {
+            ordered: value.ordered,
+            has_max_retransmit_time: value.max_retransmit_time.is_some(),
+            max_retransmit_time: value.max_retransmit_time.unwrap_or_default(),
+            has_max_retransmits: value.max_retransmits.is_some(),
+            max_retransmits: value.max_retransmits.unwrap_or_default(),
+            protocol: value.protocol,
+            id: value.id,
+            has_priority: false,
+            priority: sys_dc::ffi::Priority::Medium,
+            negotiated: value.negotiated,
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct DataChannel {
+    #[allow(dead_code)]
+    native_observer: SharedPtr<sys_dc::ffi::NativeDataChannelObserver>,
+    observer: Arc<DataChannelObserver>,
+
+    pub(crate) sys_handle: SharedPtr<sys_dc::ffi::DataChannel>,
+}
+
+impl DataChannel {
+    pub fn configure(sys_handle: SharedPtr<sys_dc::ffi::DataChannel>) -> Self {
+        unsafe {
+            let observer = Arc::new(DataChannelObserver::default());
+            let dc = Self {
+                sys_handle: sys_handle.clone(),
+                native_observer: sys_dc::ffi::create_native_data_channel_observer(
+                    Box::new(sys_dc::DataChannelObserverWrapper::new(observer.clone())),
+                    &*sys_handle as *const _ as *mut _,
+                ),
+                observer,
+            };
+
+            dc.sys_handle
+                .register_observer(&*dc.native_observer as *const _ as *mut _);
+            dc
+        }
+    }
+
+    pub fn send(&self, data: &[u8], binary: bool) -> Result<(), DataChannelError> {
+        if !binary {
+            str::from_utf8(data)?;
+        }
+
+        let buffer = sys_dc::ffi::DataBuffer {
+            ptr: data.as_ptr(),
+            len: data.len(),
+            binary,
+        };
+
+        self.sys_handle
+            .send(&buffer)
+            .then_some(())
+            .ok_or(DataChannelError::Send)
+    }
+
+    pub fn label(&self) -> String {
+        self.sys_handle.label()
+    }
+
+    pub fn state(&self) -> DataState {
+        self.sys_handle.state().into()
+    }
+
+    pub fn close(&self) {
+        self.sys_handle.close();
+    }
+
+    pub fn on_state_change(&self, handler: Option<OnStateChange>) {
+        *self.observer.state_change_handler.lock().unwrap() = handler;
+    }
+
+    pub fn on_message(&self, handler: Option<OnMessage>) {
+        *self.observer.message_handler.lock().unwrap() = handler;
+    }
+
+    pub fn on_buffered_amount_change(&self, handler: Option<OnBufferedAmountChange>) {
+        *self.observer.buffered_amount_change_handler.lock().unwrap() = handler;
+    }
+}
+
+#[derive(Default)]
+struct DataChannelObserver {
+    state_change_handler: Mutex<Option<OnStateChange>>,
+    message_handler: Mutex<Option<OnMessage>>,
+    buffered_amount_change_handler: Mutex<Option<OnBufferedAmountChange>>,
+}
+
+impl sys_dc::DataChannelObserver for DataChannelObserver {
+    fn on_state_change(&self, state: sys_dc::ffi::DataState) {
+        let mut handler = self.state_change_handler.lock().unwrap();
+        if let Some(f) = handler.as_mut() {
+            f(state.into());
+        }
+    }
+
+    fn on_message(&self, data: &[u8], binary: bool) {
+        let mut handler = self.message_handler.lock().unwrap();
+        if let Some(f) = handler.as_mut() {
+            f(DataBuffer { data, binary });
+        }
+    }
+
+    fn on_buffered_amount_change(&self, sent_data_size: u64) {
+        let mut handler = self.buffered_amount_change_handler.lock().unwrap();
+        if let Some(f) = handler.as_mut() {
+            f(sent_data_size);
+        }
+    }
+}

livekit-webrtc/src/native/ice_candidate.rs

@@ -0,0 +1,48 @@
+use crate::ice_candidate as ic;
+use crate::session_description::SdpParseError;
+use cxx::SharedPtr;
+use webrtc_sys::jsep as sys_jsep;
+
+#[derive(Clone)]
+pub struct IceCandidate {
+    pub(crate) sys_handle: SharedPtr<sys_jsep::ffi::IceCandidate>,
+}
+
+impl IceCandidate {
+    pub fn parse(
+        sdp_mid: &str,
+        sdp_mline_index: i32,
+        sdp: &str,
+    ) -> Result<ic::IceCandidate, SdpParseError> {
+        let res = sys_jsep::ffi::create_ice_candidate(
+            sdp_mid.to_string(),
+            sdp_mline_index,
+            sdp.to_string(),
+        );
+
+        match res {
+            Ok(sys_handle) => Ok(ic::IceCandidate {
+                handle: IceCandidate { sys_handle },
+            }),
+            Err(e) => Err(unsafe { sys_jsep::ffi::SdpParseError::from(e.what()).into() }),
+        }
+    }
+
+    pub fn sdp_mid(&self) -> String {
+        self.sys_handle.sdp_mid()
+    }
+
+    pub fn sdp_mline_index(&self) -> i32 {
+        self.sys_handle.sdp_mline_index()
+    }
+
+    pub fn candidate(&self) -> String {
+        self.sys_handle.candidate()
+    }
+}
+
+impl ToString for IceCandidate {
+    fn to_string(&self) -> String {
+        self.sys_handle.stringify()
+    }
+}

livekit-webrtc/src/native/media_stream.rs

@@ -0,0 +1,118 @@
+use crate::media_stream::{self, MediaStreamTrack, RtcTrackState};
+use cxx::SharedPtr;
+use webrtc_sys::media_stream as sys_ms;
+use webrtc_sys::media_stream::ffi::{
+    audio_to_media, media_to_audio, media_to_video, video_to_media,
+};
+use webrtc_sys::{MEDIA_TYPE_AUDIO, MEDIA_TYPE_VIDEO};
+
+impl From<sys_ms::ffi::TrackState> for RtcTrackState {
+    fn from(state: sys_ms::ffi::TrackState) -> Self {
+        match state {
+            sys_ms::ffi::TrackState::Live => RtcTrackState::Live,
+            sys_ms::ffi::TrackState::Ended => RtcTrackState::Ended,
+            _ => panic!("unknown TrackState"),
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct MediaStream {
+    pub(crate) sys_handle: SharedPtr<sys_ms::ffi::MediaStream>,
+}
+
+impl MediaStream {
+    pub fn id(&self) -> String {
+        self.sys_handle.id()
+    }
+
+    pub fn audio_tracks(&self) -> Vec<media_stream::RtcAudioTrack> {
+        self.sys_handle
+            .get_audio_tracks()
+            .into_iter()
+            .map(|t| media_stream::RtcAudioTrack {
+                handle: RtcAudioTrack { sys_handle: t.ptr },
+            })
+            .collect()
+    }
+
+    pub fn video_tracks(&self) -> Vec<media_stream::RtcVideoTrack> {
+        self.sys_handle
+            .get_video_tracks()
+            .into_iter()
+            .map(|t| media_stream::RtcVideoTrack {
+                handle: RtcVideoTrack { sys_handle: t.ptr },
+            })
+            .collect()
+    }
+}
+
+pub fn new_media_stream_track(
+    sys_handle: SharedPtr<sys_ms::ffi::MediaStreamTrack>,
+) -> MediaStreamTrack {
+    if sys_handle.kind() == MEDIA_TYPE_AUDIO {
+        MediaStreamTrack::Audio(media_stream::RtcAudioTrack {
+            handle: RtcAudioTrack {
+                sys_handle: media_to_audio(sys_handle),
+            },
+        })
+    } else if sys_handle.kind() == MEDIA_TYPE_VIDEO {
+        MediaStreamTrack::Video(media_stream::RtcVideoTrack {
+            handle: RtcVideoTrack {
+                sys_handle: media_to_video(sys_handle),
+            },
+        })
+    } else {
+        panic!("unknown track kind")
+    }
+}
+
+macro_rules! impl_media_stream_track {
+    ($cast:ident) => {
+        pub fn id(&self) -> String {
+            let ptr = sys_ms::ffi::$cast(self.sys_handle.clone());
+            ptr.id()
+        }
+
+        pub fn enabled(&self) -> bool {
+            let ptr = sys_ms::ffi::$cast(self.sys_handle.clone());
+            ptr.enabled()
+        }
+
+        pub fn set_enabled(&self, enabled: bool) -> bool {
+            let ptr = sys_ms::ffi::$cast(self.sys_handle.clone());
+            ptr.set_enabled(enabled)
+        }
+
+        pub fn state(&self) -> RtcTrackState {
+            let ptr = sys_ms::ffi::$cast(self.sys_handle.clone());
+            ptr.state().into()
+        }
+    };
+}
+
+#[derive(Clone)]
+pub struct RtcVideoTrack {
+    pub(crate) sys_handle: SharedPtr<sys_ms::ffi::VideoTrack>,
+}
+
+impl RtcVideoTrack {
+    impl_media_stream_track!(video_to_media);
+
+    pub fn sys_handle(&self) -> SharedPtr<sys_ms::ffi::MediaStreamTrack> {
+        video_to_media(self.sys_handle.clone())
+    }
+}
+
+#[derive(Clone)]
+pub struct RtcAudioTrack {
+    pub(crate) sys_handle: SharedPtr<sys_ms::ffi::AudioTrack>,
+}
+
+impl RtcAudioTrack {
+    impl_media_stream_track!(audio_to_media);
+
+    pub fn sys_handle(&self) -> SharedPtr<sys_ms::ffi::MediaStreamTrack> {
+        audio_to_media(self.sys_handle.clone())
+    }
+}

livekit-webrtc/src/native/mod.rs

@@ -0,0 +1,48 @@
+pub mod data_channel;
+pub mod ice_candidate;
+pub mod media_stream;
+pub mod peer_connection;
+pub mod peer_connection_factory;
+pub mod rtp_parameters;
+pub mod rtp_receiver;
+pub mod rtp_sender;
+pub mod rtp_transceiver;
+pub mod session_description;
+pub mod video_frame;
+pub mod video_source;
+pub mod video_stream;
+pub mod yuv_helper;
+
+use crate::MediaType;
+use crate::{RtcError, RtcErrorType};
+use webrtc_sys::rtc_error as sys_err;
+use webrtc_sys::webrtc as sys_rtc;
+
+impl From<sys_err::ffi::RTCErrorType> for RtcErrorType {
+    fn from(value: sys_err::ffi::RTCErrorType) -> Self {
+        match value {
+            sys_err::ffi::RTCErrorType::InvalidState => Self::InvalidState,
+            _ => Self::Internal,
+        }
+    }
+}
+
+impl From<sys_err::ffi::RTCError> for RtcError {
+    fn from(value: sys_err::ffi::RTCError) -> Self {
+        Self {
+            error_type: value.error_type.into(),
+            message: value.message,
+        }
+    }
+}
+
+impl From<MediaType> for sys_rtc::ffi::MediaType {
+    fn from(value: MediaType) -> Self {
+        match value {
+            MediaType::Audio => Self::Audio,
+            MediaType::Video => Self::Video,
+            MediaType::Data => Self::Data,
+            MediaType::Unsupported => Self::Unsupported,
+        }
+    }
+}

livekit-webrtc/src/native/peer_connection.rs

@@ -0,0 +1,579 @@
+use crate::data_channel::DataChannel;
+use crate::data_channel::DataChannelInit;
+use crate::ice_candidate::IceCandidate;
+use crate::imp::data_channel as imp_dc;
+use crate::imp::ice_candidate as imp_ic;
+use crate::imp::media_stream as imp_ms;
+use crate::imp::rtp_receiver as imp_rr;
+use crate::imp::rtp_sender as imp_rs;
+use crate::imp::rtp_transceiver as imp_rt;
+use crate::imp::session_description as imp_sdp;
+use crate::media_stream::{MediaStream, MediaStreamTrack};
+use crate::peer_connection::{
+    AnswerOptions, IceCandidateError, IceConnectionState, IceGatheringState, OfferOptions,
+    OnConnectionChange, OnDataChannel, OnIceCandidate, OnIceCandidateError, OnIceConnectionChange,
+    OnIceGatheringChange, OnNegotiationNeeded, OnSignalingChange, OnTrack, PeerConnectionState,
+    SignalingState, TrackEvent,
+};
+use crate::rtp_receiver::RtpReceiver;
+use crate::rtp_sender::RtpSender;
+use crate::rtp_transceiver::RtpTransceiver;
+use crate::rtp_transceiver::RtpTransceiverInit;
+use crate::MediaType;
+use crate::{session_description::SessionDescription, RtcError};
+use cxx::{SharedPtr, UniquePtr};
+use futures::channel::oneshot;
+use std::mem::ManuallyDrop;
+use std::sync::{Arc, Mutex};
+use webrtc_sys::data_channel as sys_dc;
+use webrtc_sys::jsep as sys_jsep;
+use webrtc_sys::peer_connection as sys_pc;
+use webrtc_sys::rtc_error as sys_err;
+
+impl From<OfferOptions> for sys_pc::ffi::RTCOfferAnswerOptions {
+    fn from(options: OfferOptions) -> Self {
+        Self {
+            ice_restart: options.ice_restart,
+            offer_to_receive_audio: options.offer_to_receive_audio as i32,
+            offer_to_receive_video: options.offer_to_receive_video as i32,
+            ..Default::default()
+        }
+    }
+}
+
+impl From<AnswerOptions> for sys_pc::ffi::RTCOfferAnswerOptions {
+    fn from(_options: AnswerOptions) -> Self {
+        Self::default()
+    }
+}
+
+impl From<sys_pc::ffi::PeerConnectionState> for PeerConnectionState {
+    fn from(state: sys_pc::ffi::PeerConnectionState) -> Self {
+        match state {
+            sys_pc::ffi::PeerConnectionState::New => PeerConnectionState::New,
+            sys_pc::ffi::PeerConnectionState::Connecting => PeerConnectionState::Connecting,
+            sys_pc::ffi::PeerConnectionState::Connected => PeerConnectionState::Connected,
+            sys_pc::ffi::PeerConnectionState::Disconnected => PeerConnectionState::Disconnected,
+            sys_pc::ffi::PeerConnectionState::Failed => PeerConnectionState::Failed,
+            sys_pc::ffi::PeerConnectionState::Closed => PeerConnectionState::Closed,
+            _ => panic!("unknown PeerConnectionState"),
+        }
+    }
+}
+
+impl From<sys_pc::ffi::IceConnectionState> for IceConnectionState {
+    fn from(state: sys_pc::ffi::IceConnectionState) -> Self {
+        match state {
+            sys_pc::ffi::IceConnectionState::IceConnectionNew => IceConnectionState::New,
+            sys_pc::ffi::IceConnectionState::IceConnectionChecking => IceConnectionState::Checking,
+            sys_pc::ffi::IceConnectionState::IceConnectionConnected => {
+                IceConnectionState::Connected
+            }
+            sys_pc::ffi::IceConnectionState::IceConnectionCompleted => {
+                IceConnectionState::Completed
+            }
+            sys_pc::ffi::IceConnectionState::IceConnectionFailed => IceConnectionState::Failed,
+            sys_pc::ffi::IceConnectionState::IceConnectionDisconnected => {
+                IceConnectionState::Disconnected
+            }
+            sys_pc::ffi::IceConnectionState::IceConnectionClosed => IceConnectionState::Closed,
+            sys_pc::ffi::IceConnectionState::IceConnectionMax => IceConnectionState::Max,
+            _ => panic!("unknown IceConnectionState"),
+        }
+    }
+}
+
+impl From<sys_pc::ffi::IceGatheringState> for IceGatheringState {
+    fn from(state: sys_pc::ffi::IceGatheringState) -> Self {
+        match state {
+            sys_pc::ffi::IceGatheringState::IceGatheringNew => IceGatheringState::New,
+            sys_pc::ffi::IceGatheringState::IceGatheringGathering => IceGatheringState::Gathering,
+            sys_pc::ffi::IceGatheringState::IceGatheringComplete => IceGatheringState::Complete,
+            _ => panic!("unknown IceGatheringState"),
+        }
+    }
+}
+
+impl From<sys_pc::ffi::SignalingState> for SignalingState {
+    fn from(state: sys_pc::ffi::SignalingState) -> Self {
+        match state {
+            sys_pc::ffi::SignalingState::Stable => SignalingState::Stable,
+            sys_pc::ffi::SignalingState::HaveLocalOffer => SignalingState::HaveLocalOffer,
+            sys_pc::ffi::SignalingState::HaveRemoteOffer => SignalingState::HaveRemoteOffer,
+            sys_pc::ffi::SignalingState::HaveLocalPrAnswer => SignalingState::HaveLocalPrAnswer,
+            sys_pc::ffi::SignalingState::HaveRemotePrAnswer => SignalingState::HaveRemotePrAnswer,
+            sys_pc::ffi::SignalingState::Closed => SignalingState::Closed,
+            _ => panic!("unknown SignalingState"),
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct PeerConnection {
+    native_observer: SharedPtr<sys_pc::ffi::NativePeerConnectionObserver>,
+    observer: Arc<PeerObserver>,
+
+    pub(crate) sys_handle: SharedPtr<sys_pc::ffi::PeerConnection>,
+}
+
+impl PeerConnection {
+    pub fn configure(
+        sys_handle: SharedPtr<sys_pc::ffi::PeerConnection>,
+        observer: Arc<PeerObserver>,
+        native_observer: SharedPtr<sys_pc::ffi::NativePeerConnectionObserver>,
+    ) -> Self {
+        Self {
+            sys_handle,
+            observer,
+            native_observer,
+        }
+    }
+
+    pub async fn create_offer(
+        &self,
+        options: OfferOptions,
+    ) -> Result<SessionDescription, RtcError> {
+        let (mut native_wrapper, mut sdp_rx, mut err_rx) = create_sdp_observer();
+
+        unsafe {
+            self.sys_handle
+                .create_offer(native_wrapper.pin_mut(), options.into());
+        }
+
+        futures::select! {
+            sdp = sdp_rx => Ok(sdp.unwrap()),
+            err = err_rx => Err(err.unwrap()),
+        }
+    }
+
+    pub async fn create_answer(
+        &self,
+        options: AnswerOptions,
+    ) -> Result<SessionDescription, RtcError> {
+        let (mut native_wrapper, mut sdp_rx, mut err_rx) = create_sdp_observer();
+
+        unsafe {
+            self.sys_handle
+                .create_answer(native_wrapper.pin_mut(), options.into());
+        }
+
+        futures::select! {
+            sdp = sdp_rx => Ok(sdp.unwrap()),
+            err = err_rx => Err(err.unwrap()),
+        }
+    }
+
+    pub async fn set_local_description(&self, desc: SessionDescription) -> Result<(), RtcError> {
+        let (tx, rx) = oneshot::channel();
+        let wrapper =
+            sys_jsep::SetLocalSdpObserverWrapper(ManuallyDrop::new(Box::new(move |error| {
+                let _ = tx.send(if error.ok() { Ok(()) } else { Err(error) });
+            })));
+
+        let mut native_wrapper =
+            sys_jsep::ffi::create_native_set_local_sdp_observer(Box::new(wrapper));
+
+        unsafe {
+            self.sys_handle
+                .set_local_description(desc.handle.sys_handle, native_wrapper.pin_mut());
+        }
+
+        rx.await.unwrap().map_err(Into::into)
+    }
+
+    pub async fn set_remote_description(&self, desc: SessionDescription) -> Result<(), RtcError> {
+        let (tx, rx) = oneshot::channel();
+        let wrapper =
+            sys_jsep::SetRemoteSdpObserverWrapper(ManuallyDrop::new(Box::new(move |error| {
+                let _ = tx.send(if error.ok() { Ok(()) } else { Err(error) });
+            })));
+
+        let mut native_wrapper =
+            sys_jsep::ffi::create_native_set_remote_sdp_observer(Box::new(wrapper));
+
+        unsafe {
+            self.sys_handle
+                .set_remote_description(desc.handle.sys_handle, native_wrapper.pin_mut());
+        }
+
+        rx.await.unwrap().map_err(Into::into)
+    }
+
+    pub async fn add_ice_candidate(&self, candidate: IceCandidate) -> Result<(), RtcError> {
+        let (tx, rx) = oneshot::channel();
+        let observer =
+            sys_pc::AddIceCandidateObserverWrapper(ManuallyDrop::new(Box::new(|error| {
+                let _ = tx.send(if error.ok() { Ok(()) } else { Err(error) });
+            })));
+
+        let mut native_observer =
+            sys_pc::ffi::create_native_add_ice_candidate_observer(Box::new(observer));
+        self.sys_handle
+            .add_ice_candidate(candidate.handle.sys_handle, native_observer.pin_mut());
+
+        rx.await.unwrap().map_err(Into::into)
+    }
+
+    pub fn create_data_channel(
+        &self,
+        label: &str,
+        init: DataChannelInit,
+    ) -> Result<DataChannel, RtcError> {
+        let native_init = sys_dc::ffi::create_data_channel_init(init.into());
+        let res = self
+            .sys_handle
+            .create_data_channel(label.to_string(), native_init);
+
+        match res {
+            Ok(sys_handle) => Ok(DataChannel {
+                handle: imp_dc::DataChannel::configure(sys_handle),
+            }),
+            Err(e) => Err(unsafe { sys_err::ffi::RTCError::from(e.what()).into() }),
+        }
+    }
+
+    pub fn add_track<T: AsRef<str>>(
+        &self,
+        track: MediaStreamTrack,
+        stream_ids: &[T],
+    ) -> Result<RtpSender, RtcError> {
+        let stream_ids = stream_ids.iter().map(|s| s.as_ref().to_owned()).collect();
+        let res = self.sys_handle.add_track(track.sys_handle(), &stream_ids);
+
+        match res {
+            Ok(sys_handle) => Ok(RtpSender {
+                handle: imp_rs::RtpSender { sys_handle },
+            }),
+            Err(e) => unsafe { Err(sys_err::ffi::RTCError::from(e.what()).into()) },
+        }
+    }
+
+    pub fn add_transceiver(
+        &self,
+        track: MediaStreamTrack,
+        init: RtpTransceiverInit,
+    ) -> Result<RtpTransceiver, RtcError> {
+        let res = self
+            .sys_handle
+            .add_transceiver(track.sys_handle(), init.into());
+
+        match res {
+            Ok(sys_handle) => Ok(RtpTransceiver {
+                handle: imp_rt::RtpTransceiver {
+                    sys_handle: sys_handle,
+                },
+            }),
+            Err(e) => unsafe { Err(sys_err::ffi::RTCError::from(e.what()).into()) },
+        }
+    }
+
+    pub fn add_transceiver_for_media(
+        &self,
+        media_type: MediaType,
+        init: RtpTransceiverInit,
+    ) -> Result<RtpTransceiver, RtcError> {
+        let res = self
+            .sys_handle
+            .add_transceiver_for_media(media_type.into(), init.into());
+
+        match res {
+            Ok(cxx_handle) => Ok(RtpTransceiver {
+                handle: imp_rt::RtpTransceiver {
+                    sys_handle: cxx_handle,
+                },
+            }),
+            Err(e) => unsafe { Err(sys_err::ffi::RTCError::from(e.what()).into()) },
+        }
+    }
+
+    pub fn close(&self) {
+        self.sys_handle.close();
+    }
+
+    pub fn connection_state(&self) -> PeerConnectionState {
+        self.sys_handle.connection_state().into()
+    }
+
+    pub fn ice_connection_state(&self) -> IceConnectionState {
+        self.sys_handle.ice_connection_state().into()
+    }
+
+    pub fn ice_gathering_state(&self) -> IceGatheringState {
+        self.sys_handle.ice_gathering_state().into()
+    }
+
+    pub fn signaling_state(&self) -> SignalingState {
+        self.sys_handle.signaling_state().into()
+    }
+
+    pub fn current_local_description(&self) -> Option<SessionDescription> {
+        let sdp = self.sys_handle.current_local_description();
+        if sdp.is_null() {
+            return None;
+        }
+
+        Some(SessionDescription {
+            handle: imp_sdp::SessionDescription { sys_handle: sdp },
+        })
+    }
+
+    pub fn current_remote_description(&self) -> Option<SessionDescription> {
+        let sdp = self.sys_handle.current_remote_description();
+        if sdp.is_null() {
+            return None;
+        }
+
+        Some(SessionDescription {
+            handle: imp_sdp::SessionDescription { sys_handle: sdp },
+        })
+    }
+
+    pub fn remove_track(&self, sender: RtpSender) -> Result<(), RtcError> {
+        self.sys_handle
+            .remove_track(sender.handle.sys_handle)
+            .map_err(|e| unsafe { sys_err::ffi::RTCError::from(e.what()).into() })
+    }
+
+    pub fn senders(&self) -> Vec<RtpSender> {
+        self.sys_handle
+            .get_senders()
+            .into_iter()
+            .map(|sender| RtpSender {
+                handle: imp_rs::RtpSender {
+                    sys_handle: sender.ptr,
+                },
+            })
+            .collect()
+    }
+
+    pub fn receivers(&self) -> Vec<RtpReceiver> {
+        self.sys_handle
+            .get_receivers()
+            .into_iter()
+            .map(|receiver| RtpReceiver {
+                handle: imp_rr::RtpReceiver {
+                    sys_handle: receiver.ptr,
+                },
+            })
+            .collect()
+    }
+
+    pub fn transceivers(&self) -> Vec<RtpTransceiver> {
+        self.sys_handle
+            .get_transceivers()
+            .into_iter()
+            .map(|transceiver| RtpTransceiver {
+                handle: imp_rt::RtpTransceiver {
+                    sys_handle: transceiver.ptr,
+                },
+            })
+            .collect()
+    }
+
+    pub fn on_connection_state_change(&self, f: Option<OnConnectionChange>) {
+        *self.observer.connection_change_handler.lock().unwrap() = f;
+    }
+
+    pub fn on_data_channel(&self, f: Option<OnDataChannel>) {
+        *self.observer.data_channel_handler.lock().unwrap() = f;
+    }
+
+    pub fn on_ice_candidate(&self, f: Option<OnIceCandidate>) {
+        *self.observer.ice_candidate_handler.lock().unwrap() = f;
+    }
+
+    pub fn on_ice_candidate_error(&self, f: Option<OnIceCandidateError>) {
+        *self.observer.ice_candidate_error_handler.lock().unwrap() = f;
+    }
+
+    pub fn on_ice_connection_state_change(&self, f: Option<OnIceConnectionChange>) {
+        *self.observer.ice_connection_change_handler.lock().unwrap() = f;
+    }
+
+    pub fn on_ice_gathering_state_change(&self, f: Option<OnIceGatheringChange>) {
+        *self.observer.ice_gathering_change_handler.lock().unwrap() = f;
+    }
+
+    pub fn on_negotiation_needed(&self, f: Option<OnNegotiationNeeded>) {
+        *self.observer.negotiation_needed_handler.lock().unwrap() = f;
+    }
+
+    pub fn on_signaling_state_change(&self, f: Option<OnSignalingChange>) {
+        *self.observer.signaling_change_handler.lock().unwrap() = f;
+    }
+
+    pub fn on_track(&self, f: Option<OnTrack>) {
+        *self.observer.track_handler.lock().unwrap() = f;
+    }
+}
+
+fn create_sdp_observer() -> (
+    UniquePtr<sys_pc::ffi::NativeCreateSdpObserverHandle>,
+    oneshot::Receiver<SessionDescription>,
+    oneshot::Receiver<RtcError>,
+) {
+    let (sdp_tx, sdp_rx) = oneshot::channel();
+    let (err_tx, err_rx) = oneshot::channel();
+
+    let wrapper = sys_jsep::CreateSdpObserverWrapper {
+        on_success: ManuallyDrop::new(Box::new(move |session_description| {
+            let _ = sdp_tx.send(SessionDescription {
+                handle: imp_sdp::SessionDescription {
+                    sys_handle: session_description,
+                },
+            });
+        })),
+        on_failure: ManuallyDrop::new(Box::new(move |error| {
+            let _ = err_tx.send(error.into());
+        })),
+    };
+
+    (
+        sys_jsep::ffi::create_native_create_sdp_observer(Box::new(wrapper)),
+        sdp_rx,
+        err_rx,
+    )
+}
+
+#[derive(Default)]
+pub struct PeerObserver {
+    pub connection_change_handler: Mutex<Option<OnConnectionChange>>,
+    pub data_channel_handler: Mutex<Option<OnDataChannel>>,
+    pub ice_candidate_handler: Mutex<Option<OnIceCandidate>>,
+    pub ice_candidate_error_handler: Mutex<Option<OnIceCandidateError>>,
+    pub ice_connection_change_handler: Mutex<Option<OnIceConnectionChange>>,
+    pub ice_gathering_change_handler: Mutex<Option<OnIceGatheringChange>>,
+    pub negotiation_needed_handler: Mutex<Option<OnNegotiationNeeded>>,
+    pub signaling_change_handler: Mutex<Option<OnSignalingChange>>,
+    pub track_handler: Mutex<Option<OnTrack>>,
+}
+
+impl sys_pc::PeerConnectionObserver for PeerObserver {
+    fn on_signaling_change(&self, new_state: sys_pc::ffi::SignalingState) {
+        if let Some(f) = self.signaling_change_handler.lock().unwrap().as_mut() {
+            f(new_state.into());
+        }
+    }
+
+    fn on_add_stream(&self, _stream: SharedPtr<webrtc_sys::media_stream::ffi::MediaStream>) {}
+
+    fn on_remove_stream(&self, _stream: SharedPtr<webrtc_sys::media_stream::ffi::MediaStream>) {}
+
+    fn on_data_channel(&self, data_channel: SharedPtr<sys_dc::ffi::DataChannel>) {
+        if let Some(f) = self.data_channel_handler.lock().unwrap().as_mut() {
+            f(DataChannel {
+                handle: imp_dc::DataChannel::configure(data_channel),
+            });
+        }
+    }
+
+    fn on_renegotiation_needed(&self) {}
+
+    fn on_negotiation_needed_event(&self, event: u32) {
+        if let Some(f) = self.negotiation_needed_handler.lock().unwrap().as_mut() {
+            f(event);
+        }
+    }
+
+    fn on_ice_connection_change(&self, _new_state: sys_pc::ffi::IceConnectionState) {}
+
+    fn on_standardized_ice_connection_change(&self, new_state: sys_pc::ffi::IceConnectionState) {
+        if let Some(f) = self.ice_connection_change_handler.lock().unwrap().as_mut() {
+            f(new_state.into());
+        }
+    }
+
+    fn on_connection_change(&self, new_state: sys_pc::ffi::PeerConnectionState) {
+        if let Some(f) = self.connection_change_handler.lock().unwrap().as_mut() {
+            f(new_state.into());
+        }
+    }
+
+    fn on_ice_gathering_change(&self, new_state: sys_pc::ffi::IceGatheringState) {
+        if let Some(f) = self.ice_gathering_change_handler.lock().unwrap().as_mut() {
+            f(new_state.into());
+        }
+    }
+
+    fn on_ice_candidate(&self, candidate: SharedPtr<sys_jsep::ffi::IceCandidate>) {
+        if let Some(f) = self.ice_candidate_handler.lock().unwrap().as_mut() {
+            f(IceCandidate {
+                handle: imp_ic::IceCandidate {
+                    sys_handle: candidate,
+                },
+            });
+        }
+    }
+
+    fn on_ice_candidate_error(
+        &self,
+        address: String,
+        port: i32,
+        url: String,
+        error_code: i32,
+        error_text: String,
+    ) {
+        if let Some(f) = self.ice_candidate_error_handler.lock().unwrap().as_mut() {
+            f(IceCandidateError {
+                address,
+                port,
+                url,
+                error_code,
+                error_text,
+            });
+        }
+    }
+
+    fn on_ice_candidates_removed(
+        &self,
+        _removed: Vec<SharedPtr<webrtc_sys::candidate::ffi::Candidate>>,
+    ) {
+    }
+
+    fn on_ice_connection_receiving_change(&self, _receiving: bool) {}
+
+    fn on_ice_selected_candidate_pair_changed(
+        &self,
+        _event: sys_pc::ffi::CandidatePairChangeEvent,
+    ) {
+    }
+
+    fn on_add_track(
+        &self,
+        _receiver: SharedPtr<webrtc_sys::rtp_receiver::ffi::RtpReceiver>,
+        _streams: Vec<SharedPtr<webrtc_sys::media_stream::ffi::MediaStream>>,
+    ) {
+    }
+
+    fn on_track(&self, transceiver: SharedPtr<webrtc_sys::rtp_transceiver::ffi::RtpTransceiver>) {
+        if let Some(f) = self.track_handler.lock().unwrap().as_mut() {
+            let receiver = transceiver.receiver();
+            let streams = receiver.streams();
+            let track = receiver.track();
+
+            f(TrackEvent {
+                receiver: RtpReceiver {
+                    handle: imp_rr::RtpReceiver {
+                        sys_handle: receiver,
+                    },
+                },
+                streams: streams
+                    .into_iter()
+                    .map(|s| MediaStream {
+                        handle: imp_ms::MediaStream { sys_handle: s.ptr },
+                    })
+                    .collect(),
+                track: imp_ms::new_media_stream_track(track),
+                transceiver: RtpTransceiver {
+                    handle: imp_rt::RtpTransceiver {
+                        sys_handle: transceiver,
+                    },
+                },
+            });
+        }
+    }
+
+    fn on_remove_track(&self, _receiver: SharedPtr<webrtc_sys::rtp_receiver::ffi::RtpReceiver>) {}
+
+    fn on_interesting_usage(&self, _usage_pattern: i32) {}
+}

livekit-webrtc/src/native/peer_connection_factory.rs

@@ -0,0 +1,146 @@
+use crate::imp::media_stream as imp_ms;
+use crate::imp::peer_connection as imp_pc;
+use crate::media_stream::RtcVideoTrack;
+use crate::peer_connection::PeerConnection;
+use crate::peer_connection_factory::{
+    ContinualGatheringPolicy, IceServer, IceTransportsType, RtcConfiguration,
+};
+use crate::rtp_parameters::RtpCapabilities;
+use crate::video_source::native::NativeVideoSource;
+use crate::MediaType;
+use crate::RtcError;
+use cxx::SharedPtr;
+use std::sync::Arc;
+use webrtc_sys::peer_connection as sys_pc;
+use webrtc_sys::peer_connection_factory as sys_pcf;
+use webrtc_sys::rtc_error as sys_err;
+use webrtc_sys::webrtc as sys_webrtc;
+
+impl From<IceServer> for sys_pcf::ffi::ICEServer {
+    fn from(value: IceServer) -> Self {
+        sys_pcf::ffi::ICEServer {
+            urls: value.urls,
+            username: value.username,
+            password: value.password,
+        }
+    }
+}
+
+impl From<ContinualGatheringPolicy> for sys_pcf::ffi::ContinualGatheringPolicy {
+    fn from(value: ContinualGatheringPolicy) -> Self {
+        match value {
+            ContinualGatheringPolicy::GatherOnce => {
+                sys_pcf::ffi::ContinualGatheringPolicy::GatherOnce
+            }
+            ContinualGatheringPolicy::GatherContinually => {
+                sys_pcf::ffi::ContinualGatheringPolicy::GatherContinually
+            }
+        }
+    }
+}
+
+impl From<IceTransportsType> for sys_pcf::ffi::IceTransportsType {
+    fn from(value: IceTransportsType) -> Self {
+        match value {
+            IceTransportsType::None => sys_pcf::ffi::IceTransportsType::None,
+            IceTransportsType::Relay => sys_pcf::ffi::IceTransportsType::Relay,
+            IceTransportsType::NoHost => sys_pcf::ffi::IceTransportsType::NoHost,
+            IceTransportsType::All => sys_pcf::ffi::IceTransportsType::All,
+        }
+    }
+}
+
+impl From<RtcConfiguration> for sys_pcf::ffi::RTCConfiguration {
+    fn from(value: RtcConfiguration) -> Self {
+        Self {
+            ice_servers: value.ice_servers.into_iter().map(Into::into).collect(),
+            continual_gathering_policy: value.continual_gathering_policy.into(),
+            ice_transport_type: value.ice_transport_type.into(),
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct RTCRuntime {
+    pub(crate) sys_handle: SharedPtr<sys_webrtc::ffi::RTCRuntime>,
+}
+
+impl Default for RTCRuntime {
+    fn default() -> Self {
+        Self {
+            sys_handle: sys_webrtc::ffi::create_rtc_runtime(),
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct PeerConnectionFactory {
+    sys_handle: SharedPtr<sys_pcf::ffi::PeerConnectionFactory>,
+
+    #[allow(unused)]
+    runtime: RTCRuntime,
+}
+
+impl Default for PeerConnectionFactory {
+    fn default() -> Self {
+        let runtime = RTCRuntime::default();
+        Self {
+            sys_handle: sys_pcf::ffi::create_peer_connection_factory(runtime.sys_handle.clone()),
+            runtime,
+        }
+    }
+}
+
+impl PeerConnectionFactory {
+    pub fn create_peer_connection(
+        &self,
+        config: RtcConfiguration,
+    ) -> Result<PeerConnection, RtcError> {
+        let native_config = sys_pcf::ffi::create_rtc_configuration(config.into());
+
+        unsafe {
+            let observer = Arc::new(imp_pc::PeerObserver::default());
+            let native_observer = sys_pc::ffi::create_native_peer_connection_observer(
+                self.runtime.clone().sys_handle,
+                Box::new(sys_pc::PeerConnectionObserverWrapper::new(observer.clone())),
+            );
+
+            let res = self
+                .sys_handle
+                .create_peer_connection(native_config, &*native_observer as *const _ as *mut _);
+
+            match res {
+                Ok(sys_handle) => Ok(PeerConnection {
+                    handle: imp_pc::PeerConnection::configure(
+                        sys_handle,
+                        observer,
+                        native_observer,
+                    ),
+                }),
+                Err(e) => Err(sys_err::ffi::RTCError::from(e.what()).into()),
+            }
+        }
+    }
+
+    pub fn create_video_track(&self, label: &str, source: NativeVideoSource) -> RtcVideoTrack {
+        RtcVideoTrack {
+            handle: imp_ms::RtcVideoTrack {
+                sys_handle: self
+                    .sys_handle
+                    .create_video_track(label.to_string(), source.handle.sys_handle()),
+            },
+        }
+    }
+
+    pub fn get_rtp_sender_capabilities(&self, media_type: MediaType) -> RtpCapabilities {
+        self.sys_handle
+            .get_rtp_sender_capabilities(media_type.into())
+            .into()
+    }
+
+    pub fn get_rtp_receiver_capabilities(&self, media_type: MediaType) -> RtpCapabilities {
+        self.sys_handle
+            .get_rtp_receiver_capabilities(media_type.into())
+            .into()
+    }
+}

livekit-webrtc/src/native/rtp_parameters.rs

@@ -0,0 +1,285 @@
+use crate::rtp_parameters::*;
+use crate::MediaType;
+use webrtc_sys::rtp_parameters as sys_rp;
+use webrtc_sys::webrtc as sys_webrtc;
+
+impl From<sys_webrtc::ffi::Priority> for Priority {
+    fn from(value: sys_webrtc::ffi::Priority) -> Self {
+        match value {
+            sys_webrtc::ffi::Priority::VeryLow => Self::VeryLow,
+            sys_webrtc::ffi::Priority::Low => Self::Low,
+            sys_webrtc::ffi::Priority::Medium => Self::Medium,
+            sys_webrtc::ffi::Priority::High => Self::High,
+            _ => panic!("unknown Priority"),
+        }
+    }
+}
+
+impl From<sys_rp::ffi::RtpExtension> for RtpHeaderExtensionParameters {
+    fn from(value: sys_rp::ffi::RtpExtension) -> Self {
+        Self {
+            uri: value.uri,
+            id: value.id,
+            encrypted: value.encrypt,
+        }
+    }
+}
+
+impl From<sys_rp::ffi::RtpParameters> for RtpParameters {
+    fn from(value: sys_rp::ffi::RtpParameters) -> Self {
+        Self {
+            codecs: value.codecs.into_iter().map(Into::into).collect(),
+            header_extensions: value
+                .header_extensions
+                .into_iter()
+                .map(Into::into)
+                .collect(),
+            rtcp: value.rtcp.into(),
+        }
+    }
+}
+
+impl From<sys_rp::ffi::RtpCodecParameters> for RtpCodecParameters {
+    fn from(value: sys_rp::ffi::RtpCodecParameters) -> Self {
+        Self {
+            mime_type: value.mime_type,
+            payload_type: value.payload_type as u8,
+            clock_rate: value.has_clock_rate.then_some(value.clock_rate as u64),
+            channels: value.has_num_channels.then_some(value.num_channels as u16),
+        }
+    }
+}
+
+impl From<sys_rp::ffi::RtcpParameters> for RtcpParameters {
+    fn from(value: sys_rp::ffi::RtcpParameters) -> Self {
+        Self {
+            cname: value.cname,
+            reduced_size: value.reduced_size,
+        }
+    }
+}
+
+impl From<sys_rp::ffi::RtpEncodingParameters> for RtpEncodingParameters {
+    fn from(value: sys_rp::ffi::RtpEncodingParameters) -> Self {
+        Self {
+            active: value.active,
+            max_bitrate: value
+                .has_max_bitrate_bps
+                .then_some(value.max_bitrate_bps as u64),
+            max_framerate: value.has_max_framerate.then_some(value.max_framerate),
+            priority: value.network_priority.into(),
+            rid: value.rid,
+            scale_resolution_down_by: value
+                .has_scale_resolution_down_by
+                .then_some(value.scale_resolution_down_by),
+        }
+    }
+}
+
+impl From<sys_rp::ffi::RtpCodecCapability> for RtpCodecCapability {
+    fn from(value: sys_rp::ffi::RtpCodecCapability) -> Self {
+        Self {
+            channels: value.has_num_channels.then_some(value.num_channels as u16),
+            mime_type: value.mime_type,
+            clock_rate: value.has_clock_rate.then_some(value.clock_rate as u64),
+            sdp_fmtp_line: {
+                let parameters: Vec<String> = value
+                    .parameters
+                    .into_iter()
+                    .map(|key_value| {
+                        if !key_value.key.is_empty() {
+                            format!("{}={}", key_value.key, key_value.value)
+                        } else {
+                            key_value.value
+                        }
+                    })
+                    .collect();
+
+                if !parameters.is_empty() {
+                    Some(parameters.join(";"))
+                } else {
+                    None
+                }
+            },
+        }
+    }
+}
+
+impl From<sys_rp::ffi::RtpHeaderExtensionCapability> for RtpHeaderExtensionCapability {
+    fn from(value: sys_rp::ffi::RtpHeaderExtensionCapability) -> Self {
+        Self {
+            direction: value.direction.into(),
+            uri: value.uri,
+        }
+    }
+}
+
+impl From<sys_rp::ffi::RtpCapabilities> for RtpCapabilities {
+    fn from(value: sys_rp::ffi::RtpCapabilities) -> Self {
+        Self {
+            codecs: value.codecs.into_iter().map(Into::into).collect(),
+            header_extensions: value
+                .header_extensions
+                .into_iter()
+                .map(Into::into)
+                .collect(),
+        }
+    }
+}
+
+impl From<Priority> for sys_webrtc::ffi::Priority {
+    fn from(value: Priority) -> Self {
+        match value {
+            Priority::VeryLow => Self::VeryLow,
+            Priority::Low => Self::Low,
+            Priority::Medium => Self::Medium,
+            Priority::High => Self::High,
+        }
+    }
+}
+
+impl From<RtpHeaderExtensionParameters> for sys_rp::ffi::RtpExtension {
+    fn from(value: RtpHeaderExtensionParameters) -> Self {
+        Self {
+            uri: value.uri,
+            id: value.id,
+            encrypt: value.encrypted,
+        }
+    }
+}
+
+impl From<RtpParameters> for sys_rp::ffi::RtpParameters {
+    fn from(value: RtpParameters) -> Self {
+        Self {
+            codecs: value.codecs.into_iter().map(Into::into).collect(),
+            header_extensions: value
+                .header_extensions
+                .into_iter()
+                .map(Into::into)
+                .collect(),
+            encodings: Vec::new(),
+            rtcp: value.rtcp.into(),
+            transaction_id: "".to_string(),
+            mid: "".to_string(),
+            has_degradation_preference: false,
+            degradation_preference: sys_rp::ffi::DegradationPreference::Balanced,
+        }
+    }
+}
+
+impl From<RtpCodecParameters> for sys_rp::ffi::RtpCodecParameters {
+    fn from(value: RtpCodecParameters) -> Self {
+        Self {
+            payload_type: value.payload_type as i32,
+            mime_type: value.mime_type,
+            has_clock_rate: value.clock_rate.is_some(),
+            clock_rate: value.clock_rate.unwrap_or_default() as i32,
+            has_num_channels: value.channels.is_some(),
+            num_channels: value.channels.unwrap_or_default() as i32,
+            name: "".to_string(),
+            kind: sys_rp::ffi::MediaType::Audio,
+            has_max_ptime: false,
+            max_ptime: 0,
+            has_ptime: false,
+            ptime: 0,
+            rtcp_feedback: Vec::new(),
+            parameters: Vec::new(),
+        }
+    }
+}
+
+impl From<RtcpParameters> for sys_rp::ffi::RtcpParameters {
+    fn from(value: RtcpParameters) -> Self {
+        Self {
+            cname: value.cname,
+            reduced_size: value.reduced_size,
+            has_ssrc: false,
+            ssrc: 0,
+            mux: false,
+        }
+    }
+}
+
+impl From<RtpEncodingParameters> for sys_rp::ffi::RtpEncodingParameters {
+    fn from(value: RtpEncodingParameters) -> Self {
+        Self {
+            active: value.active,
+            has_max_bitrate_bps: value.max_bitrate.is_some(),
+            max_bitrate_bps: value.max_bitrate.unwrap_or_default() as i32,
+            has_max_framerate: value.max_framerate.is_some(),
+            max_framerate: value.max_framerate.unwrap_or_default(),
+            network_priority: value.priority.into(),
+            rid: value.rid,
+            has_scale_resolution_down_by: value.scale_resolution_down_by.is_some(),
+            scale_resolution_down_by: value.scale_resolution_down_by.unwrap_or_default(),
+            adaptive_ptime: false,
+            bitrate_priority: sys_rp::DEFAULT_BITRATE_PRIORITY,
+            has_min_bitrate_bps: false,
+            min_bitrate_bps: 0,
+            has_num_temporal_layers: false,
+            num_temporal_layers: 0,
+            has_scalability_mode: false,
+            scalability_mode: "".to_string(),
+            has_ssrc: false,
+            ssrc: 0,
+        }
+    }
+}
+
+impl From<RtpCodecCapability> for sys_rp::ffi::RtpCodecCapability {
+    fn from(value: RtpCodecCapability) -> Self {
+        let mime_type: Vec<&str> = value.mime_type.split('/').collect();
+        let kind = match mime_type[0] {
+            "audio" => sys_webrtc::ffi::MediaType::Audio,
+            "video" => sys_webrtc::ffi::MediaType::Video,
+            _ => panic!("invalid media type"),
+        };
+        let name = mime_type[1].to_string();
+
+        Self {
+            name,
+            kind,
+            has_clock_rate: value.clock_rate.is_some(),
+            clock_rate: value.clock_rate.unwrap_or_default() as i32,
+            has_num_channels: value.channels.is_some(),
+            num_channels: value.channels.unwrap_or_default() as i32,
+            parameters: {
+                value
+                    .sdp_fmtp_line
+                    .map(|sdp_fmtp_line| {
+                        sdp_fmtp_line
+                            .split(';')
+                            .map(|v| {
+                                let key_value: Vec<&str> = v.split('=').collect();
+                                if key_value.len() == 2 {
+                                    sys_rp::ffi::StringKeyValue {
+                                        key: key_value[0].to_string(),
+                                        value: key_value[1].to_string(),
+                                    }
+                                } else {
+                                    sys_rp::ffi::StringKeyValue {
+                                        key: "".to_string(),
+                                        value: key_value[0].to_string(),
+                                    }
+                                }
+                            })
+                            .collect()
+                    })
+                    .unwrap_or_default()
+            },
+            // Ignore
+            mime_type: String::default(), // !!
+            has_preferred_payload_type: false,
+            preferred_payload_type: 0,
+            has_max_ptime: false,
+            max_ptime: 0,
+            has_ptime: false,
+            ptime: 0,
+            rtcp_feedback: Vec::default(),
+            options: Vec::default(),
+            max_temporal_layer_extensions: 0,
+            max_spatial_layer_extensions: 0,
+            svc_multi_stream_support: false,
+        }
+    }
+}

livekit-webrtc/src/native/rtp_receiver.rs

@@ -0,0 +1,24 @@
+use super::media_stream::new_media_stream_track;
+use crate::{media_stream::MediaStreamTrack, rtp_parameters::RtpParameters};
+use cxx::SharedPtr;
+use webrtc_sys::rtp_receiver as sys_rr;
+
+#[derive(Clone)]
+pub struct RtpReceiver {
+    pub(crate) sys_handle: SharedPtr<sys_rr::ffi::RtpReceiver>,
+}
+
+impl RtpReceiver {
+    pub fn track(&self) -> Option<MediaStreamTrack> {
+        let track_handle = self.sys_handle.track();
+        if track_handle.is_null() {
+            return None;
+        }
+
+        Some(new_media_stream_track(track_handle))
+    }
+
+    pub fn parameters(&self) -> RtpParameters {
+        self.sys_handle.get_parameters().into()
+    }
+}

livekit-webrtc/src/native/rtp_sender.rs

@@ -0,0 +1,46 @@
+use super::media_stream::new_media_stream_track;
+use crate::{
+    media_stream::MediaStreamTrack, rtp_parameters::RtpParameters, RtcError, RtcErrorType,
+};
+use cxx::SharedPtr;
+use webrtc_sys::{rtc_error::ffi::RTCError, rtp_sender as sys_rs};
+
+#[derive(Clone)]
+pub struct RtpSender {
+    pub(crate) sys_handle: SharedPtr<sys_rs::ffi::RtpSender>,
+}
+
+impl RtpSender {
+    pub fn track(&self) -> Option<MediaStreamTrack> {
+        let track_handle = self.sys_handle.track();
+        if track_handle.is_null() {
+            return None;
+        }
+
+        Some(new_media_stream_track(track_handle))
+    }
+
+    pub fn set_track(&self, track: Option<MediaStreamTrack>) -> Result<(), RtcError> {
+        if !self
+            .sys_handle
+            .set_track(track.map_or(SharedPtr::null(), |t| t.sys_handle()))
+        {
+            return Err(RtcError {
+                error_type: RtcErrorType::InvalidState,
+                message: "Failed to set track".to_string(),
+            });
+        }
+
+        Ok(())
+    }
+
+    pub fn parameters(&self) -> RtpParameters {
+        self.sys_handle.get_parameters().into()
+    }
+
+    pub fn set_parameters(&self, parameters: RtpParameters) -> Result<(), RtcError> {
+        self.sys_handle
+            .set_parameters(parameters.into())
+            .map_err(|e| unsafe { RTCError::from(e.what()).into() })
+    }
+}

livekit-webrtc/src/native/rtp_transceiver.rs

@@ -0,0 +1,94 @@
+use crate::imp::rtp_receiver::RtpReceiver;
+use crate::imp::rtp_sender::RtpSender;
+use crate::rtp_parameters::RtpCodecCapability;
+use crate::rtp_receiver;
+use crate::rtp_sender;
+use crate::rtp_transceiver::RtpTransceiverDirection;
+use crate::rtp_transceiver::RtpTransceiverInit;
+use crate::MediaType;
+use crate::RtcError;
+use cxx::SharedPtr;
+use webrtc_sys::rtc_error as sys_err;
+use webrtc_sys::rtp_transceiver as sys_rt;
+use webrtc_sys::webrtc as sys_webrtc;
+
+impl From<sys_webrtc::ffi::RtpTransceiverDirection> for RtpTransceiverDirection {
+    fn from(value: sys_webrtc::ffi::RtpTransceiverDirection) -> Self {
+        match value {
+            sys_webrtc::ffi::RtpTransceiverDirection::SendRecv => Self::SendRecv,
+            sys_webrtc::ffi::RtpTransceiverDirection::SendOnly => Self::SendOnly,
+            sys_webrtc::ffi::RtpTransceiverDirection::RecvOnly => Self::RecvOnly,
+            sys_webrtc::ffi::RtpTransceiverDirection::Inactive => Self::Inactive,
+            _ => panic!("unknown RtpTransceiverDirection"),
+        }
+    }
+}
+
+impl From<RtpTransceiverDirection> for sys_webrtc::ffi::RtpTransceiverDirection {
+    fn from(value: RtpTransceiverDirection) -> Self {
+        match value {
+            RtpTransceiverDirection::SendRecv => Self::SendRecv,
+            RtpTransceiverDirection::SendOnly => Self::SendOnly,
+            RtpTransceiverDirection::RecvOnly => Self::RecvOnly,
+            RtpTransceiverDirection::Inactive => Self::Inactive,
+            _ => panic!("unknown RtpTransceiverDirection"),
+        }
+    }
+}
+
+impl From<RtpTransceiverInit> for sys_rt::ffi::RtpTransceiverInit {
+    fn from(value: RtpTransceiverInit) -> Self {
+        Self {
+            direction: value.direction.into(),
+            stream_ids: value.stream_ids,
+            send_encodings: value.send_encodings.into_iter().map(Into::into).collect(),
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct RtpTransceiver {
+    pub(crate) sys_handle: SharedPtr<sys_rt::ffi::RtpTransceiver>,
+}
+
+impl RtpTransceiver {
+    pub fn mid(&self) -> Option<String> {
+        self.sys_handle.mid().ok()
+    }
+
+    pub fn current_direction(&self) -> Option<RtpTransceiverDirection> {
+        self.sys_handle.current_direction().ok().map(Into::into)
+    }
+
+    pub fn direction(&self) -> RtpTransceiverDirection {
+        self.sys_handle.direction().into()
+    }
+
+    pub fn sender(&self) -> rtp_sender::RtpSender {
+        rtp_sender::RtpSender {
+            handle: RtpSender {
+                sys_handle: self.sys_handle.sender(),
+            },
+        }
+    }
+
+    pub fn receiver(&self) -> rtp_receiver::RtpReceiver {
+        rtp_receiver::RtpReceiver {
+            handle: RtpReceiver {
+                sys_handle: self.sys_handle.receiver(),
+            },
+        }
+    }
+
+    pub fn set_codec_preferences(&self, codecs: Vec<RtpCodecCapability>) -> Result<(), RtcError> {
+        self.sys_handle
+            .set_codec_preferences(codecs.into_iter().map(Into::into).collect())
+            .map_err(|e| unsafe { sys_err::ffi::RTCError::from(e.what()).into() })
+    }
+
+    pub fn stop(&self) -> Result<(), RtcError> {
+        self.sys_handle
+            .stop_standard()
+            .map_err(|e| unsafe { sys_err::ffi::RTCError::from(e.what()).into() })
+    }
+}

livekit-webrtc/src/native/session_description.rs

@@ -0,0 +1,72 @@
+use crate::session_description::{self, SdpParseError, SdpType};
+use cxx::UniquePtr;
+use webrtc_sys::jsep as sys_jsep;
+
+impl From<sys_jsep::ffi::SdpType> for SdpType {
+    fn from(sdp_type: sys_jsep::ffi::SdpType) -> Self {
+        match sdp_type {
+            sys_jsep::ffi::SdpType::Offer => SdpType::Offer,
+            sys_jsep::ffi::SdpType::PrAnswer => SdpType::PrAnswer,
+            sys_jsep::ffi::SdpType::Answer => SdpType::Answer,
+            sys_jsep::ffi::SdpType::Rollback => SdpType::Rollback,
+            _ => panic!("unknown SdpType"),
+        }
+    }
+}
+
+impl From<SdpType> for sys_jsep::ffi::SdpType {
+    fn from(sdp_type: SdpType) -> Self {
+        match sdp_type {
+            SdpType::Offer => sys_jsep::ffi::SdpType::Offer,
+            SdpType::PrAnswer => sys_jsep::ffi::SdpType::PrAnswer,
+            SdpType::Answer => sys_jsep::ffi::SdpType::Answer,
+            SdpType::Rollback => sys_jsep::ffi::SdpType::Rollback,
+        }
+    }
+}
+
+impl From<sys_jsep::ffi::SdpParseError> for SdpParseError {
+    fn from(e: sys_jsep::ffi::SdpParseError) -> Self {
+        Self {
+            line: e.line,
+            description: e.description,
+        }
+    }
+}
+
+pub struct SessionDescription {
+    pub(crate) sys_handle: UniquePtr<sys_jsep::ffi::SessionDescription>,
+}
+
+impl SessionDescription {
+    pub fn parse(
+        sdp: &str,
+        sdp_type: SdpType,
+    ) -> Result<session_description::SessionDescription, SdpParseError> {
+        let res = sys_jsep::ffi::create_session_description(sdp_type.into(), sdp.to_owned());
+        match res {
+            Ok(sys_handle) => Ok(session_description::SessionDescription {
+                handle: SessionDescription { sys_handle },
+            }),
+            Err(e) => Err(unsafe { sys_jsep::ffi::SdpParseError::from(e.what()).into() }),
+        }
+    }
+
+    pub fn sdp_type(&self) -> SdpType {
+        self.sys_handle.sdp_type().into()
+    }
+}
+
+impl ToString for SessionDescription {
+    fn to_string(&self) -> String {
+        self.sys_handle.stringify()
+    }
+}
+
+impl Clone for SessionDescription {
+    fn clone(&self) -> Self {
+        SessionDescription {
+            sys_handle: self.sys_handle.clone(),
+        }
+    }
+}

livekit-webrtc/src/native/video_frame.rs

@@ -0,0 +1,764 @@
+use super::yuv_helper::{self, ConvertError};
+use crate::video_frame::VideoRotation;
+use crate::video_frame::{self as vf, VideoFormatType};
+use cxx::UniquePtr;
+use std::slice;
+use webrtc_sys::video_frame as vf_sys;
+use webrtc_sys::video_frame_buffer as vfb_sys;
+
+/// We don't use vf::VideoFrameBuffer trait for the types inside this module to avoid confusion
+/// because irectly using platform specific types is not valid (e.g user callback)
+/// All the types inside this module are only used internally. For public types, see the top level video_frame.rs
+
+pub fn new_video_frame_buffer(
+    mut sys_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>,
+) -> Box<dyn vf::VideoFrameBuffer + Send + Sync> {
+    unsafe {
+        match sys_handle.buffer_type().into() {
+            vfb_sys::ffi::VideoFrameBufferType::Native => Box::new(vf::native::NativeBuffer {
+                handle: NativeBuffer { sys_handle },
+            }),
+            vfb_sys::ffi::VideoFrameBufferType::I420 => Box::new(vf::I420Buffer {
+                handle: I420Buffer {
+                    sys_handle: sys_handle.pin_mut().get_i420(),
+                },
+            }),
+            vfb_sys::ffi::VideoFrameBufferType::I420A => Box::new(vf::I420ABuffer {
+                handle: I420ABuffer {
+                    sys_handle: sys_handle.pin_mut().get_i420a(),
+                },
+            }),
+            vfb_sys::ffi::VideoFrameBufferType::I422 => Box::new(vf::I422Buffer {
+                handle: I422Buffer {
+                    sys_handle: sys_handle.pin_mut().get_i422(),
+                },
+            }),
+            vfb_sys::ffi::VideoFrameBufferType::I444 => Box::new(vf::I444Buffer {
+                handle: I444Buffer {
+                    sys_handle: sys_handle.pin_mut().get_i444(),
+                },
+            }),
+            vfb_sys::ffi::VideoFrameBufferType::I010 => Box::new(vf::I010Buffer {
+                handle: I010Buffer {
+                    sys_handle: sys_handle.pin_mut().get_i010(),
+                },
+            }),
+            vfb_sys::ffi::VideoFrameBufferType::NV12 => Box::new(vf::NV12Buffer {
+                handle: NV12Buffer {
+                    sys_handle: sys_handle.pin_mut().get_nv12(),
+                },
+            }),
+            _ => unreachable!(),
+        }
+    }
+}
+
+impl From<vf_sys::ffi::VideoRotation> for VideoRotation {
+    fn from(rotation: vf_sys::ffi::VideoRotation) -> Self {
+        match rotation {
+            vf_sys::ffi::VideoRotation::VideoRotation0 => Self::VideoRotation0,
+            vf_sys::ffi::VideoRotation::VideoRotation90 => Self::VideoRotation90,
+            vf_sys::ffi::VideoRotation::VideoRotation180 => Self::VideoRotation180,
+            vf_sys::ffi::VideoRotation::VideoRotation270 => Self::VideoRotation270,
+            _ => panic!("invalid VideoRotation"),
+        }
+    }
+}
+
+impl From<VideoRotation> for vf_sys::ffi::VideoRotation {
+    fn from(rotation: VideoRotation) -> Self {
+        match rotation {
+            VideoRotation::VideoRotation0 => Self::VideoRotation0,
+            VideoRotation::VideoRotation90 => Self::VideoRotation90,
+            VideoRotation::VideoRotation180 => Self::VideoRotation180,
+            VideoRotation::VideoRotation270 => Self::VideoRotation270,
+        }
+    }
+}
+
+macro_rules! recursive_cast {
+    ($ptr:expr $(, $fnc:ident)*) => {
+        {
+            let ptr = $ptr;
+            $(
+                let ptr = vfb_sys::ffi::$fnc(ptr);
+            )*
+            ptr
+        }
+    };
+}
+
+pub struct NativeBuffer {
+    sys_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>,
+}
+
+pub struct I420Buffer {
+    sys_handle: UniquePtr<vfb_sys::ffi::I420Buffer>,
+}
+
+pub struct I420ABuffer {
+    sys_handle: UniquePtr<vfb_sys::ffi::I420ABuffer>,
+}
+
+pub struct I422Buffer {
+    sys_handle: UniquePtr<vfb_sys::ffi::I422Buffer>,
+}
+
+pub struct I444Buffer {
+    sys_handle: UniquePtr<vfb_sys::ffi::I444Buffer>,
+}
+
+pub struct I010Buffer {
+    sys_handle: UniquePtr<vfb_sys::ffi::I010Buffer>,
+}
+
+pub struct NV12Buffer {
+    sys_handle: UniquePtr<vfb_sys::ffi::NV12Buffer>,
+}
+
+macro_rules! impl_to_argb {
+    (I420Buffer [$($variant:ident: $fnc:ident),+], $format:ident, $self:ident, $dst:ident, $dst_stride:ident, $dst_width:ident, $dst_height:ident) => {
+        match $format {
+        $(
+            VideoFormatType::$variant => {
+                let (data_y, data_u, data_v) = $self.data();
+                yuv_helper::$fnc(
+                    data_y,
+                    $self.stride_y(),
+                    data_u,
+                    $self.stride_u(),
+                    data_v,
+                    $self.stride_v(),
+                    $dst,
+                    $dst_stride,
+                    $dst_width,
+                    $dst_height,
+                )
+            }
+        )+
+        }
+    };
+    (I420ABuffer) => {
+        todo!();
+    }
+}
+
+#[allow(unused_unsafe)]
+impl NativeBuffer {
+    pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
+        &*self.sys_handle
+    }
+
+    pub fn width(&self) -> i32 {
+        self.sys_handle.width()
+    }
+
+    pub fn height(&self) -> i32 {
+        self.sys_handle.height()
+    }
+
+    pub fn to_i420(&self) -> I420Buffer {
+        I420Buffer {
+            sys_handle: unsafe { self.sys_handle.to_i420() },
+        }
+    }
+
+    pub fn to_argb(
+        &self,
+        format: VideoFormatType,
+        dst: &mut [u8],
+        dst_stride: i32,
+        dst_width: i32,
+        dst_height: i32,
+    ) -> Result<(), ConvertError> {
+        self.to_i420()
+            .to_argb(format, dst, dst_stride, dst_width, dst_height)
+    }
+}
+
+impl I420Buffer {
+    pub fn new(width: u32, height: u32) -> vf::I420Buffer {
+        vf::I420Buffer {
+            handle: I420Buffer {
+                sys_handle: vfb_sys::ffi::new_i420_buffer(
+                    width.try_into().unwrap(),
+                    height.try_into().unwrap(),
+                ),
+            },
+        }
+    }
+
+    pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
+        unsafe { &*recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb) }
+    }
+
+    pub fn width(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
+            (*ptr).width()
+        }
+    }
+
+    pub fn height(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
+            (*ptr).height()
+        }
+    }
+
+    pub fn chroma_width(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
+            (*ptr).chroma_width()
+        }
+    }
+
+    pub fn chroma_height(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
+            (*ptr).chroma_height()
+        }
+    }
+
+    pub fn stride_y(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
+            (*ptr).stride_y()
+        }
+    }
+
+    pub fn stride_u(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
+            (*ptr).stride_u()
+        }
+    }
+
+    pub fn stride_v(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
+            (*ptr).stride_v()
+        }
+    }
+
+    pub fn to_i420(&self) -> I420Buffer {
+        I420Buffer {
+            sys_handle: unsafe {
+                // We make a copy of the buffer because internally, when calling ToI420()
+                // if the buffer is of type I420, libwebrtc will reuse the same underlying pointer
+                // for the new created type
+                let copy = vfb_sys::ffi::copy_i420_buffer(&self.sys_handle);
+                let ptr = recursive_cast!(&*copy, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
+                (*ptr).to_i420()
+            },
+        }
+    }
+
+    pub fn to_argb(
+        &self,
+        format: VideoFormatType,
+        dst: &mut [u8],
+        dst_stride: i32,
+        dst_width: i32,
+        dst_height: i32,
+    ) -> Result<(), ConvertError> {
+        impl_to_argb!(
+            I420Buffer
+            [
+                ARGB: i420_to_argb,
+                BGRA: i420_to_bgra,
+                ABGR: i420_to_abgr,
+                RGBA: i420_to_rgba
+            ],
+            format, self, dst, dst_stride, dst_width, dst_height
+        )
+    }
+
+    pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8);
+            let chroma_height = (self.height() + 1) / 2;
+            (
+                slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
+                slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * chroma_height) as usize),
+                slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * chroma_height) as usize),
+            )
+        }
+    }
+}
+
+impl I420ABuffer {
+    pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
+        unsafe { &*recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb) }
+    }
+
+    pub fn width(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
+            (*ptr).width()
+        }
+    }
+
+    pub fn height(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
+            (*ptr).height()
+        }
+    }
+
+    pub fn chroma_width(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
+            (*ptr).chroma_width()
+        }
+    }
+
+    pub fn chroma_height(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
+            (*ptr).chroma_height()
+        }
+    }
+
+    pub fn stride_y(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
+            (*ptr).stride_y()
+        }
+    }
+
+    pub fn stride_u(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
+            (*ptr).stride_u()
+        }
+    }
+
+    pub fn stride_v(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
+            (*ptr).stride_v()
+        }
+    }
+
+    pub fn stride_a(&self) -> i32 {
+        self.sys_handle.stride_a()
+    }
+
+    pub fn to_i420(&self) -> I420Buffer {
+        I420Buffer {
+            sys_handle: unsafe {
+                let ptr =
+                    recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
+                (*ptr).to_i420()
+            },
+        }
+    }
+
+    pub fn to_argb(
+        &self,
+        format: VideoFormatType,
+        dst: &mut [u8],
+        dst_stride: i32,
+        dst_width: i32,
+        dst_height: i32,
+    ) -> Result<(), ConvertError> {
+        self.to_i420()
+            .to_argb(format, dst, dst_stride, dst_width, dst_height)
+    }
+
+    pub fn data(&self) -> (&[u8], &[u8], &[u8], Option<&[u8]>) {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8);
+            let chroma_height = (self.height() + 1) / 2;
+            let data_a = self.sys_handle.data_a();
+            let has_data_a = !data_a.is_null();
+            (
+                slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
+                slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * chroma_height) as usize),
+                slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * chroma_height) as usize),
+                has_data_a.then_some(slice::from_raw_parts(
+                    data_a,
+                    (self.stride_a() * self.height()) as usize,
+                )),
+            )
+        }
+    }
+}
+
+impl I422Buffer {
+    pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
+        unsafe { &*recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb) }
+    }
+
+    pub fn width(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
+            (*ptr).width()
+        }
+    }
+
+    pub fn height(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
+            (*ptr).height()
+        }
+    }
+
+    pub fn chroma_width(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
+            (*ptr).chroma_width()
+        }
+    }
+
+    pub fn chroma_height(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
+            (*ptr).chroma_height()
+        }
+    }
+
+    pub fn stride_y(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
+            (*ptr).stride_y()
+        }
+    }
+
+    pub fn stride_u(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
+            (*ptr).stride_u()
+        }
+    }
+
+    pub fn stride_v(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
+            (*ptr).stride_v()
+        }
+    }
+
+    pub fn to_i420(&self) -> I420Buffer {
+        I420Buffer {
+            sys_handle: unsafe {
+                let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
+                (*ptr).to_i420()
+            },
+        }
+    }
+
+    pub fn to_argb(
+        &self,
+        format: VideoFormatType,
+        dst: &mut [u8],
+        dst_stride: i32,
+        dst_width: i32,
+        dst_height: i32,
+    ) -> Result<(), ConvertError> {
+        self.to_i420()
+            .to_argb(format, dst, dst_stride, dst_width, dst_height)
+    }
+
+    pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8);
+            (
+                slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
+                slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * self.height()) as usize),
+                slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * self.height()) as usize),
+            )
+        }
+    }
+}
+impl I444Buffer {
+    pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
+        unsafe { &*recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb) }
+    }
+
+    pub fn width(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
+            (*ptr).width()
+        }
+    }
+
+    pub fn height(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
+            (*ptr).height()
+        }
+    }
+
+    pub fn chroma_width(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
+            (*ptr).chroma_width()
+        }
+    }
+
+    pub fn chroma_height(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
+            (*ptr).chroma_height()
+        }
+    }
+
+    pub fn stride_y(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
+            (*ptr).stride_y()
+        }
+    }
+
+    pub fn stride_u(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
+            (*ptr).stride_u()
+        }
+    }
+
+    pub fn stride_v(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
+            (*ptr).stride_v()
+        }
+    }
+
+    pub fn to_i420(&self) -> I420Buffer {
+        I420Buffer {
+            sys_handle: unsafe {
+                let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
+                (*ptr).to_i420()
+            },
+        }
+    }
+
+    pub fn to_argb(
+        &self,
+        format: VideoFormatType,
+        dst: &mut [u8],
+        dst_stride: i32,
+        dst_width: i32,
+        dst_height: i32,
+    ) -> Result<(), ConvertError> {
+        self.to_i420()
+            .to_argb(format, dst, dst_stride, dst_width, dst_height)
+    }
+
+    pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8);
+            (
+                slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
+                slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * self.height()) as usize),
+                slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * self.height()) as usize),
+            )
+        }
+    }
+}
+
+impl I010Buffer {
+    pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
+        unsafe { &*recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb) }
+    }
+
+    pub fn width(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb);
+            (*ptr).width()
+        }
+    }
+
+    pub fn height(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb);
+            (*ptr).height()
+        }
+    }
+
+    pub fn chroma_width(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
+            (*ptr).chroma_width()
+        }
+    }
+
+    pub fn chroma_height(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
+            (*ptr).chroma_height()
+        }
+    }
+
+    pub fn stride_y(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
+            (*ptr).stride_y()
+        }
+    }
+
+    pub fn stride_u(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
+            (*ptr).stride_u()
+        }
+    }
+
+    pub fn stride_v(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
+            (*ptr).stride_v()
+        }
+    }
+
+    pub fn to_i420(&self) -> I420Buffer {
+        I420Buffer {
+            sys_handle: unsafe {
+                let ptr =
+                    recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb);
+                (*ptr).to_i420()
+            },
+        }
+    }
+
+    pub fn to_argb(
+        &self,
+        format: VideoFormatType,
+        dst: &mut [u8],
+        dst_stride: i32,
+        dst_width: i32,
+        dst_height: i32,
+    ) -> Result<(), ConvertError> {
+        self.to_i420()
+            .to_argb(format, dst, dst_stride, dst_width, dst_height)
+    }
+
+    pub fn data(&self) -> (&[u16], &[u16], &[u16]) {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b);
+            let chroma_height = (self.height() + 1) / 2;
+            (
+                slice::from_raw_parts(
+                    (*ptr).data_y(),
+                    (self.stride_y() * self.height()) as usize / 2,
+                ),
+                slice::from_raw_parts(
+                    (*ptr).data_u(),
+                    (self.stride_u() * chroma_height) as usize / 2,
+                ),
+                slice::from_raw_parts(
+                    (*ptr).data_v(),
+                    (self.stride_v() * chroma_height) as usize / 2,
+                ),
+            )
+        }
+    }
+}
+
+impl NV12Buffer {
+    pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
+        unsafe {
+            &*recursive_cast!(
+                &*self.sys_handle,
+                nv12_to_biyuv8,
+                biyuv8_to_biyuv,
+                biyuv_to_vfb
+            )
+        }
+    }
+
+    pub fn width(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(
+                &*self.sys_handle,
+                nv12_to_biyuv8,
+                biyuv8_to_biyuv,
+                biyuv_to_vfb
+            );
+            (*ptr).width()
+        }
+    }
+
+    pub fn height(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(
+                &*self.sys_handle,
+                nv12_to_biyuv8,
+                biyuv8_to_biyuv,
+                biyuv_to_vfb
+            );
+            (*ptr).height()
+        }
+    }
+
+    pub fn chroma_width(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv);
+            (*ptr).chroma_width()
+        }
+    }
+
+    pub fn chroma_height(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv);
+            (*ptr).chroma_height()
+        }
+    }
+
+    pub fn stride_y(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv);
+            (*ptr).stride_y()
+        }
+    }
+
+    pub fn stride_uv(&self) -> i32 {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv);
+            (*ptr).stride_uv()
+        }
+    }
+
+    pub fn to_i420(&self) -> I420Buffer {
+        I420Buffer {
+            sys_handle: unsafe {
+                let ptr = recursive_cast!(
+                    &*self.sys_handle,
+                    nv12_to_biyuv8,
+                    biyuv8_to_biyuv,
+                    biyuv_to_vfb
+                );
+                (*ptr).to_i420()
+            },
+        }
+    }
+
+    pub fn to_argb(
+        &self,
+        format: VideoFormatType,
+        dst: &mut [u8],
+        dst_stride: i32,
+        dst_width: i32,
+        dst_height: i32,
+    ) -> Result<(), ConvertError> {
+        self.to_i420()
+            .to_argb(format, dst, dst_stride, dst_width, dst_height)
+    }
+
+    pub fn data(&self) -> (&[u8], &[u8]) {
+        unsafe {
+            let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8);
+            let chroma_height = (self.height() + 1) / 2;
+
+            (
+                slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
+                slice::from_raw_parts(
+                    (*ptr).data_uv(),
+                    (self.stride_uv() * chroma_height) as usize,
+                ),
+            )
+        }
+    }
+}

livekit-webrtc/src/native/video_source.rs

@@ -0,0 +1,34 @@
+use crate::video_frame::{VideoFrame, VideoFrameBuffer};
+use cxx::SharedPtr;
+use webrtc_sys::media_stream as ms_sys;
+use webrtc_sys::video_frame as vf_sys;
+
+#[derive(Clone)]
+pub struct NativeVideoSource {
+    sys_handle: SharedPtr<ms_sys::ffi::AdaptedVideoTrackSource>,
+}
+
+impl Default for NativeVideoSource {
+    fn default() -> Self {
+        Self {
+            sys_handle: ms_sys::ffi::new_adapted_video_track_source(),
+        }
+    }
+}
+
+impl NativeVideoSource {
+    pub fn sys_handle(&self) -> SharedPtr<ms_sys::ffi::AdaptedVideoTrackSource> {
+        self.sys_handle.clone()
+    }
+
+    pub fn capture_frame<T: VideoFrameBuffer>(&self, frame: &VideoFrame<T>) {
+        let mut builder = vf_sys::ffi::new_video_frame_builder();
+        builder.pin_mut().set_rotation(frame.rotation.into());
+        builder
+            .pin_mut()
+            .set_video_frame_buffer(frame.buffer.sys_handle());
+
+        let frame = builder.pin_mut().build();
+        self.sys_handle.on_captured_frame(&frame);
+    }
+}

livekit-webrtc/src/native/video_stream.rs

@@ -0,0 +1,84 @@
+use super::video_frame::new_video_frame_buffer;
+use crate::media_stream::RtcVideoTrack;
+use crate::video_frame::{BoxVideoFrame, VideoFrame};
+use cxx::UniquePtr;
+use futures::stream::Stream;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+use tokio::sync::mpsc;
+use webrtc_sys::media_stream as sys_ms;
+
+pub struct NativeVideoStream {
+    native_observer: UniquePtr<sys_ms::ffi::NativeVideoFrameSink>,
+    _observer: Box<VideoTrackObserver>,
+    video_track: RtcVideoTrack,
+    frame_rx: mpsc::UnboundedReceiver<BoxVideoFrame>,
+}
+
+impl NativeVideoStream {
+    pub fn new(video_track: RtcVideoTrack) -> Self {
+        let (frame_tx, frame_rx) = mpsc::unbounded_channel();
+        let mut observer = Box::new(VideoTrackObserver { frame_tx });
+        let mut native_observer = unsafe {
+            sys_ms::ffi::new_native_video_frame_sink(Box::new(sys_ms::VideoFrameSinkWrapper::new(
+                &mut *observer,
+            )))
+        };
+
+        unsafe {
+            sys_ms::ffi::media_to_video(video_track.sys_handle())
+                .add_sink(native_observer.pin_mut());
+        }
+
+        Self {
+            native_observer,
+            _observer: observer,
+            video_track,
+            frame_rx,
+        }
+    }
+
+    pub fn track(&self) -> RtcVideoTrack {
+        self.video_track.clone()
+    }
+
+    pub fn close(&mut self) {
+        self.frame_rx.close();
+        unsafe {
+            sys_ms::ffi::media_to_video(self.video_track.sys_handle())
+                .remove_sink(self.native_observer.pin_mut());
+        }
+    }
+}
+
+impl Drop for NativeVideoStream {
+    fn drop(&mut self) {
+        self.close();
+    }
+}
+
+impl Stream for NativeVideoStream {
+    type Item = BoxVideoFrame;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
+        self.frame_rx.poll_recv(cx)
+    }
+}
+
+struct VideoTrackObserver {
+    frame_tx: mpsc::UnboundedSender<BoxVideoFrame>,
+}
+
+impl sys_ms::VideoFrameSink for VideoTrackObserver {
+    fn on_frame(&self, frame: UniquePtr<webrtc_sys::video_frame::ffi::VideoFrame>) {
+        let _ = self.frame_tx.send(VideoFrame {
+            rotation: frame.rotation().into(),
+            timestamp: frame.timestamp_us(),
+            buffer: new_video_frame_buffer(unsafe { frame.video_frame_buffer() }),
+        });
+    }
+
+    fn on_discarded_frame(&self) {}
+
+    fn on_constraints_changed(&self, _constraints: sys_ms::ffi::VideoTrackSourceConstraints) {}
+}

livekit-webrtc/src/native/yuv_helper.rs

@@ -0,0 +1,183 @@
+use thiserror::Error;
+use webrtc_sys::yuv_helper as yuv_sys;
+
+#[derive(Error, Debug)]
+pub enum ConvertError {
+    #[error("conversion failed: {0}")]
+    Convert(&'static str),
+}
+
+#[inline]
+fn argb_assert_safety(
+    src: &[u8],
+    src_stride: i32,
+    _width: i32,
+    height: i32,
+) -> Result<(), ConvertError> {
+    let min = (src_stride * height) as usize;
+
+    if src.len() < min {
+        return Err(ConvertError::Convert("dst isn't large enough"));
+    }
+
+    Ok(())
+}
+
+#[inline]
+fn i420_assert_safety(
+    src_y: &[u8],
+    src_stride_y: i32,
+    src_u: &[u8],
+    src_stride_u: i32,
+    src_v: &[u8],
+    src_stride_v: i32,
+    _width: i32,
+    height: i32,
+) -> Result<(), ConvertError> {
+    let chroma_height = (height + 1) / 2;
+    let min_y = (src_stride_y * height) as usize;
+    let min_u = (src_stride_u * chroma_height) as usize;
+    let min_v = (src_stride_v * chroma_height) as usize;
+
+    if src_y.len() < min_y {
+        return Err(ConvertError::Convert("src_y isn't large enough"));
+    }
+
+    if src_u.len() < min_u {
+        return Err(ConvertError::Convert("src_u isn't large enough"));
+    }
+
+    if src_v.len() < min_v {
+        return Err(ConvertError::Convert("src_v isn't large enough"));
+    }
+
+    Ok(())
+}
+
+macro_rules! i420_to_x {
+    ($x:ident) => {
+        pub fn $x(
+            src_y: &[u8],
+            src_stride_y: i32,
+            src_u: &[u8],
+            src_stride_u: i32,
+            src_v: &[u8],
+            src_stride_v: i32,
+            dst: &mut [u8],
+            dst_stride: i32,
+            width: i32,
+            height: i32,
+        ) -> Result<(), ConvertError> {
+            argb_assert_safety(dst, dst_stride, width, height)?;
+            i420_assert_safety(
+                src_y,
+                src_stride_y,
+                src_u,
+                src_stride_u,
+                src_v,
+                src_stride_v,
+                width,
+                height,
+            )?;
+
+            unsafe {
+                yuv_sys::ffi::$x(
+                    src_y.as_ptr(),
+                    src_stride_y,
+                    src_u.as_ptr(),
+                    src_stride_u,
+                    src_v.as_ptr(),
+                    src_stride_v,
+                    dst.as_mut_ptr(),
+                    dst_stride,
+                    width,
+                    height,
+                )
+                .unwrap();
+            }
+
+            Ok(())
+        }
+    };
+}
+
+macro_rules! x_to_i420 {
+    ($x:ident) => {
+        pub fn $x(
+            src_argb: &[u8],
+            src_stride_argb: i32,
+            dst_y: &mut [u8],
+            dst_stride_y: i32,
+            dst_u: &mut [u8],
+            dst_stride_u: i32,
+            dst_v: &mut [u8],
+            dst_stride_v: i32,
+            width: i32,
+            height: i32,
+        ) -> Result<(), ConvertError> {
+            argb_assert_safety(src_argb, src_stride_argb, width, height)?;
+            i420_assert_safety(
+                dst_y,
+                dst_stride_y,
+                dst_u,
+                dst_stride_u,
+                dst_v,
+                dst_stride_v,
+                width,
+                height,
+            )?;
+
+            unsafe {
+                yuv_sys::ffi::$x(
+                    src_argb.as_ptr(),
+                    src_stride_argb,
+                    dst_y.as_mut_ptr(),
+                    dst_stride_y,
+                    dst_u.as_mut_ptr(),
+                    dst_stride_u,
+                    dst_v.as_mut_ptr(),
+                    dst_stride_v,
+                    width,
+                    height,
+                )
+                .unwrap();
+            }
+
+            Ok(())
+        }
+    };
+}
+
+pub fn argb_to_rgb24(
+    src_argb: &[u8],
+    src_stride_argb: i32,
+    dst_rgb24: &mut [u8],
+    dst_stride_rgb24: i32,
+    width: i32,
+    height: i32,
+) -> Result<(), ConvertError> {
+    argb_assert_safety(src_argb, src_stride_argb, width, height)?;
+    argb_assert_safety(dst_rgb24, dst_stride_rgb24, width, height)?;
+
+    unsafe {
+        yuv_sys::ffi::argb_to_rgb24(
+            src_argb.as_ptr(),
+            src_stride_argb,
+            dst_rgb24.as_mut_ptr(),
+            dst_stride_rgb24,
+            width,
+            height,
+        )
+        .unwrap();
+    }
+
+    Ok(())
+}
+
+x_to_i420!(argb_to_i420);
+x_to_i420!(abgr_to_i420);
+
+i420_to_x!(i420_to_argb);
+i420_to_x!(i420_to_bgra);
+i420_to_x!(i420_to_abgr);
+i420_to_x!(i420_to_rgba);

livekit-webrtc/src/peer_connection.rs

@@ -1,737 +1,240 @@
-use crate::prelude::*;
+use std::fmt::Debug;
-use cxx::{SharedPtr, UniquePtr};
-use log::trace;
-use std::fmt::{Debug, Formatter};
-use std::mem::ManuallyDrop;
-use std::sync::{Arc, Mutex};
-use tokio::sync::{mpsc, oneshot};
 
-use webrtc_sys::candidate as sys_ca;
+use crate::data_channel::{DataChannel, DataChannelInit};
-use webrtc_sys::data_channel as sys_dc;
+use crate::ice_candidate::IceCandidate;
-use webrtc_sys::jsep as sys_jsep;
+use crate::imp::peer_connection as imp_pc;
-use webrtc_sys::media_stream as sys_ms;
+use crate::media_stream::{MediaStream, MediaStreamTrack};
-use webrtc_sys::peer_connection as sys_pc;
+use crate::rtp_receiver::RtpReceiver;
-use webrtc_sys::rtp_receiver as sys_rr;
+use crate::rtp_sender::RtpSender;
-use webrtc_sys::rtp_sender as sys_rs;
+use crate::rtp_transceiver::{RtpTransceiver, RtpTransceiverInit};
-use webrtc_sys::rtp_transceiver as sys_rt;
+use crate::session_description::SessionDescription;
+use crate::{MediaType, RtcError};
 
-pub use webrtc_sys::peer_connection::ffi::IceConnectionState;
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-pub use webrtc_sys::peer_connection::ffi::IceGatheringState;
+pub enum PeerConnectionState {
-pub use webrtc_sys::peer_connection::ffi::PeerConnectionState;
+    New,
-pub use webrtc_sys::peer_connection::ffi::RTCOfferAnswerOptions;
+    Connecting,
-pub use webrtc_sys::peer_connection::ffi::SignalingState;
+    Connected,
-
+    Disconnected,
-pub struct PeerConnection {
+    Failed,
-    cxx_handle: UniquePtr<sys_pc::ffi::PeerConnection>,
+    Closed,
-    observer: Box<InternalObserver>,
-
-    // Keep alive for C++
-    #[allow(unused)]
-    native_observer: UniquePtr<sys_pc::ffi::NativePeerConnectionObserver>,
 }
 
-impl Debug for PeerConnection {
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
+pub enum IceConnectionState {
-        f.debug_struct("PeerConnection")
+    New,
-            .field("signaling_state", &self.signaling_state())
+    Checking,
-            .field("ice_connection_state", &self.ice_connection_state())
+    Connected,
-            .field("ice_gathering_state", &self.ice_gathering_state())
+    Completed,
-            .finish()
+    Failed,
-    }
+    Disconnected,
+    Closed,
+    Max,
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub enum IceGatheringState {
+    New,
+    Gathering,
+    Complete,
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub enum SignalingState {
+    Stable,
+    HaveLocalOffer,
+    HaveLocalPrAnswer,
+    HaveRemoteOffer,
+    HaveRemotePrAnswer,
+    Closed,
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct OfferOptions {
+    pub ice_restart: bool,
+    pub offer_to_receive_audio: bool,
+    pub offer_to_receive_video: bool,
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct AnswerOptions {}
+
+#[derive(Debug, Clone)]
+pub struct IceCandidateError {
+    pub address: String,
+    pub port: i32,
+    pub url: String,
+    pub error_code: i32,
+    pub error_text: String,
+}
+
+#[derive(Debug, Clone)]
+pub struct TrackEvent {
+    pub receiver: RtpReceiver,
+    pub streams: Vec<MediaStream>,
+    pub track: MediaStreamTrack,
+    pub transceiver: RtpTransceiver,
+}
+
+pub type OnConnectionChange = Box<dyn FnMut(PeerConnectionState) + Send + Sync>;
+pub type OnDataChannel = Box<dyn FnMut(DataChannel) + Send + Sync>;
+pub type OnIceCandidate = Box<dyn FnMut(IceCandidate) + Send + Sync>;
+pub type OnIceCandidateError = Box<dyn FnMut(IceCandidateError) + Send + Sync>;
+pub type OnIceConnectionChange = Box<dyn FnMut(IceConnectionState) + Send + Sync>;
+pub type OnIceGatheringChange = Box<dyn FnMut(IceGatheringState) + Send + Sync>;
+pub type OnNegotiationNeeded = Box<dyn FnMut(u32) + Send + Sync>;
+pub type OnSignalingChange = Box<dyn FnMut(SignalingState) + Send + Sync>;
+pub type OnTrack = Box<dyn FnMut(TrackEvent) + Send + Sync>;
+
+#[derive(Clone)]
+pub struct PeerConnection {
+    pub(crate) handle: imp_pc::PeerConnection,
 }
 
 impl PeerConnection {
-    pub(crate) fn new(
-        cxx_handle: UniquePtr<sys_pc::ffi::PeerConnection>,
-        observer: Box<InternalObserver>,
-        native_observer: UniquePtr<sys_pc::ffi::NativePeerConnectionObserver>,
-    ) -> Self {
-        Self {
-            cxx_handle,
-            observer,
-            native_observer,
-        }
-    }
-
-    fn create_sdp_observer() -> (
-        UniquePtr<sys_pc::ffi::NativeCreateSdpObserverHandle>,
-        mpsc::Receiver<Result<SessionDescription, RTCError>>,
-    ) {
-        let (tx, rx) = mpsc::channel(1);
-        let wrapper = sys_jsep::CreateSdpObserverWrapper {
-            on_success: ManuallyDrop::new(Box::new({
-                let tx = tx.clone();
-                move |session_description| {
-                    let _ = tx.blocking_send(Ok(SessionDescription::new(session_description)));
-                }
-            })),
-            on_failure: ManuallyDrop::new(Box::new(move |error| {
-                let _ = tx.blocking_send(Err(error));
-            })),
-        };
-
-        (
-            sys_jsep::ffi::create_native_create_sdp_observer(Box::new(wrapper)),
-            rx,
-        )
-    }
-
     pub async fn create_offer(
         &self,
-        options: RTCOfferAnswerOptions,
+        options: OfferOptions,
-    ) -> Result<SessionDescription, RTCError> {
+    ) -> Result<SessionDescription, RtcError> {
-        let (mut native_wrapper, mut rx) = Self::create_sdp_observer();
+        self.handle.create_offer(options).await
-
-        unsafe {
-            self.cxx_handle
-                .create_offer(native_wrapper.pin_mut(), options);
-        }
-
-        rx.recv().await.unwrap()
     }
 
     pub async fn create_answer(
         &self,
-        options: RTCOfferAnswerOptions,
+        options: AnswerOptions,
-    ) -> Result<SessionDescription, RTCError> {
+    ) -> Result<SessionDescription, RtcError> {
-        let (mut native_wrapper, mut rx) = Self::create_sdp_observer();
+        self.handle.create_answer(options).await
-
-        unsafe {
-            self.cxx_handle
-                .create_answer(native_wrapper.pin_mut(), options);
     }
 
-        rx.recv().await.unwrap()
+    pub async fn set_local_description(&self, desc: SessionDescription) -> Result<(), RtcError> {
+        self.handle.set_local_description(desc).await
     }
 
-    pub async fn set_local_description(&self, desc: SessionDescription) -> Result<(), RTCError> {
+    pub async fn set_remote_description(&self, desc: SessionDescription) -> Result<(), RtcError> {
-        let (tx, rx) = oneshot::channel();
+        self.handle.set_remote_description(desc).await
-        let wrapper =
-            sys_jsep::SetLocalSdpObserverWrapper(ManuallyDrop::new(Box::new(move |error| {
-                let _ = tx.send(if error.ok() { Ok(()) } else { Err(error) });
-            })));
-        let mut native_wrapper =
-            sys_jsep::ffi::create_native_set_local_sdp_observer(Box::new(wrapper));
-
-        unsafe {
-            self.cxx_handle
-                .set_local_description(desc.release(), native_wrapper.pin_mut());
     }
 
-        rx.await.unwrap()
+    pub async fn add_ice_candidate(&self, candidate: IceCandidate) -> Result<(), RtcError> {
-    }
+        self.handle.add_ice_candidate(candidate).await
-
-    pub async fn set_remote_description(&self, desc: SessionDescription) -> Result<(), RTCError> {
-        let (tx, rx) = oneshot::channel();
-        let wrapper =
-            sys_jsep::SetRemoteSdpObserverWrapper(ManuallyDrop::new(Box::new(move |error| {
-                let _ = tx.send(if error.ok() { Ok(()) } else { Err(error) });
-            })));
-        let mut native_wrapper =
-            sys_jsep::ffi::create_native_set_remote_sdp_observer(Box::new(wrapper));
-
-        unsafe {
-            self.cxx_handle
-                .set_remote_description(desc.release(), native_wrapper.pin_mut());
-        }
-
-        rx.await.unwrap()
-    }
-
-    pub fn add_track(
-        &self,
-        track: MediaStreamTrackHandle,
-        stream_ids: &Vec<String>,
-    ) -> Result<RtpSender, RTCError> {
-        let res = self.cxx_handle.add_track(track.cxx_handle(), stream_ids);
-        match res {
-            Ok(cxx_handle) => Ok(RtpSender::new(cxx_handle)),
-            Err(e) => unsafe { Err(RTCError::from(e.what())) },
-        }
-    }
-
-    pub fn remove_track(&self, sender: RtpSender) -> Result<(), RTCError> {
-        self.cxx_handle
-            .remove_track(sender.cxx_handle())
-            .map_err(|e| unsafe { RTCError::from(e.what()) })
-    }
-
-    pub fn add_transceiver(
-        &self,
-        track: MediaStreamTrackHandle,
-        init: RtpTransceiverInit,
-    ) -> Result<RtpTransceiver, RTCError> {
-        let res = self
-            .cxx_handle
-            .add_transceiver(track.cxx_handle(), init.into());
-
-        match res {
-            Ok(cxx_handle) => Ok(RtpTransceiver::new(cxx_handle)),
-            Err(e) => unsafe { Err(RTCError::from(e.what())) },
-        }
-    }
-
-    pub fn add_transceiver_for_media(
-        &self,
-        media_type: MediaType,
-        init: RtpTransceiverInit,
-    ) -> Result<RtpTransceiver, RTCError> {
-        let res = self
-            .cxx_handle
-            .add_transceiver_for_media(media_type, init.into());
-
-        match res {
-            Ok(cxx_handle) => Ok(RtpTransceiver::new(cxx_handle)),
-            Err(e) => unsafe { Err(RTCError::from(e.what())) },
-        }
-    }
-
-    pub fn senders(&self) -> Vec<RtpSender> {
-        self.cxx_handle
-            .get_senders()
-            .into_iter()
-            .map(|sender| RtpSender::new(sender.ptr))
-            .collect()
-    }
-
-    pub fn receivers(&self) -> Vec<RtpReceiver> {
-        self.cxx_handle
-            .get_receivers()
-            .into_iter()
-            .map(|receiver| RtpReceiver::new(receiver.ptr))
-            .collect()
-    }
-
-    pub fn transceivers(&self) -> Vec<RtpTransceiver> {
-        self.cxx_handle
-            .get_transceivers()
-            .into_iter()
-            .map(|transceiver| RtpTransceiver::new(transceiver.ptr))
-            .collect()
     }
 
     pub fn create_data_channel(
         &self,
         label: &str,
         init: DataChannelInit,
-    ) -> Result<DataChannel, RTCError> {
+    ) -> Result<DataChannel, RtcError> {
-        let native_init = sys_dc::ffi::create_data_channel_init(init.into());
+        self.handle.create_data_channel(label, init)
-        let res = self
-            .cxx_handle
-            .create_data_channel(label.to_string(), native_init);
-
-        match res {
-            Ok(cxx_handle) => Ok(DataChannel::new(cxx_handle)),
-            Err(e) => Err(unsafe { RTCError::from(e.what()) }),
-        }
     }
 
-    // TODO(theomonnom) Use IceCandidateInit instead of IceCandidate
+    pub fn add_track<T: AsRef<str>>(
-    pub async fn add_ice_candidate(&self, candidate: IceCandidate) -> Result<(), RTCError> {
+        &self,
-        let (tx, rx) = oneshot::channel();
+        track: MediaStreamTrack,
-        let observer =
+        streams_ids: &[T],
-            sys_pc::AddIceCandidateObserverWrapper(ManuallyDrop::new(Box::new(|error| {
+    ) -> Result<RtpSender, RtcError> {
-                let _ = tx.send(if error.ok() { Ok(()) } else { Err(error) });
+        self.handle.add_track(track, streams_ids)
-            })));
-
-        let mut native_observer =
-            sys_pc::ffi::create_native_add_ice_candidate_observer(Box::new(observer));
-        self.cxx_handle
-            .add_ice_candidate(candidate.release(), native_observer.pin_mut());
-
-        rx.await.unwrap()
     }
 
-    pub fn local_description(&self) -> Option<SessionDescription> {
+    pub fn remove_track(&self, sender: RtpSender) -> Result<(), RtcError> {
-        let local_description = self.cxx_handle.local_description();
+        self.handle.remove_track(sender)
-        if local_description.is_null() {
-            None
-        } else {
-            Some(SessionDescription::new(local_description))
-        }
     }
 
-    pub fn remote_description(&self) -> Option<SessionDescription> {
+    pub fn add_transceiver(
-        let remote_description = self.cxx_handle.remote_description();
+        &self,
-        if remote_description.is_null() {
+        track: MediaStreamTrack,
-            None
+        init: RtpTransceiverInit,
-        } else {
+    ) -> Result<RtpTransceiver, RtcError> {
-            Some(SessionDescription::new(remote_description))
+        self.handle.add_transceiver(track, init)
-        }
     }
 
-    pub fn signaling_state(&self) -> SignalingState {
+    pub fn add_transceiver_for_media(
-        self.cxx_handle.signaling_state()
+        &self,
+        media_type: MediaType,
+        init: RtpTransceiverInit,
+    ) -> Result<RtpTransceiver, RtcError> {
+        self.handle.add_transceiver_for_media(media_type, init)
+    }
+    pub fn close(&self) {
+        self.handle.close()
     }
 
-    pub fn ice_gathering_state(&self) -> IceGatheringState {
+    pub fn connection_state(&self) -> PeerConnectionState {
-        self.cxx_handle.ice_gathering_state()
+        self.handle.connection_state()
     }
 
     pub fn ice_connection_state(&self) -> IceConnectionState {
-        self.cxx_handle.ice_connection_state()
+        self.handle.ice_connection_state()
     }
 
-    pub fn close(&mut self) {
+    pub fn ice_gathering_state(&self) -> IceGatheringState {
-        self.cxx_handle.pin_mut().close();
+        self.handle.ice_gathering_state()
     }
 
-    pub fn on_signaling_change(&mut self, handler: OnSignalingChangeHandler) {
+    pub fn signaling_state(&self) -> SignalingState {
-        *self.observer.on_signaling_change_handler.lock().unwrap() = Some(handler);
+        self.handle.signaling_state()
     }
 
-    pub fn on_add_stream(&mut self, handler: OnAddStreamHandler) {
+    pub fn current_local_description(&self) -> Option<SessionDescription> {
-        *self.observer.on_add_stream_handler.lock().unwrap() = Some(handler);
+        self.handle.current_local_description()
     }
 
-    pub fn on_remove_stream(&mut self, handler: OnRemoveStreamHandler) {
+    pub fn current_remote_description(&self) -> Option<SessionDescription> {
-        *self.observer.on_remove_stream_handler.lock().unwrap() = Some(handler);
+        self.handle.current_remote_description()
     }
 
-    pub fn on_data_channel(&mut self, handler: OnDataChannelHandler) {
+    pub fn senders(&self) -> Vec<RtpSender> {
-        *self.observer.on_data_channel_handler.lock().unwrap() = Some(handler);
+        self.handle.senders()
     }
 
-    pub fn on_renegotiation_needed(&mut self, handler: OnRenegotiationNeededHandler) {
+    pub fn receivers(&self) -> Vec<RtpReceiver> {
-        *self
+        self.handle.receivers()
-            .observer
-            .on_renegotiation_needed_handler
-            .lock()
-            .unwrap() = Some(handler);
     }
 
-    pub fn on_ice_connection_change(&mut self, handler: OnIceConnectionChangeHandler) {
+    pub fn transceivers(&self) -> Vec<RtpTransceiver> {
-        *self
+        self.handle.transceivers()
-            .observer
-            .on_ice_connection_change_handler
-            .lock()
-            .unwrap() = Some(handler);
     }
 
-    pub fn on_standardized_ice_connection_change(
+    pub fn on_connection_state_change(&self, f: Option<OnConnectionChange>) {
-        &mut self,
+        self.handle.on_connection_state_change(f)
-        handler: OnStandardizedIceConnectionChangeHandler,
-    ) {
-        *self
-            .observer
-            .on_standardized_ice_connection_change_handler
-            .lock()
-            .unwrap() = Some(handler);
     }
 
-    pub fn on_connection_change(&mut self, handler: OnConnectionChangeHandler) {
+    pub fn on_data_channel(&self, f: Option<OnDataChannel>) {
-        *self.observer.on_connection_change_handler.lock().unwrap() = Some(handler);
+        self.handle.on_data_channel(f)
     }
 
-    pub fn on_ice_gathering_change(&mut self, handler: OnIceGatheringChangeHandler) {
+    pub fn on_ice_candidate(&self, f: Option<OnIceCandidate>) {
-        *self
+        self.handle.on_ice_candidate(f)
-            .observer
-            .on_ice_gathering_change_handler
-            .lock()
-            .unwrap() = Some(handler);
     }
 
-    pub fn on_ice_candidate(&mut self, handler: OnIceCandidateHandler) {
+    pub fn on_ice_candidate_error(&self, f: Option<OnIceCandidateError>) {
-        *self.observer.on_ice_candidate_handler.lock().unwrap() = Some(handler);
+        self.handle.on_ice_candidate_error(f)
     }
 
-    pub fn on_ice_candidate_error(&mut self, handler: OnIceCandidateErrorHandler) {
+    pub fn on_ice_connection_state_change(&self, f: Option<OnIceConnectionChange>) {
-        *self.observer.on_ice_candidate_error_handler.lock().unwrap() = Some(handler);
+        self.handle.on_ice_connection_state_change(f)
     }
 
-    pub fn on_ice_candidates_removed(&mut self, handler: OnIceCandidatesRemovedHandler) {
+    pub fn on_ice_gathering_state_change(&self, f: Option<OnIceGatheringChange>) {
-        *self
+        self.handle.on_ice_gathering_state_change(f)
-            .observer
-            .on_ice_candidates_removed_handler
-            .lock()
-            .unwrap() = Some(handler);
     }
 
-    pub fn on_ice_connection_receiving_change(
+    pub fn on_negotiation_needed(&self, f: Option<OnNegotiationNeeded>) {
-        &mut self,
+        self.handle.on_negotiation_needed(f)
-        handler: OnIceConnectionReceivingChangeHandler,
-    ) {
-        *self
-            .observer
-            .on_ice_connection_receiving_change_handler
-            .lock()
-            .unwrap() = Some(handler);
     }
 
-    pub fn on_ice_selected_candidate_pair_changed(
+    pub fn on_signaling_state_change(&self, f: Option<OnSignalingChange>) {
-        &mut self,
+        self.handle.on_signaling_state_change(f)
-        handler: OnIceSelectedCandidatePairChangedHandler,
-    ) {
-        *self
-            .observer
-            .on_ice_selected_candidate_pair_changed_handler
-            .lock()
-            .unwrap() = Some(handler);
     }
 
-    pub fn on_add_track(&mut self, handler: OnAddTrackHandler) {
+    pub fn on_track(&self, f: Option<OnTrack>) {
-        *self.observer.on_add_track_handler.lock().unwrap() = Some(handler);
+        self.handle.on_track(f)
-    }
-
-    pub fn on_track(&mut self, handler: OnTrackHandler) {
-        *self.observer.on_track_handler.lock().unwrap() = Some(handler);
-    }
-
-    pub fn on_remove_track(&mut self, handler: OnRemoveTrackHandler) {
-        *self.observer.on_remove_track_handler.lock().unwrap() = Some(handler);
-    }
-
-    pub fn on_interesting_usage(&mut self, handler: OnInterestingUsageHandler) {
-        *self.observer.on_interesting_usage_handler.lock().unwrap() = Some(handler);
     }
 }
 
-// TODO(theomonnom) Should we return futures?
+impl Debug for PeerConnection {
-pub type OnSignalingChangeHandler = Box<dyn FnMut(SignalingState) + Send + Sync>;
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-pub type OnAddStreamHandler = Box<dyn FnMut(MediaStream) + Send + Sync>;
+        f.debug_struct("PeerConnection")
-pub type OnRemoveStreamHandler = Box<dyn FnMut(MediaStream) + Send + Sync>;
+            .field("state", &self.connection_state())
-pub type OnDataChannelHandler = Box<dyn FnMut(DataChannel) + Send + Sync>;
+            .field("ice_state", &self.ice_connection_state())
-pub type OnRenegotiationNeededHandler = Box<dyn FnMut() + Send + Sync>;
+            .finish()
-pub type OnNegotiationNeededEventHandler = Box<dyn FnMut(u32) + Send + Sync>;
-pub type OnIceConnectionChangeHandler = Box<dyn FnMut(IceConnectionState) + Send + Sync>;
-pub type OnStandardizedIceConnectionChangeHandler =
-    Box<dyn FnMut(IceConnectionState) + Send + Sync>;
-pub type OnConnectionChangeHandler = Box<dyn FnMut(PeerConnectionState) + Send + Sync>;
-pub type OnIceGatheringChangeHandler = Box<dyn FnMut(IceGatheringState) + Send + Sync>;
-pub type OnIceCandidateHandler = Box<dyn FnMut(IceCandidate) + Send + Sync>;
-pub type OnIceCandidateErrorHandler =
-    Box<dyn FnMut(String, i32, String, i32, String) + Send + Sync>;
-pub type OnIceCandidatesRemovedHandler = Box<dyn FnMut(Vec<IceCandidate>) + Send + Sync>;
-pub type OnIceConnectionReceivingChangeHandler = Box<dyn FnMut(bool) + Send + Sync>;
-pub type OnIceSelectedCandidatePairChangedHandler =
-    Box<dyn FnMut(webrtc_sys::peer_connection::ffi::CandidatePairChangeEvent) + Send + Sync>;
-pub type OnAddTrackHandler = Box<dyn FnMut(RtpReceiver, Vec<MediaStream>) + Send + Sync>;
-pub type OnTrackHandler = Box<dyn FnMut(RtpTransceiver) + Send + Sync>;
-pub type OnRemoveTrackHandler = Box<dyn FnMut(RtpReceiver) + Send + Sync>;
-pub type OnInterestingUsageHandler = Box<dyn FnMut(i32) + Send + Sync>;
-
-pub(crate) struct InternalObserver {
-    on_signaling_change_handler: Arc<Mutex<Option<OnSignalingChangeHandler>>>,
-    on_add_stream_handler: Arc<Mutex<Option<OnAddStreamHandler>>>,
-    on_remove_stream_handler: Arc<Mutex<Option<OnRemoveStreamHandler>>>,
-    on_data_channel_handler: Arc<Mutex<Option<OnDataChannelHandler>>>,
-    on_renegotiation_needed_handler: Arc<Mutex<Option<OnRenegotiationNeededHandler>>>,
-    on_negotiation_needed_event_handler: Arc<Mutex<Option<OnNegotiationNeededEventHandler>>>,
-    on_ice_connection_change_handler: Arc<Mutex<Option<OnIceConnectionChangeHandler>>>,
-    on_standardized_ice_connection_change_handler:
-        Arc<Mutex<Option<OnStandardizedIceConnectionChangeHandler>>>,
-    on_connection_change_handler: Arc<Mutex<Option<OnConnectionChangeHandler>>>,
-    on_ice_gathering_change_handler: Arc<Mutex<Option<OnIceGatheringChangeHandler>>>,
-    on_ice_candidate_handler: Arc<Mutex<Option<OnIceCandidateHandler>>>,
-    on_ice_candidate_error_handler: Arc<Mutex<Option<OnIceCandidateErrorHandler>>>,
-    on_ice_candidates_removed_handler: Arc<Mutex<Option<OnIceCandidatesRemovedHandler>>>,
-    on_ice_connection_receiving_change_handler:
-        Arc<Mutex<Option<OnIceConnectionReceivingChangeHandler>>>,
-    on_ice_selected_candidate_pair_changed_handler:
-        Arc<Mutex<Option<OnIceSelectedCandidatePairChangedHandler>>>,
-    on_add_track_handler: Arc<Mutex<Option<OnAddTrackHandler>>>,
-    on_track_handler: Arc<Mutex<Option<OnTrackHandler>>>,
-    on_remove_track_handler: Arc<Mutex<Option<OnRemoveTrackHandler>>>,
-    on_interesting_usage_handler: Arc<Mutex<Option<OnInterestingUsageHandler>>>,
-}
-
-impl Default for InternalObserver {
-    fn default() -> Self {
-        Self {
-            on_signaling_change_handler: Arc::new(Default::default()),
-            on_add_stream_handler: Arc::new(Default::default()),
-            on_remove_stream_handler: Arc::new(Default::default()),
-            on_data_channel_handler: Arc::new(Default::default()),
-            on_renegotiation_needed_handler: Arc::new(Default::default()),
-            on_negotiation_needed_event_handler: Arc::new(Default::default()),
-            on_ice_connection_change_handler: Arc::new(Default::default()),
-            on_standardized_ice_connection_change_handler: Arc::new(Default::default()),
-            on_connection_change_handler: Arc::new(Default::default()),
-            on_ice_gathering_change_handler: Arc::new(Default::default()),
-            on_ice_candidate_handler: Arc::new(Default::default()),
-            on_ice_candidate_error_handler: Arc::new(Default::default()),
-            on_ice_candidates_removed_handler: Arc::new(Default::default()),
-            on_ice_connection_receiving_change_handler: Arc::new(Default::default()),
-            on_ice_selected_candidate_pair_changed_handler: Arc::new(Default::default()),
-            on_add_track_handler: Arc::new(Default::default()),
-            on_track_handler: Arc::new(Default::default()),
-            on_remove_track_handler: Arc::new(Default::default()),
-            on_interesting_usage_handler: Arc::new(Default::default()),
-        }
-    }
-}
-
-// Observers are being called on the Signaling Thread
-impl sys_pc::PeerConnectionObserver for InternalObserver {
-    fn on_signaling_change(&self, new_state: SignalingState) {
-        trace!("on_signaling_change, {:?}", new_state);
-        let mut handler = self.on_signaling_change_handler.lock().unwrap();
-        if let Some(f) = handler.as_mut() {
-            f(new_state);
-        }
-    }
-
-    fn on_add_stream(&self, stream: SharedPtr<sys_ms::ffi::MediaStream>) {
-        trace!("on_add_stream");
-        let mut handler = self.on_add_stream_handler.lock().unwrap();
-        if let Some(f) = handler.as_mut() {
-            // TODO(theomonnom)
-        }
-    }
-
-    fn on_remove_stream(&self, stream: SharedPtr<sys_ms::ffi::MediaStream>) {
-        trace!("on_remove_stream");
-        let mut handler = self.on_remove_stream_handler.lock().unwrap();
-        if let Some(f) = handler.as_mut() {
-            // TODO(theomonnom)
-        }
-    }
-
-    fn on_data_channel(&self, data_channel: UniquePtr<sys_dc::ffi::DataChannel>) {
-        trace!("on_data_channel");
-        let mut handler = self.on_data_channel_handler.lock().unwrap();
-        if let Some(f) = handler.as_mut() {
-            f(DataChannel::new(data_channel));
-        }
-    }
-
-    fn on_renegotiation_needed(&self) {
-        trace!("on_renegotiation_needed");
-        let mut handler = self.on_renegotiation_needed_handler.lock().unwrap();
-        if let Some(f) = handler.as_mut() {
-            f();
-        }
-    }
-
-    fn on_negotiation_needed_event(&self, event: u32) {
-        trace!("on_negotiation_needed_event");
-        let mut handler = self.on_negotiation_needed_event_handler.lock().unwrap();
-        if let Some(f) = handler.as_mut() {
-            f(event);
-        }
-    }
-
-    fn on_ice_connection_change(&self, new_state: IceConnectionState) {
-        trace!("on_ice_connection_change (new_state: {:?})", new_state);
-        let mut handler = self.on_ice_connection_change_handler.lock().unwrap();
-        if let Some(f) = handler.as_mut() {
-            f(new_state);
-        }
-    }
-
-    fn on_standardized_ice_connection_change(&self, new_state: IceConnectionState) {
-        trace!(
-            "on_standardized_ice_connection_change (new_state: {:?}",
-            new_state
-        );
-        let mut handler = self
-            .on_standardized_ice_connection_change_handler
-            .lock()
-            .unwrap();
-        if let Some(f) = handler.as_mut() {
-            f(new_state);
-        }
-    }
-
-    fn on_connection_change(&self, new_state: PeerConnectionState) {
-        trace!("on_connection_change (new_state: {:?})", new_state);
-        let mut handler = self.on_connection_change_handler.lock().unwrap();
-        if let Some(f) = handler.as_mut() {
-            f(new_state);
-        }
-    }
-
-    fn on_ice_gathering_change(&self, new_state: IceGatheringState) {
-        trace!("on_ice_gathering_change (new_state: {:?}", new_state);
-        let mut handler = self.on_ice_gathering_change_handler.lock().unwrap();
-        if let Some(f) = handler.as_mut() {
-            f(new_state);
-        }
-    }
-
-    fn on_ice_candidate(&self, candidate: SharedPtr<sys_jsep::ffi::IceCandidate>) {
-        trace!("on_ice_candidate");
-        let mut handler = self.on_ice_candidate_handler.lock().unwrap();
-        if let Some(f) = handler.as_mut() {
-            f(IceCandidate::new(candidate));
-        }
-    }
-
-    fn on_ice_candidate_error(
-        &self,
-        address: String,
-        port: i32,
-        url: String,
-        error_code: i32,
-        error_text: String,
-    ) {
-        trace!("on_ice_candidate_error");
-        let mut handler = self.on_ice_candidate_error_handler.lock().unwrap();
-        if let Some(f) = handler.as_mut() {
-            f(address, port, url, error_code, error_text);
-        }
-    }
-
-    fn on_ice_candidates_removed(&self, removed: Vec<SharedPtr<sys_ca::ffi::Candidate>>) {
-        trace!("on_ice_candidates_removed");
-        let mut handler = self.on_ice_candidates_removed_handler.lock().unwrap();
-        if let Some(f) = handler.as_mut() {
-            // TODO(theomonnom)
-        }
-    }
-
-    fn on_ice_connection_receiving_change(&self, receiving: bool) {
-        trace!("on_ice_connection_receiving_change");
-        let mut handler = self
-            .on_ice_connection_receiving_change_handler
-            .lock()
-            .unwrap();
-        if let Some(f) = handler.as_mut() {
-            f(receiving);
-        }
-    }
-
-    fn on_ice_selected_candidate_pair_changed(&self, event: sys_pc::ffi::CandidatePairChangeEvent) {
-        trace!("on_ice_selected_candidate_pair_changed");
-        let mut handler = self
-            .on_ice_selected_candidate_pair_changed_handler
-            .lock()
-            .unwrap();
-        if let Some(f) = handler.as_mut() {
-            f(event);
-        }
-    }
-
-    fn on_add_track(
-        &self,
-        receiver: SharedPtr<sys_rr::ffi::RtpReceiver>,
-        streams: Vec<SharedPtr<sys_ms::ffi::MediaStream>>,
-    ) {
-        trace!("on_add_track");
-        let mut handler = self.on_add_track_handler.lock().unwrap();
-        if let Some(f) = handler.as_mut() {
-            let streams = streams.into_iter().map(MediaStream::new).collect();
-            f(RtpReceiver::new(receiver), streams)
-        }
-    }
-
-    fn on_track(&self, transceiver: SharedPtr<sys_rt::ffi::RtpTransceiver>) {
-        trace!("on_track");
-        let mut handler = self.on_track_handler.lock().unwrap();
-        if let Some(f) = handler.as_mut() {
-            // TODO(theomonnom)
-        }
-    }
-
-    fn on_remove_track(&self, receiver: SharedPtr<sys_rr::ffi::RtpReceiver>) {
-        trace!("on_remove_track");
-        let mut handler = self.on_remove_track_handler.lock().unwrap();
-        if let Some(f) = handler.as_mut() {
-            // TODO(theomonnom)
-        }
-    }
-
-    fn on_interesting_usage(&self, usage_pattern: i32) {
-        trace!("on_interesting_usage");
-        let mut handler = self.on_interesting_usage_handler.lock().unwrap();
-        if let Some(f) = handler.as_mut() {
-            f(usage_pattern);
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use log::trace;
-    use tokio::sync::mpsc;
-
-    use webrtc_sys::peer_connection::ffi::RTCOfferAnswerOptions;
-    use webrtc_sys::peer_connection_factory::ffi::{ContinualGatheringPolicy, IceTransportsType};
-
-    use crate::data_channel::{DataChannel, DataChannelInit};
-    use crate::jsep::IceCandidate;
-    use crate::peer_connection_factory::{ICEServer, PeerConnectionFactory, RTCConfiguration};
-    use crate::webrtc::RTCRuntime;
-
-    fn init_log() {
-        let _ = env_logger::builder().is_test(true).try_init();
-    }
-
-    #[tokio::test]
-    async fn create_pc() {
-        init_log();
-
-        let rtc_runtime = RTCRuntime::new();
-
-        let factory = PeerConnectionFactory::new(rtc_runtime);
-        let config = RTCConfiguration {
-            ice_servers: vec![ICEServer {
-                urls: vec!["stun:stun1.l.google.com:19302".to_string()],
-                username: "".into(),
-                password: "".into(),
-            }],
-            continual_gathering_policy: ContinualGatheringPolicy::GatherOnce,
-            ice_transport_type: IceTransportsType::All,
-        };
-
-        let mut bob = factory.create_peer_connection(config.clone()).unwrap();
-        let mut alice = factory.create_peer_connection(config.clone()).unwrap();
-
-        let (bob_ice_tx, mut bob_ice_rx) = mpsc::channel::<IceCandidate>(16);
-        let (alice_ice_tx, mut alice_ice_rx) = mpsc::channel::<IceCandidate>(16);
-        let (alice_dc_tx, mut alice_dc_rx) = mpsc::channel::<DataChannel>(16);
-
-        bob.on_ice_candidate(Box::new(move |candidate| {
-            bob_ice_tx.blocking_send(candidate).unwrap();
-        }));
-
-        alice.on_ice_candidate(Box::new(move |candidate| {
-            alice_ice_tx.blocking_send(candidate).unwrap();
-        }));
-
-        alice.on_data_channel(Box::new(move |dc| {
-            alice_dc_tx.blocking_send(dc).unwrap();
-        }));
-
-        let mut bob_dc = bob
-            .create_data_channel("test_dc", DataChannelInit::default())
-            .unwrap();
-
-        let offer = bob
-            .create_offer(RTCOfferAnswerOptions::default())
-            .await
-            .unwrap();
-        trace!("Bob offer: {:?}", offer);
-        bob.set_local_description(offer.clone()).await.unwrap();
-        alice.set_remote_description(offer).await.unwrap();
-
-        let answer = alice
-            .create_answer(RTCOfferAnswerOptions::default())
-            .await
-            .unwrap();
-
-        trace!("Alice answer: {:?}", answer);
-        alice.set_local_description(answer.clone()).await.unwrap();
-        bob.set_remote_description(answer).await.unwrap();
-
-        let bob_ice = bob_ice_rx.recv().await.unwrap();
-        let alice_ice = alice_ice_rx.recv().await.unwrap();
-
-        bob.add_ice_candidate(alice_ice).await.unwrap();
-        alice.add_ice_candidate(bob_ice).await.unwrap();
-
-        let (data_tx, mut data_rx) = mpsc::channel::<String>(1);
-        let mut alice_dc = alice_dc_rx.recv().await.unwrap();
-        alice_dc.on_message(Box::new(move |data, _| {
-            data_tx
-                .blocking_send(String::from_utf8_lossy(data).to_string())
-                .unwrap();
-        }));
-
-        bob_dc.send(b"This is a test", true).unwrap();
-        assert_eq!(data_rx.recv().await.unwrap(), "This is a test");
-
-        alice.close();
-        bob.close();
     }
 }

livekit-webrtc/src/peer_connection_factory.rs

@@ -1,51 +1,78 @@
-use cxx::UniquePtr;
+use crate::imp::peer_connection_factory as imp_pcf;
+use crate::peer_connection::PeerConnection;
+use crate::rtp_parameters::RtpCapabilities;
+use crate::MediaType;
+use crate::RtcError;
+use std::fmt::Debug;
 
-pub use sys_factory::ffi::{
+#[derive(Debug, Clone)]
-    ContinualGatheringPolicy, ICEServer, IceTransportsType, RTCConfiguration,
+pub struct IceServer {
-};
+    pub urls: Vec<String>,
-use webrtc_sys::peer_connection as sys_pc;
+    pub username: String,
-use webrtc_sys::peer_connection_factory as sys_factory;
+    pub password: String,
+}
 
-use crate::peer_connection::{InternalObserver, PeerConnection};
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-use crate::rtc_error::RTCError;
+pub enum ContinualGatheringPolicy {
-use crate::webrtc::RTCRuntime;
+    GatherOnce,
+    GatherContinually,
+}
 
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub enum IceTransportsType {
+    None,
+    Relay,
+    NoHost,
+    All,
+}
+
+#[derive(Debug, Clone)]
+pub struct RtcConfiguration {
+    pub ice_servers: Vec<IceServer>,
+    pub continual_gathering_policy: ContinualGatheringPolicy,
+    pub ice_transport_type: IceTransportsType,
+}
+
+#[derive(Clone, Default)]
 pub struct PeerConnectionFactory {
-    cxx_handle: UniquePtr<sys_factory::ffi::PeerConnectionFactory>,
+    pub(crate) handle: imp_pcf::PeerConnectionFactory,
-    rtc_runtime: RTCRuntime,
+}
+
+impl Debug for PeerConnectionFactory {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        f.debug_struct("PeerConnectionFactory").finish()
+    }
 }
 
 impl PeerConnectionFactory {
-    pub fn new(rtc_runtime: RTCRuntime) -> Self {
-        Self {
-            cxx_handle: sys_factory::ffi::create_peer_connection_factory(
-                rtc_runtime.clone().release(),
-            ),
-            rtc_runtime,
-        }
-    }
-
     pub fn create_peer_connection(
         &self,
-        config: RTCConfiguration,
+        config: RtcConfiguration,
-    ) -> Result<PeerConnection, RTCError> {
+    ) -> Result<PeerConnection, RtcError> {
-        let native_config = sys_factory::ffi::create_rtc_configuration(config);
+        self.handle.create_peer_connection(config)
-
-        unsafe {
-            let mut observer = Box::new(InternalObserver::default());
-            let mut native_observer = sys_pc::ffi::create_native_peer_connection_observer(
-                self.rtc_runtime.clone().release(),
-                Box::new(sys_pc::PeerConnectionObserverWrapper::new(&mut *observer)),
-            );
-
-            let res = self
-                .cxx_handle
-                .create_peer_connection(native_config, native_observer.pin_mut());
-
-            match res {
-                Ok(cxx_handle) => Ok(PeerConnection::new(cxx_handle, observer, native_observer)),
-                Err(e) => Err(RTCError::from(e.what())),
     }
+
+    pub fn get_rtp_sender_capabilities(&self, media_type: MediaType) -> RtpCapabilities {
+        self.handle.get_rtp_sender_capabilities(media_type)
+    }
+
+    pub fn get_rtp_receiver_capabilities(&self, media_type: MediaType) -> RtpCapabilities {
+        self.handle.get_rtp_receiver_capabilities(media_type)
+    }
+}
+
+pub mod native {
+    use super::PeerConnectionFactory;
+    use crate::media_stream::RtcVideoTrack;
+    use crate::video_source::native::NativeVideoSource;
+
+    pub trait PeerConnectionFactoryExt {
+        fn create_video_track(&self, label: &str, source: NativeVideoSource) -> RtcVideoTrack;
+    }
+
+    impl PeerConnectionFactoryExt for PeerConnectionFactory {
+        fn create_video_track(&self, label: &str, source: NativeVideoSource) -> RtcVideoTrack {
+            self.handle.create_video_track(label, source)
         }
     }
 }

livekit-webrtc/src/prelude.rs

@@ -1,22 +1,24 @@
-pub use crate::data_channel::{DataChannel, DataChannelInit, DataState};
+pub use crate::data_channel::{
-pub use crate::jsep::{IceCandidate, SessionDescription};
+    DataBuffer, DataChannel, DataChannelError, DataChannelInit, DataState,
+};
+pub use crate::ice_candidate::IceCandidate;
 pub use crate::media_stream::{
-    AudioTrack, MediaStream, MediaStreamTrackHandle, MediaStreamTrackTrait,
+    MediaStream, MediaStreamTrack, RtcAudioTrack, RtcTrackState, RtcVideoTrack,
-    OnConstraintsChangedHandler, OnDiscardedFrameHandler, OnFrameHandler, VideoTrack,
 };
 pub use crate::peer_connection::{
-    IceConnectionState, IceGatheringState, PeerConnection, PeerConnectionState,
+    AnswerOptions, IceConnectionState, IceGatheringState, OfferOptions, PeerConnection,
-    RTCOfferAnswerOptions, SignalingState,
+    PeerConnectionState, SignalingState,
 };
 pub use crate::peer_connection_factory::{
-    ContinualGatheringPolicy, ICEServer, IceTransportsType, PeerConnectionFactory, RTCConfiguration,
+    ContinualGatheringPolicy, IceServer, IceTransportsType, PeerConnectionFactory, RtcConfiguration,
 };
-pub use crate::rtc_error::RTCError;
 pub use crate::rtp_parameters::*;
 pub use crate::rtp_receiver::RtpReceiver;
 pub use crate::rtp_sender::RtpSender;
-pub use crate::rtp_transceiver::{RtpTransceiver, RtpTransceiverInit};
+pub use crate::rtp_transceiver::{RtpTransceiver, RtpTransceiverDirection, RtpTransceiverInit};
-pub use crate::video_frame::{VideoFrame, VideoRotation};
+pub use crate::session_description::{SdpType, SessionDescription};
-pub use crate::video_frame_buffer::*;
+pub use crate::video_frame::{
-pub use crate::webrtc::*;
+    BoxVideoFrame, I010Buffer, I420ABuffer, I420Buffer, I422Buffer, I444Buffer, NV12Buffer,
-pub use crate::yuv_helper::ConvertError;
+    VideoFormatType, VideoFrame, VideoFrameBuffer, VideoFrameBufferType, VideoRotation,
+};
+pub use crate::{RtcError, RtcErrorType};

livekit-webrtc/src/rtc_error.rs

@@ -1,2 +0,0 @@
-// TODO(theomonnom) Wrap the RTCError ffi so we can use Option(u16)
-pub use webrtc_sys::rtc_error::ffi::RTCError;

livekit-webrtc/src/rtp_parameters.rs

@@ -1,462 +1,91 @@
-use crate::prelude::*;
+use crate::rtp_transceiver::RtpTransceiverDirection;
-use std::collections::HashMap;
-use std::vec::Vec;
-use webrtc_sys::rtp_parameters as ps_sys;
 
-// Don't exporting structs here (only enum), cxx doesn't support Option and HashMap
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-pub use ps_sys::ffi::{
+pub enum Priority {
-    DegradationPreference, FecMechanism, RtcpFeedbackMessageType, RtcpFeedbackType,
+    VeryLow,
-    RtpExtensionFilter,
+    Low,
-};
+    Medium,
+    High,
+}
 
 #[derive(Debug, Clone)]
-pub struct RtcpFeedback {
+pub struct RtpHeaderExtensionParameters {
-    pub feedback_type: RtcpFeedbackType,
+    pub uri: String,
-    pub message_type: Option<RtcpFeedbackMessageType>,
+    pub id: i32,
+    pub encrypted: bool,
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct RtpParameters {
+    pub codecs: Vec<RtpCodecParameters>,
+    pub header_extensions: Vec<RtpHeaderExtensionParameters>,
+    pub rtcp: RtcpParameters,
+}
+
+#[derive(Debug, Clone)]
+pub struct RtpCodecParameters {
+    pub payload_type: u8,
+    pub mime_type: String, // read-only
+    pub clock_rate: Option<u64>,
+    pub channels: Option<u16>,
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct RtcpParameters {
+    pub cname: String,
+    pub reduced_size: bool,
+}
+
+#[derive(Debug, Clone)]
+pub struct RtpEncodingParameters {
+    pub active: bool,
+    pub max_bitrate: Option<u64>,
+    pub max_framerate: Option<f64>,
+    pub priority: Priority,
+    pub rid: String,
+    pub scale_resolution_down_by: Option<f64>,
 }
 
 #[derive(Debug, Clone)]
 pub struct RtpCodecCapability {
+    pub channels: Option<u16>,
+    pub clock_rate: Option<u64>,
     pub mime_type: String,
-    pub name: String,
+    pub sdp_fmtp_line: Option<String>,
-    pub kind: MediaType,
-    pub clock_rate: Option<i32>,
-    pub preferred_payload_type: Option<i32>,
-    pub max_ptime: Option<i32>,
-    pub ptime: Option<i32>,
-    pub num_channels: Option<i32>,
-    pub rtcp_feedback: Vec<RtcpFeedback>,
-    pub parameters: HashMap<String, String>,
-    pub options: HashMap<String, String>,
-    pub max_temporal_layer_extensions: i32,
-    pub max_spatial_layer_extensions: i32,
-    pub svc_multi_stream_support: bool,
 }
 
 #[derive(Debug, Clone)]
 pub struct RtpHeaderExtensionCapability {
     pub uri: String,
-    pub preferred_id: Option<i32>,
-    pub preferred_encrypt: bool,
     pub direction: RtpTransceiverDirection,
 }
 
-#[derive(Debug, Clone)]
-pub struct RtpExtension {
-    pub uri: String,
-    pub id: i32,
-    pub encrypt: bool,
-}
-
-#[derive(Debug, Clone)]
-pub struct RtpFecParameters {
-    pub ssrc: Option<u32>,
-    pub mechanism: FecMechanism,
-}
-
-#[derive(Debug, Clone)]
-pub struct RtpRtxParameters {
-    pub ssrc: Option<u32>,
-}
-
-#[derive(Debug, Clone)]
-pub struct RtpEncodingParameters {
-    pub ssrc: Option<u32>,
-    pub bitrate_priority: f64,
-    pub network_priority: Priority,
-    pub max_bitrate_bps: Option<i32>,
-    pub min_bitrate_bps: Option<i32>,
-    pub max_framerate: Option<f64>,
-    pub num_temporal_layers: Option<i32>,
-    pub scale_resolution_down_by: Option<f64>,
-    pub scalability_mode: Option<String>,
-    pub active: bool,
-    pub rid: String,
-    pub adaptive_ptime: bool,
-}
-
-#[derive(Debug, Clone)]
-pub struct RtpCodecParameters {
-    pub mime_type: String,
-    pub name: String,
-    pub kind: MediaType,
-    pub payload_type: i32,
-    pub clock_rate: Option<i32>,
-    pub num_channels: Option<i32>,
-    pub max_ptime: Option<i32>,
-    pub ptime: Option<i32>,
-    pub rtcp_feedback: Vec<RtcpFeedback>,
-    pub parameters: HashMap<String, String>,
-}
-
 #[derive(Debug, Clone)]
 pub struct RtpCapabilities {
     pub codecs: Vec<RtpCodecCapability>,
     pub header_extensions: Vec<RtpHeaderExtensionCapability>,
-    pub fec: Vec<FecMechanism>,
 }
 
-#[derive(Debug, Clone)]
+impl Default for RtpCodecParameters {
-pub struct RtcpParameters {
+    fn default() -> Self {
-    pub ssrc: Option<u32>,
-    pub cname: String,
-    pub reduced_size: bool,
-    pub mux: bool,
-}
-
-#[derive(Debug, Clone)]
-pub struct RtpParameters {
-    pub transaction_id: String,
-    pub mid: String,
-    pub codecs: Vec<RtpCodecParameters>,
-    pub header_extensions: Vec<RtpExtension>,
-    pub encodings: Vec<RtpEncodingParameters>,
-    pub rtcp: RtcpParameters,
-    pub degradation_preference: Option<DegradationPreference>,
-}
-
-fn into_map(vec: Vec<ps_sys::ffi::StringKeyValue>) -> HashMap<String, String> {
-    let mut map = HashMap::with_capacity(vec.len());
-    for pair in vec {
-        map.insert(pair.key, pair.value);
-    }
-    map
-}
-
-impl From<ps_sys::ffi::RtcpFeedback> for RtcpFeedback {
-    fn from(value: ps_sys::ffi::RtcpFeedback) -> Self {
         Self {
-            feedback_type: value.feedback_type,
+            payload_type: 0,
-            message_type: value.has_message_type.then_some(value.message_type),
+            mime_type: String::default(),
+            clock_rate: None,
+            channels: None,
         }
     }
 }
 
-impl From<ps_sys::ffi::RtpCodecCapability> for RtpCodecCapability {
+impl Default for RtpEncodingParameters {
-    fn from(value: ps_sys::ffi::RtpCodecCapability) -> Self {
+    fn default() -> Self {
         Self {
-            mime_type: value.mime_type,
+            active: true,
-            name: value.name,
+            max_bitrate: None,
-            kind: value.kind,
+            max_framerate: None,
-            clock_rate: value.has_clock_rate.then_some(value.clock_rate),
+            priority: Priority::Low,
-            preferred_payload_type: value
+            rid: String::default(),
-                .has_preferred_payload_type
+            scale_resolution_down_by: None,
-                .then_some(value.preferred_payload_type),
-            max_ptime: value.has_max_ptime.then_some(value.max_ptime),
-            ptime: value.has_ptime.then_some(value.ptime),
-            num_channels: value.has_num_channels.then_some(value.num_channels),
-            rtcp_feedback: value.rtcp_feedback.into_iter().map(Into::into).collect(),
-            parameters: into_map(value.parameters),
-            options: into_map(value.options),
-            max_temporal_layer_extensions: value.max_temporal_layer_extensions,
-            max_spatial_layer_extensions: value.max_spatial_layer_extensions,
-            svc_multi_stream_support: value.svc_multi_stream_support,
-        }
-    }
-}
-
-impl From<ps_sys::ffi::RtpHeaderExtensionCapability> for RtpHeaderExtensionCapability {
-    fn from(value: ps_sys::ffi::RtpHeaderExtensionCapability) -> Self {
-        Self {
-            uri: value.uri,
-            preferred_id: value.has_preferred_id.then_some(value.preferred_id),
-            preferred_encrypt: value.preferred_encrypt,
-            direction: value.direction,
-        }
-    }
-}
-
-impl From<ps_sys::ffi::RtpExtension> for RtpExtension {
-    fn from(value: ps_sys::ffi::RtpExtension) -> Self {
-        Self {
-            uri: value.uri,
-            id: value.id,
-            encrypt: value.encrypt,
-        }
-    }
-}
-
-impl From<ps_sys::ffi::RtpFecParameters> for RtpFecParameters {
-    fn from(value: ps_sys::ffi::RtpFecParameters) -> Self {
-        Self {
-            ssrc: value.has_ssrc.then_some(value.ssrc),
-            mechanism: value.mechanism,
-        }
-    }
-}
-
-impl From<ps_sys::ffi::RtpRtxParameters> for RtpRtxParameters {
-    fn from(value: ps_sys::ffi::RtpRtxParameters) -> Self {
-        Self {
-            ssrc: value.has_ssrc.then_some(value.ssrc),
-        }
-    }
-}
-
-impl From<ps_sys::ffi::RtpEncodingParameters> for RtpEncodingParameters {
-    fn from(value: ps_sys::ffi::RtpEncodingParameters) -> Self {
-        Self {
-            ssrc: value.has_ssrc.then_some(value.ssrc),
-            bitrate_priority: value.bitrate_priority,
-            network_priority: value.network_priority,
-            max_bitrate_bps: value.has_max_bitrate_bps.then_some(value.max_bitrate_bps),
-            min_bitrate_bps: value.has_min_bitrate_bps.then_some(value.min_bitrate_bps),
-            max_framerate: value.has_max_framerate.then_some(value.max_framerate),
-            num_temporal_layers: value
-                .has_num_temporal_layers
-                .then_some(value.num_temporal_layers),
-            scale_resolution_down_by: value
-                .has_scale_resolution_down_by
-                .then_some(value.scale_resolution_down_by),
-            scalability_mode: value.has_scalability_mode.then_some(value.scalability_mode),
-            active: value.active,
-            rid: value.rid,
-            adaptive_ptime: value.adaptive_ptime,
-        }
-    }
-}
-
-impl From<ps_sys::ffi::RtpCodecParameters> for RtpCodecParameters {
-    fn from(value: ps_sys::ffi::RtpCodecParameters) -> Self {
-        Self {
-            mime_type: value.mime_type,
-            name: value.name,
-            kind: value.kind,
-            payload_type: value.payload_type,
-            clock_rate: value.has_clock_rate.then_some(value.clock_rate),
-            num_channels: value.has_num_channels.then_some(value.num_channels),
-            max_ptime: value.has_max_ptime.then_some(value.max_ptime),
-            ptime: value.has_ptime.then_some(value.ptime),
-            rtcp_feedback: value.rtcp_feedback.into_iter().map(Into::into).collect(),
-            parameters: into_map(value.parameters),
-        }
-    }
-}
-
-impl From<ps_sys::ffi::RtpCapabilities> for RtpCapabilities {
-    fn from(value: ps_sys::ffi::RtpCapabilities) -> Self {
-        Self {
-            codecs: value.codecs.into_iter().map(Into::into).collect(),
-            header_extensions: value
-                .header_extensions
-                .into_iter()
-                .map(Into::into)
-                .collect(),
-            fec: value.fec.into_iter().map(Into::into).collect(),
-        }
-    }
-}
-
-impl From<ps_sys::ffi::RtcpParameters> for RtcpParameters {
-    fn from(value: ps_sys::ffi::RtcpParameters) -> Self {
-        Self {
-            ssrc: value.has_ssrc.then_some(value.ssrc),
-            cname: value.cname,
-            reduced_size: value.reduced_size,
-            mux: value.mux,
-        }
-    }
-}
-
-impl From<ps_sys::ffi::RtpParameters> for RtpParameters {
-    fn from(value: ps_sys::ffi::RtpParameters) -> Self {
-        Self {
-            transaction_id: value.transaction_id,
-            mid: value.mid,
-            codecs: value.codecs.into_iter().map(Into::into).collect(),
-            header_extensions: value
-                .header_extensions
-                .into_iter()
-                .map(Into::into)
-                .collect(),
-            encodings: value.encodings.into_iter().map(Into::into).collect(),
-            rtcp: value.rtcp.into(),
-            degradation_preference: value
-                .has_degradation_preference
-                .then_some(value.degradation_preference),
-        }
-    }
-}
-
-// Ignore the value inside unwrap_or for the following implementations
-
-fn into_vec(map: HashMap<String, String>) -> Vec<ps_sys::ffi::StringKeyValue> {
-    let mut vec = Vec::with_capacity(map.len());
-    for (key, value) in map {
-        vec.push(ps_sys::ffi::StringKeyValue { key, value })
-    }
-    vec
-}
-
-impl From<RtcpFeedback> for ps_sys::ffi::RtcpFeedback {
-    fn from(value: RtcpFeedback) -> Self {
-        Self {
-            feedback_type: value.feedback_type,
-            has_message_type: value.message_type.is_some(),
-            message_type: value
-                .message_type
-                .unwrap_or(RtcpFeedbackMessageType::GenericNACK),
-        }
-    }
-}
-
-impl From<RtpCodecCapability> for ps_sys::ffi::RtpCodecCapability {
-    fn from(value: RtpCodecCapability) -> Self {
-        Self {
-            mime_type: value.mime_type,
-            name: value.name,
-            kind: value.kind,
-            has_clock_rate: value.clock_rate.is_some(),
-            clock_rate: value.clock_rate.unwrap_or(0),
-            has_preferred_payload_type: value.preferred_payload_type.is_some(),
-            preferred_payload_type: value.preferred_payload_type.unwrap_or(0),
-            has_max_ptime: value.max_ptime.is_some(),
-            max_ptime: value.max_ptime.unwrap_or(0),
-            has_ptime: value.ptime.is_some(),
-            ptime: value.ptime.unwrap_or(0),
-            has_num_channels: value.num_channels.is_some(),
-            num_channels: value.num_channels.unwrap_or(0),
-            rtcp_feedback: value.rtcp_feedback.into_iter().map(Into::into).collect(),
-            parameters: into_vec(value.parameters),
-            options: into_vec(value.options),
-            max_temporal_layer_extensions: value.max_temporal_layer_extensions,
-            max_spatial_layer_extensions: value.max_spatial_layer_extensions,
-            svc_multi_stream_support: value.svc_multi_stream_support,
-        }
-    }
-}
-
-impl From<RtpHeaderExtensionCapability> for ps_sys::ffi::RtpHeaderExtensionCapability {
-    fn from(value: RtpHeaderExtensionCapability) -> Self {
-        Self {
-            uri: value.uri,
-            has_preferred_id: value.preferred_id.is_some(),
-            preferred_id: value.preferred_id.unwrap_or(0),
-            preferred_encrypt: value.preferred_encrypt,
-            direction: value.direction,
-        }
-    }
-}
-
-impl From<RtpExtension> for ps_sys::ffi::RtpExtension {
-    fn from(value: RtpExtension) -> Self {
-        Self {
-            uri: value.uri,
-            id: value.id,
-            encrypt: value.encrypt,
-        }
-    }
-}
-
-impl From<RtpFecParameters> for ps_sys::ffi::RtpFecParameters {
-    fn from(value: RtpFecParameters) -> Self {
-        Self {
-            has_ssrc: value.ssrc.is_some(),
-            ssrc: value.ssrc.unwrap_or(0),
-            mechanism: value.mechanism,
-        }
-    }
-}
-
-impl From<RtpRtxParameters> for ps_sys::ffi::RtpRtxParameters {
-    fn from(value: RtpRtxParameters) -> Self {
-        Self {
-            has_ssrc: value.ssrc.is_some(),
-            ssrc: value.ssrc.unwrap_or(0),
-        }
-    }
-}
-
-impl From<RtpEncodingParameters> for ps_sys::ffi::RtpEncodingParameters {
-    fn from(value: RtpEncodingParameters) -> Self {
-        Self {
-            has_ssrc: value.ssrc.is_some(),
-            ssrc: value.ssrc.unwrap_or(0),
-            bitrate_priority: value.bitrate_priority,
-            network_priority: value.network_priority,
-            has_max_bitrate_bps: value.max_bitrate_bps.is_some(),
-            max_bitrate_bps: value.max_bitrate_bps.unwrap_or(0),
-            has_min_bitrate_bps: value.min_bitrate_bps.is_some(),
-            min_bitrate_bps: value.min_bitrate_bps.unwrap_or(0),
-            has_max_framerate: value.max_framerate.is_some(),
-            max_framerate: value.max_framerate.unwrap_or(0.0),
-            has_num_temporal_layers: value.num_temporal_layers.is_some(),
-            num_temporal_layers: value.num_temporal_layers.unwrap_or(0),
-            has_scale_resolution_down_by: value.scale_resolution_down_by.is_some(),
-            scale_resolution_down_by: value.scale_resolution_down_by.unwrap_or(0.0),
-            has_scalability_mode: value.scalability_mode.is_some(),
-            scalability_mode: value.scalability_mode.unwrap_or(String::new()),
-            active: value.active,
-            rid: value.rid,
-            adaptive_ptime: value.adaptive_ptime,
-        }
-    }
-}
-
-impl From<RtpCodecParameters> for ps_sys::ffi::RtpCodecParameters {
-    fn from(value: RtpCodecParameters) -> Self {
-        Self {
-            mime_type: value.mime_type,
-            name: value.name,
-            kind: value.kind,
-            payload_type: value.payload_type,
-            has_clock_rate: value.clock_rate.is_some(),
-            clock_rate: value.clock_rate.unwrap_or(0),
-            has_num_channels: value.num_channels.is_some(),
-            num_channels: value.num_channels.unwrap_or(0),
-            has_max_ptime: value.max_ptime.is_some(),
-            max_ptime: value.max_ptime.unwrap_or(0),
-            has_ptime: value.ptime.is_some(),
-            ptime: value.ptime.unwrap_or(0),
-            rtcp_feedback: value.rtcp_feedback.into_iter().map(Into::into).collect(),
-            parameters: into_vec(value.parameters),
-        }
-    }
-}
-
-impl From<RtpCapabilities> for ps_sys::ffi::RtpCapabilities {
-    fn from(value: RtpCapabilities) -> Self {
-        Self {
-            codecs: value.codecs.into_iter().map(Into::into).collect(),
-            header_extensions: value
-                .header_extensions
-                .into_iter()
-                .map(Into::into)
-                .collect(),
-            fec: value.fec.into_iter().map(Into::into).collect(),
-        }
-    }
-}
-
-impl From<RtcpParameters> for ps_sys::ffi::RtcpParameters {
-    fn from(value: RtcpParameters) -> Self {
-        Self {
-            has_ssrc: value.ssrc.is_some(),
-            ssrc: value.ssrc.unwrap_or(0),
-            cname: value.cname,
-            reduced_size: value.reduced_size,
-            mux: value.mux,
-        }
-    }
-}
-
-impl From<RtpParameters> for ps_sys::ffi::RtpParameters {
-    fn from(value: RtpParameters) -> Self {
-        Self {
-            transaction_id: value.transaction_id,
-            mid: value.mid,
-            codecs: value.codecs.into_iter().map(Into::into).collect(),
-            header_extensions: value
-                .header_extensions
-                .into_iter()
-                .map(Into::into)
-                .collect(),
-            encodings: value.encodings.into_iter().map(Into::into).collect(),
-            rtcp: value.rtcp.into(),
-            has_degradation_preference: value.degradation_preference.is_some(),
-            degradation_preference: value
-                .degradation_preference
-                .unwrap_or(DegradationPreference::Balanced),
         }
     }
 }

livekit-webrtc/src/rtp_receiver.rs

@@ -1,67 +1,29 @@
-use crate::media_stream::{MediaStream, MediaStreamTrackHandle};
+use std::fmt::Debug;
-use crate::rtp_parameters::RtpParameters;
-use cxx::SharedPtr;
-use std::fmt::{Debug, Formatter};
-use webrtc_sys::rtp_receiver as sys_rec;
-use webrtc_sys::webrtc as sys_webrtc;
 
-pub use sys_webrtc::ffi::MediaType;
+use crate::{
+    imp::rtp_receiver as imp_rr, media_stream::MediaStreamTrack, rtp_parameters::RtpParameters,
+};
 
 #[derive(Clone)]
 pub struct RtpReceiver {
-    cxx_handle: SharedPtr<sys_rec::ffi::RtpReceiver>,
+    pub(crate) handle: imp_rr::RtpReceiver,
-}
-
-impl Debug for RtpReceiver {
-    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
-        f.debug_struct("RtpReceiver")
-            .field("track", &self.track())
-            .field("media_type", &self.media_type())
-            .field("id", &self.id())
-            .finish()
-    }
 }
 
 impl RtpReceiver {
-    pub(crate) fn new(cxx_handle: SharedPtr<sys_rec::ffi::RtpReceiver>) -> Self {
+    pub fn track(&self) -> Option<MediaStreamTrack> {
-        Self { cxx_handle }
+        self.handle.track()
-    }
-
-    pub(crate) fn cxx_handle(&self) -> SharedPtr<sys_rec::ffi::RtpReceiver> {
-        self.cxx_handle.clone()
-    }
-
-    pub fn track(&self) -> MediaStreamTrackHandle {
-        MediaStreamTrackHandle::new(self.cxx_handle.track())
-    }
-
-    pub fn stream_ids(&self) -> Vec<String> {
-        self.cxx_handle.stream_ids()
-    }
-
-    pub fn streams(&self) -> Vec<MediaStream> {
-        let ptrs = self.cxx_handle.streams();
-        let mut vec = Vec::with_capacity(ptrs.len());
-        for stream in ptrs {
-            vec.push(MediaStream::new(stream.ptr));
-        }
-        vec
-    }
-
-    pub fn media_type(&self) -> MediaType {
-        self.cxx_handle.media_type()
-    }
-
-    pub fn id(&self) -> String {
-        self.cxx_handle.id()
     }
 
     pub fn parameters(&self) -> RtpParameters {
-        self.cxx_handle.get_parameters().into()
+        self.handle.parameters()
     }
+}
 
-    pub fn set_jitter_buffer_minimum_delay(&self, delay_seconds: Option<f64>) {
+impl Debug for RtpReceiver {
-        self.cxx_handle
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-            .set_jitter_buffer_minimum_delay(delay_seconds.is_some(), delay_seconds.unwrap_or(0.0));
+        f.debug_struct("RtpReceiver")
+            .field("track", &self.track())
+            .field("cname", &self.parameters().rtcp.cname)
+            .finish()
     }
 }

livekit-webrtc/src/rtp_sender.rs

@@ -1,81 +1,37 @@
-use crate::media_stream::{MediaStream, MediaStreamTrackHandle};
+use std::fmt::Debug;
-use crate::prelude::*;
-use crate::rtp_parameters::{RtpEncodingParameters, RtpParameters};
-use cxx::SharedPtr;
-use std::fmt::{Debug, Formatter};
-use webrtc_sys::rtp_sender as sys_rs;
-use webrtc_sys::webrtc as sys_webrtc;
 
-pub use sys_webrtc::ffi::MediaType;
+use crate::{
+    imp::rtp_sender as imp_rs, media_stream::MediaStreamTrack, rtp_parameters::RtpParameters,
+    RtcError,
+};
 
 #[derive(Clone)]
 pub struct RtpSender {
-    cxx_handle: SharedPtr<sys_rs::ffi::RtpSender>,
+    pub(crate) handle: imp_rs::RtpSender,
-}
-
-impl Debug for RtpSender {
-    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
-        f.debug_struct("RtpSender")
-            .field("track", &self.track())
-            .field("media_type", &self.media_type())
-            .field("ssrc", &self.ssrc())
-            .field("id", &self.id())
-            .finish()
-    }
 }
 
 impl RtpSender {
-    pub(crate) fn new(cxx_handle: SharedPtr<sys_rs::ffi::RtpSender>) -> Self {
+    pub fn track(&self) -> Option<MediaStreamTrack> {
-        Self { cxx_handle }
+        self.handle.track()
     }
 
-    pub(crate) fn cxx_handle(&self) -> SharedPtr<sys_rs::ffi::RtpSender> {
+    pub fn set_track(&self, track: Option<MediaStreamTrack>) -> Result<(), RtcError> {
-        self.cxx_handle.clone()
+        self.handle.set_track(track)
-    }
-
-    pub fn set_track(&self, track: MediaStreamTrackHandle) -> bool {
-        self.cxx_handle.set_track(track.cxx_handle())
-    }
-
-    pub fn track(&self) -> MediaStreamTrackHandle {
-        MediaStreamTrackHandle::new(self.cxx_handle.track())
-    }
-
-    pub fn ssrc(&self) -> u32 {
-        self.cxx_handle.ssrc()
-    }
-
-    pub fn media_type(&self) -> MediaType {
-        self.cxx_handle.media_type()
-    }
-
-    pub fn id(&self) -> String {
-        self.cxx_handle.id()
-    }
-
-    pub fn stream_ids(&self) -> Vec<String> {
-        self.cxx_handle.stream_ids()
-    }
-
-    pub fn set_streams(&self, stream_ids: &Vec<String>) {
-        self.cxx_handle.set_streams(stream_ids);
-    }
-
-    pub fn init_send_encodings(&self) -> Vec<RtpEncodingParameters> {
-        self.cxx_handle
-            .init_send_encodings()
-            .into_iter()
-            .map(Into::into)
-            .collect()
     }
 
     pub fn parameters(&self) -> RtpParameters {
-        self.cxx_handle.get_parameters().into()
+        self.handle.parameters()
     }
 
-    pub fn set_parameters(&self, params: RtpParameters) -> Result<(), RTCError> {
+    pub fn set_parameters(&self, parameters: RtpParameters) -> Result<(), RtcError> {
-        self.cxx_handle
+        self.handle.set_parameters(parameters)
-            .set_parameters(params.into())
+    }
-            .map_err(|e| unsafe { RTCError::from(e.what()) })
+}
+
+impl Debug for RtpSender {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("RtpReceiver")
+            .field("cname", &self.parameters().rtcp.cname)
+            .finish()
     }
 }

livekit-webrtc/src/rtp_transceiver.rs

@@ -1,119 +1,68 @@
-use crate::prelude::*;
+use crate::imp::rtp_transceiver as imp_rt;
-use cxx::SharedPtr;
+use crate::rtp_parameters::{RtpCodecCapability, RtpEncodingParameters};
-use std::fmt::{Debug, Formatter};
+use crate::rtp_receiver::RtpReceiver;
-use webrtc_sys::rtp_transceiver as sys_rt;
+use crate::rtp_sender::RtpSender;
+use crate::RtcError;
+use std::fmt::Debug;
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct RtpTransceiverInit {
     pub direction: RtpTransceiverDirection,
     pub stream_ids: Vec<String>,
     pub send_encodings: Vec<RtpEncodingParameters>,
 }
 
-impl From<RtpTransceiverInit> for sys_rt::ffi::RtpTransceiverInit {
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-    fn from(value: RtpTransceiverInit) -> Self {
+pub enum RtpTransceiverDirection {
-        Self {
+    SendRecv,
-            direction: value.direction,
+    SendOnly,
-            stream_ids: value.stream_ids,
+    RecvOnly,
-            send_encodings: value.send_encodings.into_iter().map(Into::into).collect()
+    Inactive,
-        }
+    Stopped,
-    }
 }
 
 #[derive(Clone)]
 pub struct RtpTransceiver {
-    cxx_handle: SharedPtr<sys_rt::ffi::RtpTransceiver>,
+    pub(crate) handle: imp_rt::RtpTransceiver,
-}
-
-impl Debug for RtpTransceiver {
-    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
-        f.debug_struct("RtpTransceiver")
-            .field("media_type", &self.media_type())
-            .field("mid", &self.mid())
-            .field("direction", &self.direction())
-            .field("stopped", &self.stopped())
-            .field("stopping", &self.stopping())
-            .finish()
-    }
 }
 
 impl RtpTransceiver {
-    pub(crate) fn new(cxx_handle: SharedPtr<sys_rt::ffi::RtpTransceiver>) -> Self {
-        Self { cxx_handle }
-    }
-
-    pub(crate) fn cxx_handle(&self) -> SharedPtr<sys_rt::ffi::RtpTransceiver> {
-        self.cxx_handle.clone()
-    }
-
-    pub fn media_type(&self) -> MediaType {
-        self.cxx_handle.media_type()
-    }
-
     pub fn mid(&self) -> Option<String> {
-        self.cxx_handle.mid().ok()
+        self.handle.mid()
-    }
-
-    pub fn sender(&self) -> RtpSender {
-        RtpSender::new(self.cxx_handle.sender())
-    }
-
-    pub fn receiver(&self) -> RtpReceiver {
-        RtpReceiver::new(self.cxx_handle.receiver())
-    }
-
-    pub fn stopped(&self) -> bool {
-        self.cxx_handle.stopped()
-    }
-
-    pub fn stopping(&self) -> bool {
-        self.cxx_handle.stopping()
-    }
-
-    pub fn direction(&self) -> RtpTransceiverDirection {
-        self.cxx_handle.direction()
-    }
-
-    pub fn set_direction(&self, direction: RtpTransceiverDirection) -> Result<(), RTCError> {
-        self.cxx_handle.set_direction(direction)
-            .map_err(|e| unsafe { RTCError::from(e.what()) })
     }
 
     pub fn current_direction(&self) -> Option<RtpTransceiverDirection> {
-        self.cxx_handle.current_direction().ok()
+        self.handle.current_direction()
     }
 
-    pub fn fired_direction(&self) -> Option<RtpTransceiverDirection> {
+    pub fn direction(&self) -> RtpTransceiverDirection {
-        self.cxx_handle.fired_direction().ok()
+        self.handle.direction()
     }
 
-    pub fn stop_standard(&self) -> Result<(), RTCError> {
+    pub fn sender(&self) -> RtpSender {
-            self.cxx_handle.stop_standard()
+        self.handle.sender()
-            .map_err(|e| unsafe { RTCError::from(e.what()) })
     }
 
-    pub fn set_codec_preferences(&self, codecs: Vec<RtpCodecCapability>) -> Result<(), RTCError> {
+    pub fn receiver(&self) -> RtpReceiver {
-        let ffi_codecs = codecs.into_iter().map(Into::into).collect();
+        self.handle.receiver()
-        self.cxx_handle.set_codec_preferences(ffi_codecs)
-            .map_err(|e| unsafe { RTCError::from(e.what()) })
     }
 
-    pub fn codec_preferences(&self) -> Vec<RtpCodecCapability> {
+    pub fn set_codec_preferences(&self, codecs: Vec<RtpCodecCapability>) -> Result<(), RtcError> {
-        self.cxx_handle.codec_preferences().into_iter().map(Into::into).collect()
+        self.handle.set_codec_preferences(codecs)
     }
 
-    pub fn header_extensions_to_offer(&self) -> Vec<RtpHeaderExtensionCapability> {
+    pub fn stop(&self) -> Result<(), RtcError> {
-        self.cxx_handle.header_extensions_to_offer().into_iter().map(Into::into).collect()
+        self.handle.stop()
+    }
+}
+
+impl Debug for RtpTransceiver {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("RtpTransceiver")
+            .field("mid", &self.mid())
+            .field("direction", &self.direction())
+            .field("sender", &self.sender())
+            .field("receiver", &self.receiver())
+            .finish()
     }
-
-    pub fn header_extensions_negotiated(&self) -> Vec<RtpHeaderExtensionCapability> {
-        self.cxx_handle.header_extensions_negotiated().into_iter().map(Into::into).collect()
-    }
-
-    pub fn set_offered_rtp_header_extensions(&self, headers: Vec<RtpHeaderExtensionCapability>) -> Result<(), RTCError> {   
-        let ffi_headers = headers.into_iter().map(Into::into).collect();
-        self.cxx_handle.set_offered_rtp_header_extensions(ffi_headers)
-            .map_err(|e| unsafe { RTCError::from(e.what()) })
-    }
-  
 }

livekit-webrtc/src/session_description.rs

@@ -0,0 +1,61 @@
+use crate::imp::session_description as sd_imp;
+use std::{fmt::Debug, str::FromStr};
+use thiserror::Error;
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub enum SdpType {
+    Offer,
+    PrAnswer,
+    Answer,
+    Rollback,
+}
+
+impl FromStr for SdpType {
+    type Err = &'static str;
+
+    fn from_str(sdp_type: &str) -> Result<Self, Self::Err> {
+        match sdp_type {
+            "offer" => Ok(Self::Offer),
+            "pranswer" => Ok(Self::PrAnswer),
+            "answer" => Ok(Self::Answer),
+            "rollback" => Ok(Self::Rollback),
+            _ => Err("invalid SdpType"),
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct SessionDescription {
+    pub(crate) handle: sd_imp::SessionDescription,
+}
+
+#[derive(Clone, Error, Debug)]
+#[error("Failed to parse sdp: {line} - {description}")]
+pub struct SdpParseError {
+    pub line: String,
+    pub description: String,
+}
+
+impl SessionDescription {
+    pub fn parse(sdp: &str, sdp_type: SdpType) -> Result<Self, SdpParseError> {
+        sd_imp::SessionDescription::parse(sdp, sdp_type)
+    }
+
+    pub fn sdp_type(&self) -> SdpType {
+        self.handle.sdp_type()
+    }
+}
+
+impl ToString for SessionDescription {
+    fn to_string(&self) -> String {
+        self.handle.to_string()
+    }
+}
+
+impl Debug for SessionDescription {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("SessionDescription")
+            .field("sdp_type", &self.sdp_type())
+            .finish()
+    }
+}

livekit-webrtc/src/video_frame.rs

@@ -1,9 +1,14 @@
-use crate::video_frame_buffer::VideoFrameBuffer;
+use crate::imp::video_frame as vf_imp;
-use cxx::UniquePtr;
+use std::fmt::Debug;
-use std::fmt::{Debug, Formatter};
+use thiserror::Error;
-use webrtc_sys::video_frame as vf_sys;
 
-#[derive(Debug)]
+#[derive(Debug, Error)]
+pub enum SinkError {
+    #[error("platform error: {0}")]
+    Platform(String),
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
 pub enum VideoRotation {
     VideoRotation0 = 0,
     VideoRotation90 = 90,
@@ -11,141 +16,479 @@ pub enum VideoRotation {
     VideoRotation270 = 270,
 }
 
-impl From<vf_sys::ffi::VideoRotation> for VideoRotation {
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-    fn from(rotation: vf_sys::ffi::VideoRotation) -> Self {
+pub enum VideoFormatType {
-        match rotation {
+    ARGB,
-            vf_sys::ffi::VideoRotation::VideoRotation0 => Self::VideoRotation0,
+    BGRA,
-            vf_sys::ffi::VideoRotation::VideoRotation90 => Self::VideoRotation90,
+    ABGR,
-            vf_sys::ffi::VideoRotation::VideoRotation180 => Self::VideoRotation180,
+    RGBA,
-            vf_sys::ffi::VideoRotation::VideoRotation270 => Self::VideoRotation270,
-            _ => unreachable!(),
-        }
-    }
 }
 
-impl From<VideoRotation> for vf_sys::ffi::VideoRotation {
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-    fn from(rotation: VideoRotation) -> Self {
+#[non_exhaustive]
-        match rotation {
+pub enum VideoFrameBufferType {
-            VideoRotation::VideoRotation0 => Self::VideoRotation0,
+    Native,
-            VideoRotation::VideoRotation90 => Self::VideoRotation90,
+    I420,
-            VideoRotation::VideoRotation180 => Self::VideoRotation180,
+    I420A,
-            VideoRotation::VideoRotation270 => Self::VideoRotation270,
+    I422,
-        }
+    I444,
-    }
+    I010,
+    NV12,
+    WebGl,
 }
 
-pub struct VideoFrame {
+#[derive(Debug)]
-    cxx_handle: UniquePtr<vf_sys::ffi::VideoFrame>,
+pub struct VideoFrame<T>
+where
+    T: VideoFrameBuffer,
+{
+    pub rotation: VideoRotation,
+    pub timestamp: i64, // When the frame was captured
+    pub buffer: T,
 }
 
-impl Debug for VideoFrame {
+pub type BoxVideoFrame = VideoFrame<Box<dyn VideoFrameBuffer + Send + Sync>>;
-    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
+
-        f.debug_struct("VideoFrame")
+macro_rules! new_buffer_type {
+    ($type:ident, $variant:ident, $as:ident) => {
+        pub struct $type {
+            pub(crate) handle: vf_imp::$type,
+        }
+
+        impl $crate::video_frame::internal::BufferInternal for $type {
+            #[cfg(not(target_arch = "wasm32"))]
+            fn sys_handle(&self) -> &webrtc_sys::video_frame_buffer::ffi::VideoFrameBuffer {
+                self.handle.sys_handle()
+            }
+
+            #[cfg(not(target_arch = "wasm32"))]
+            fn to_i420(&self) -> I420Buffer {
+                I420Buffer {
+                    handle: self.handle.to_i420(),
+                }
+            }
+
+            #[cfg(not(target_arch = "wasm32"))]
+            fn to_argb(
+                &self,
+                format: VideoFormatType,
+                dst: &mut [u8],
+                stride: i32,
+                width: i32,
+                height: i32,
+            ) -> Result<(), $crate::video_frame::native::ConvertError> {
+                self.handle.to_argb(format, dst, stride, width, height)
+            }
+        }
+
+        impl VideoFrameBuffer for $type {
+            fn width(&self) -> i32 {
+                self.handle.width()
+            }
+
+            fn height(&self) -> i32 {
+                self.handle.height()
+            }
+
+            fn buffer_type(&self) -> VideoFrameBufferType {
+                VideoFrameBufferType::$variant
+            }
+
+            fn $as(&self) -> Option<&$type> {
+                Some(self)
+            }
+        }
+
+        impl Debug for $type {
+            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                f.debug_struct(stringify!($type))
                     .field("width", &self.width())
                     .field("height", &self.height())
-            .field("id", &self.id())
-            .field("rotation", &self.rotation())
-            .field("timestamp", &self.timestamp())
                     .finish()
             }
+        }
+    };
 }
 
-impl VideoFrame {
+pub(crate) mod internal {
-    pub(crate) fn new(cxx_handle: UniquePtr<vf_sys::ffi::VideoFrame>) -> Self {
+    use super::{I420Buffer, VideoFormatType};
-        Self { cxx_handle }
-    }
 
-    pub fn width(&self) -> i32 {
+    pub trait BufferInternal {
-        self.cxx_handle.width()
+        #[cfg(not(target_arch = "wasm32"))]
-    }
+        fn sys_handle(&self) -> &webrtc_sys::video_frame_buffer::ffi::VideoFrameBuffer;
 
-    pub fn height(&self) -> i32 {
+        #[cfg(not(target_arch = "wasm32"))]
-        self.cxx_handle.height()
+        fn to_i420(&self) -> I420Buffer;
-    }
 
-    pub fn size(&self) -> u32 {
+        #[cfg(not(target_arch = "wasm32"))]
-        self.cxx_handle.size()
+        fn to_argb(
-    }
+            &self,
-
+            format: VideoFormatType,
-    pub fn id(&self) -> u16 {
+            dst: &mut [u8],
-        self.cxx_handle.id()
+            dst_stride: i32,
-    }
+            dst_width: i32,
-
+            dst_height: i32,
-    pub fn timestamp_us(&self) -> i64 {
+        ) -> Result<(), super::native::ConvertError>;
-        self.cxx_handle.timestamp_us()
-    }
-
-    pub fn ntp_time_ms(&self) -> i64 {
-        self.cxx_handle.ntp_time_ms()
-    }
-
-    pub fn transport_frame_id(&self) -> u32 {
-        self.cxx_handle.transport_frame_id()
-    }
-
-    pub fn timestamp(&self) -> u32 {
-        self.cxx_handle.timestamp()
-    }
-
-    pub fn rotation(&self) -> VideoRotation {
-        self.cxx_handle.rotation().into()
-    }
-
-    /// # Safety
-    /// Must be called only once, this function create the safe Rust
-    /// wrapper around a VideoFrameBuffer.
-    /// Only one wrapper musts exist at a time.
-    pub(crate) unsafe fn video_frame_buffer(&self) -> VideoFrameBuffer {
-        VideoFrameBuffer::new(self.cxx_handle.video_frame_buffer())
-    }
-
-    pub fn builder() -> VideoFrameBuilder {
-        VideoFrameBuilder::default()
     }
 }
 
-pub struct VideoFrameBuilder {
+pub trait VideoFrameBuffer: internal::BufferInternal + Debug {
-    cxx_handle: UniquePtr<vf_sys::ffi::VideoFrameBuilder>,
+    fn width(&self) -> i32;
-}
+    fn height(&self) -> i32;
+    fn buffer_type(&self) -> VideoFrameBufferType;
 
-impl Debug for VideoFrameBuilder {
+    #[cfg(not(target_arch = "wasm32"))]
-    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
+    fn as_native(&self) -> Option<&native::NativeBuffer> {
-        f.debug_struct("VideoFrameBuilder").finish()
+        None
+    }
+
+    fn as_i420(&self) -> Option<&I420Buffer> {
+        None
+    }
+
+    fn as_i420a(&self) -> Option<&I420ABuffer> {
+        None
+    }
+
+    fn as_i422(&self) -> Option<&I422Buffer> {
+        None
+    }
+
+    fn as_i444(&self) -> Option<&I444Buffer> {
+        None
+    }
+
+    fn as_i010(&self) -> Option<&I010Buffer> {
+        None
+    }
+
+    fn as_nv12(&self) -> Option<&NV12Buffer> {
+        None
     }
 }
 
-impl Default for VideoFrameBuilder {
+new_buffer_type!(I420Buffer, I420, as_i420);
-    fn default() -> Self {
+new_buffer_type!(I420ABuffer, I420A, as_i420a);
-        Self {
+new_buffer_type!(I422Buffer, I422, as_i422);
-            cxx_handle: vf_sys::ffi::create_video_frame_builder(),
+new_buffer_type!(I444Buffer, I444, as_i444);
+new_buffer_type!(I010Buffer, I010, as_i010);
+new_buffer_type!(NV12Buffer, NV12, as_nv12);
+
+impl I420Buffer {
+    pub fn chroma_width(&self) -> i32 {
+        self.handle.chroma_width()
+    }
+
+    pub fn chroma_height(&self) -> i32 {
+        self.handle.chroma_height()
+    }
+
+    pub fn stride_y(&self) -> i32 {
+        self.handle.stride_y()
+    }
+
+    pub fn stride_u(&self) -> i32 {
+        self.handle.stride_u()
+    }
+
+    pub fn stride_v(&self) -> i32 {
+        self.handle.stride_v()
+    }
+
+    pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
+        self.handle.data()
+    }
+
+    pub fn data_mut(&mut self) -> (&mut [u8], &mut [u8], &mut [u8]) {
+        let (data_y, data_u, data_v) = self.handle.data();
+        unsafe {
+            (
+                std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()),
+                std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u8, data_u.len()),
+                std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u8, data_v.len()),
+            )
         }
     }
 }
 
-impl VideoFrameBuilder {
+impl I420ABuffer {
-    pub fn set_video_frame_buffer(mut self, buffer: VideoFrameBuffer) -> Self {
+    pub fn chroma_width(&self) -> i32 {
-        self.cxx_handle
+        self.handle.chroma_width()
-            .pin_mut()
-            .set_video_frame_buffer(buffer.release());
-        self
     }
 
-    pub fn set_timestamp_us(mut self, ts_us: i64) -> Self {
+    pub fn chroma_height(&self) -> i32 {
-        self.cxx_handle.pin_mut().set_timestamp_us(ts_us);
+        self.handle.chroma_height()
-        self
     }
 
-    pub fn set_rotation(mut self, rotation: VideoRotation) -> Self {
+    pub fn stride_y(&self) -> i32 {
-        self.cxx_handle.pin_mut().set_rotation(rotation.into());
+        self.handle.stride_y()
-        self
     }
 
-    pub fn set_id(mut self, id: u16) -> Self {
+    pub fn stride_u(&self) -> i32 {
-        self.cxx_handle.pin_mut().set_id(id);
+        self.handle.stride_u()
-        self
     }
 
-    pub fn build(mut self) -> VideoFrame {
+    pub fn stride_v(&self) -> i32 {
-        VideoFrame::new(self.cxx_handle.pin_mut().build())
+        self.handle.stride_v()
+    }
+
+    pub fn stride_a(&self) -> i32 {
+        self.handle.stride_a()
+    }
+
+    pub fn data(&self) -> (&[u8], &[u8], &[u8], Option<&[u8]>) {
+        self.handle.data()
+    }
+
+    pub fn data_mut(&self) -> (&mut [u8], &mut [u8], &mut [u8], Option<&mut [u8]>) {
+        let (data_y, data_u, data_v, data_a) = self.handle.data();
+        unsafe {
+            (
+                std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()),
+                std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u8, data_u.len()),
+                std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u8, data_v.len()),
+                data_a.map(|data_a| {
+                    std::slice::from_raw_parts_mut(data_a.as_ptr() as *mut u8, data_a.len())
+                }),
+            )
+        }
     }
 }
+
+impl I422Buffer {
+    pub fn chroma_width(&self) -> i32 {
+        self.handle.chroma_width()
+    }
+
+    pub fn chroma_height(&self) -> i32 {
+        self.handle.chroma_height()
+    }
+
+    pub fn stride_y(&self) -> i32 {
+        self.handle.stride_y()
+    }
+
+    pub fn stride_u(&self) -> i32 {
+        self.handle.stride_u()
+    }
+
+    pub fn stride_v(&self) -> i32 {
+        self.handle.stride_v()
+    }
+
+    pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
+        self.handle.data()
+    }
+
+    pub fn data_mut(&mut self) -> (&mut [u8], &mut [u8], &mut [u8]) {
+        let (data_y, data_u, data_v) = self.handle.data();
+        unsafe {
+            (
+                std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()),
+                std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u8, data_u.len()),
+                std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u8, data_v.len()),
+            )
+        }
+    }
+}
+
+impl I444Buffer {
+    pub fn chroma_width(&self) -> i32 {
+        self.handle.chroma_width()
+    }
+
+    pub fn chroma_height(&self) -> i32 {
+        self.handle.chroma_height()
+    }
+
+    pub fn stride_y(&self) -> i32 {
+        self.handle.stride_y()
+    }
+
+    pub fn stride_u(&self) -> i32 {
+        self.handle.stride_u()
+    }
+
+    pub fn stride_v(&self) -> i32 {
+        self.handle.stride_v()
+    }
+
+    pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
+        self.handle.data()
+    }
+
+    pub fn data_mut(&mut self) -> (&mut [u8], &mut [u8], &mut [u8]) {
+        let (data_y, data_u, data_v) = self.handle.data();
+        unsafe {
+            (
+                std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()),
+                std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u8, data_u.len()),
+                std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u8, data_v.len()),
+            )
+        }
+    }
+}
+
+impl I010Buffer {
+    pub fn chroma_width(&self) -> i32 {
+        self.handle.chroma_width()
+    }
+
+    pub fn chroma_height(&self) -> i32 {
+        self.handle.chroma_height()
+    }
+
+    pub fn stride_y(&self) -> i32 {
+        self.handle.stride_y()
+    }
+
+    pub fn stride_u(&self) -> i32 {
+        self.handle.stride_u()
+    }
+
+    pub fn stride_v(&self) -> i32 {
+        self.handle.stride_v()
+    }
+
+    pub fn data(&self) -> (&[u16], &[u16], &[u16]) {
+        self.handle.data()
+    }
+
+    pub fn data_mut(&mut self) -> (&mut [u16], &mut [u16], &mut [u16]) {
+        let (data_y, data_u, data_v) = self.handle.data();
+        unsafe {
+            (
+                std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u16, data_y.len()),
+                std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u16, data_u.len()),
+                std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u16, data_v.len()),
+            )
+        }
+    }
+}
+
+impl NV12Buffer {
+    pub fn chroma_width(&self) -> i32 {
+        self.handle.chroma_width()
+    }
+
+    pub fn chroma_height(&self) -> i32 {
+        self.handle.chroma_height()
+    }
+
+    pub fn stride_y(&self) -> i32 {
+        self.handle.stride_y()
+    }
+
+    pub fn stride_uv(&self) -> i32 {
+        self.handle.stride_uv()
+    }
+
+    pub fn data(&self) -> (&[u8], &[u8]) {
+        self.handle.data()
+    }
+
+    pub fn data_mut(&mut self) -> (&mut [u8], &mut [u8]) {
+        let (data_y, data_uv) = self.handle.data();
+        unsafe {
+            (
+                std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()),
+                std::slice::from_raw_parts_mut(data_uv.as_ptr() as *mut u8, data_uv.len()),
+            )
+        }
+    }
+}
+
+#[cfg(not(target_arch = "wasm32"))]
+pub mod native {
+    use super::{vf_imp, I420Buffer, VideoFormatType, VideoFrameBuffer, VideoFrameBufferType};
+    use std::fmt::Debug;
+
+    pub use crate::imp::yuv_helper::ConvertError;
+
+    new_buffer_type!(NativeBuffer, Native, as_native);
+
+    pub trait I420BufferExt {
+        fn new(width: u32, height: u32) -> I420Buffer;
+    }
+
+    impl I420BufferExt for I420Buffer {
+        fn new(width: u32, height: u32) -> I420Buffer {
+            vf_imp::I420Buffer::new(width, height)
+        }
+    }
+
+    pub trait VideoFrameBufferExt: VideoFrameBuffer {
+        fn to_i420(&self) -> I420Buffer;
+        fn to_argb(
+            &self,
+            format: VideoFormatType,
+            dst: &mut [u8],
+            dst_stride: i32,
+            dst_width: i32,
+            dst_height: i32,
+        ) -> Result<(), ConvertError>;
+    }
+
+    impl<T: VideoFrameBuffer> VideoFrameBufferExt for T {
+        fn to_i420(&self) -> I420Buffer {
+            self.to_i420()
+        }
+
+        fn to_argb(
+            &self,
+            format: VideoFormatType,
+            dst: &mut [u8],
+            dst_stride: i32,
+            dst_width: i32,
+            dst_height: i32,
+        ) -> Result<(), ConvertError> {
+            self.to_argb(format, dst, dst_stride, dst_width, dst_height)
+        }
+    }
+}
+
+impl<T: VideoFrameBuffer + ?Sized> internal::BufferInternal for Box<T> {
+    fn sys_handle(&self) -> &webrtc_sys::video_frame_buffer::ffi::VideoFrameBuffer {
+        self.as_ref().sys_handle()
+    }
+
+    fn to_i420(&self) -> I420Buffer {
+        self.as_ref().to_i420()
+    }
+
+    fn to_argb(
+        &self,
+        format: VideoFormatType,
+        dst: &mut [u8],
+        dst_stride: i32,
+        dst_width: i32,
+        dst_height: i32,
+    ) -> Result<(), self::native::ConvertError> {
+        self.as_ref()
+            .to_argb(format, dst, dst_stride, dst_width, dst_height)
+    }
+}
+
+impl<T: VideoFrameBuffer + ?Sized> VideoFrameBuffer for Box<T> {
+    fn width(&self) -> i32 {
+        self.as_ref().width()
+    }
+
+    fn height(&self) -> i32 {
+        self.as_ref().height()
+    }
+
+    fn buffer_type(&self) -> VideoFrameBufferType {
+        self.as_ref().buffer_type()
+    }
+}
+
+#[cfg(target_arch = "wasm32")]
+pub mod web {
+    use super::VideoFrameBuffer;
+
+    #[derive(Debug)]
+    pub struct WebGlBuffer {}
+
+    impl VideoFrameBuffer for WebGlBuffer {}
+}

livekit-webrtc/src/video_frame_buffer.rs

@@ -1,578 +0,0 @@
-use cxx::UniquePtr;
-use livekit_utils::enum_dispatch;
-use std::pin::Pin;
-use std::slice;
-use webrtc_sys::video_frame_buffer as vfb_sys;
-
-use crate::yuv_helper::{self, ConvertError};
-
-macro_rules! recursive_cast {
-    ($ptr:expr $(, $fnc:ident)*) => {
-        {
-            let ptr = $ptr;
-            $(
-                let ptr = unsafe { vfb_sys::ffi::$fnc(ptr) };
-            )*
-            ptr
-        }
-    };
-}
-
-#[derive(Debug)]
-pub enum VideoFrameBufferType {
-    Native,
-    I420,
-    I420A,
-    I422,
-    I444,
-    I010,
-    NV12,
-}
-
-// types to convert to
-#[derive(Debug)]
-pub enum VideoFormatType {
-    ARGB,
-    BGRA,
-    ABGR,
-    RGBA,
-}
-
-impl From<vfb_sys::ffi::VideoFrameBufferType> for VideoFrameBufferType {
-    fn from(buffer_type: vfb_sys::ffi::VideoFrameBufferType) -> Self {
-        match buffer_type {
-            vfb_sys::ffi::VideoFrameBufferType::Native => Self::Native,
-            vfb_sys::ffi::VideoFrameBufferType::I420 => Self::I420,
-            vfb_sys::ffi::VideoFrameBufferType::I420A => Self::I420A,
-            vfb_sys::ffi::VideoFrameBufferType::I422 => Self::I422,
-            vfb_sys::ffi::VideoFrameBufferType::I444 => Self::I444,
-            vfb_sys::ffi::VideoFrameBufferType::I010 => Self::I010,
-            vfb_sys::ffi::VideoFrameBufferType::NV12 => Self::NV12,
-            _ => unreachable!(),
-        }
-    }
-}
-
-pub trait VideoFrameBufferTrait {
-    fn buffer_type(&self) -> VideoFrameBufferType; // Useful for the FFI
-    fn width(&self) -> i32;
-    fn height(&self) -> i32;
-    fn to_i420(self) -> I420Buffer;
-}
-
-pub trait PlanarYuvBuffer: VideoFrameBufferTrait {
-    fn chroma_width(&self) -> i32;
-    fn chroma_height(&self) -> i32;
-    fn stride_y(&self) -> i32;
-    fn stride_u(&self) -> i32;
-    fn stride_v(&self) -> i32;
-}
-
-pub trait PlanarYuv8Buffer: PlanarYuvBuffer {
-    fn data_y(&self) -> &[u8];
-    fn data_u(&self) -> &[u8];
-    fn data_v(&self) -> &[u8];
-}
-
-pub trait PlanarYuv16BBuffer: PlanarYuvBuffer {
-    fn data_y(&self) -> &[u16];
-    fn data_u(&self) -> &[u16];
-    fn data_v(&self) -> &[u16];
-}
-
-pub trait BiplanarYuvBuffer: VideoFrameBufferTrait {
-    fn chroma_width(&self) -> i32;
-    fn chroma_height(&self) -> i32;
-    fn stride_y(&self) -> i32;
-    fn stride_uv(&self) -> i32;
-}
-
-pub trait BiplanarYuv8Buffer: BiplanarYuvBuffer {
-    fn data_y(&self) -> &[u8];
-    fn data_uv(&self) -> &[u8];
-}
-
-pub enum VideoFrameBuffer {
-    Native(NativeBuffer),
-    I420(I420Buffer),
-    I420A(I420ABuffer),
-    I422(I422Buffer),
-    I444(I444Buffer),
-    I010(I010Buffer),
-    NV12(NV12Buffer),
-}
-
-impl VideoFrameBuffer {
-    pub(crate) fn new(mut cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>) -> Self {
-        unsafe {
-            match cxx_handle.buffer_type().into() {
-                VideoFrameBufferType::Native => Self::Native(NativeBuffer::from(cxx_handle)),
-                VideoFrameBufferType::I420 => {
-                    Self::I420(I420Buffer::from(cxx_handle.pin_mut().get_i420()))
-                }
-                VideoFrameBufferType::I420A => {
-                    Self::I420A(I420ABuffer::from(cxx_handle.pin_mut().get_i420a()))
-                }
-                VideoFrameBufferType::I422 => {
-                    Self::I422(I422Buffer::from(cxx_handle.pin_mut().get_i422()))
-                }
-                VideoFrameBufferType::I444 => {
-                    Self::I444(I444Buffer::from(cxx_handle.pin_mut().get_i444()))
-                }
-                VideoFrameBufferType::I010 => {
-                    Self::I010(I010Buffer::from(cxx_handle.pin_mut().get_i010()))
-                }
-                VideoFrameBufferType::NV12 => {
-                    Self::NV12(NV12Buffer::from(cxx_handle.pin_mut().get_nv12()))
-                }
-            }
-        }
-    }
-
-    #[allow(unused_unsafe)]
-    pub(crate) fn release(self) -> UniquePtr<vfb_sys::ffi::VideoFrameBuffer> {
-        unsafe {
-            match self {
-                VideoFrameBuffer::Native(native) => native.release(),
-                VideoFrameBuffer::I420(i420) => UniquePtr::from_raw(recursive_cast!(
-                    i420.release().into_raw(),
-                    i420_to_yuv8,
-                    yuv8_to_yuv,
-                    yuv_to_vfb
-                ) as *mut _),
-                VideoFrameBuffer::I420A(i420a) => UniquePtr::from_raw(recursive_cast!(
-                    i420a.release().into_raw(),
-                    i420a_to_yuv8,
-                    yuv8_to_yuv,
-                    yuv_to_vfb
-                ) as *mut _),
-                VideoFrameBuffer::I422(i422) => UniquePtr::from_raw(recursive_cast!(
-                    i422.release().into_raw(),
-                    i422_to_yuv8,
-                    yuv8_to_yuv,
-                    yuv_to_vfb
-                ) as *mut _),
-                VideoFrameBuffer::I444(i444) => UniquePtr::from_raw(recursive_cast!(
-                    i444.release().into_raw(),
-                    i444_to_yuv8,
-                    yuv8_to_yuv,
-                    yuv_to_vfb
-                ) as *mut _),
-                VideoFrameBuffer::I010(i010) => UniquePtr::from_raw(recursive_cast!(
-                    i010.release().into_raw(),
-                    i010_to_yuv16b,
-                    yuv16b_to_yuv,
-                    yuv_to_vfb
-                ) as *mut _),
-                VideoFrameBuffer::NV12(nv12) => UniquePtr::from_raw(recursive_cast!(
-                    nv12.release().into_raw(),
-                    nv12_to_biyuv8,
-                    biyuv8_to_biyuv,
-                    biyuv_to_vfb
-                ) as *mut _),
-            }
-        }
-    }
-
-    pub fn to_argb(
-        &self,
-        format: VideoFormatType,
-        dst: &mut [u8],
-        dst_stride: i32,
-        dst_width: i32,
-        dst_height: i32,
-    ) -> Result<(), ConvertError> {
-        match self {
-            Self::I420(i420) => match format {
-                VideoFormatType::ARGB => yuv_helper::i420_to_argb(
-                    i420.data_y(),
-                    i420.stride_y(),
-                    i420.data_u(),
-                    i420.stride_u(),
-                    i420.data_v(),
-                    i420.stride_v(),
-                    dst,
-                    dst_stride,
-                    dst_width,
-                    dst_height,
-                )?,
-                VideoFormatType::BGRA => yuv_helper::i420_to_bgra(
-                    i420.data_y(),
-                    i420.stride_y(),
-                    i420.data_u(),
-                    i420.stride_u(),
-                    i420.data_v(),
-                    i420.stride_v(),
-                    dst,
-                    dst_stride,
-                    dst_width,
-                    dst_height,
-                )?,
-                VideoFormatType::ABGR => yuv_helper::i420_to_abgr(
-                    i420.data_y(),
-                    i420.stride_y(),
-                    i420.data_u(),
-                    i420.stride_u(),
-                    i420.data_v(),
-                    i420.stride_v(),
-                    dst,
-                    dst_stride,
-                    dst_width,
-                    dst_height,
-                )?,
-                VideoFormatType::RGBA => yuv_helper::i420_to_rgba(
-                    i420.data_y(),
-                    i420.stride_y(),
-                    i420.data_u(),
-                    i420.stride_u(),
-                    i420.data_v(),
-                    i420.stride_v(),
-                    dst,
-                    dst_stride,
-                    dst_width,
-                    dst_height,
-                )?,
-            },
-            _ => {
-                // TODO(theomonnom): Support other buffer types
-            }
-        };
-
-        Ok(())
-    }
-}
-
-impl VideoFrameBufferTrait for VideoFrameBuffer {
-    enum_dispatch!(
-        [Native, I420, I420A, I422, I444, I010, NV12]
-        fnc!(buffer_type, &Self, [], VideoFrameBufferType);
-        fnc!(width, &Self, [], i32);
-        fnc!(height, &Self, [], i32);
-        fnc!(to_i420, Self, [], I420Buffer);
-    );
-}
-
-macro_rules! impl_video_frame_buffer {
-    ($x:ty $(, $cast:ident)*) => {
-
-        // Allow unused_unsafe when we don't do any cast ( e.g. NativeBuffer )
-        #[allow(unused_unsafe)]
-        impl VideoFrameBufferTrait for $x {
-            fn buffer_type(&self) -> VideoFrameBufferType {
-                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
-                unsafe {
-                    (*ptr).buffer_type().into()
-                }
-            }
-
-            fn width(&self) -> i32 {
-                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
-                unsafe {
-                    (*ptr).width()
-                }
-            }
-
-            fn height(&self) -> i32 {
-                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
-                unsafe {
-                    (*ptr).height()
-                }
-            }
-
-            // Require ownership because libwebrtc uses the same pointers
-            fn to_i420(self) -> I420Buffer {
-                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*)
-                    as *const vfb_sys::ffi::VideoFrameBuffer
-                    as *mut vfb_sys::ffi::VideoFrameBuffer;
-
-                unsafe {
-                    I420Buffer::from(Pin::new_unchecked(&mut *ptr).to_i420())
-                }
-            }
-        }
-    };
-}
-
-macro_rules! impl_yuv_buffer {
-    ($x:ty $(, $cast:ident)*) => {
-        impl PlanarYuvBuffer for $x {
-            fn chroma_width(&self) -> i32 {
-                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
-                unsafe {
-                    (*ptr).chroma_width()
-                }
-            }
-
-            fn chroma_height(&self) -> i32 {
-                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
-                unsafe {
-                    (*ptr).chroma_height()
-                }
-            }
-
-            fn stride_y(&self) -> i32 {
-                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
-                unsafe {
-                    (*ptr).stride_y()
-                }
-            }
-
-            fn stride_u(&self) -> i32 {
-                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
-                unsafe {
-                    (*ptr).stride_u()
-                }
-            }
-
-            fn stride_v(&self) -> i32 {
-                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
-                unsafe {
-                    (*ptr).stride_v()
-                }
-            }
-        }
-    };
-}
-
-macro_rules! impl_yuv8_buffer {
-    ($x:ty $(, $cast:ident)*) => {
-        impl PlanarYuv8Buffer for $x {
-            fn data_y(&self) -> &[u8] {
-                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
-                unsafe {
-                    slice::from_raw_parts((*ptr).data_y(), (self.width() * self.height()) as usize)
-                }
-            }
-
-            fn data_u(&self) -> &[u8] {
-                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
-                unsafe {
-                    let chroma_height = (self.height() + 1) / 2;
-                    slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * chroma_height) as usize)
-                }
-            }
-
-            fn data_v(&self) -> &[u8] {
-                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
-                unsafe {
-                    let chroma_height = (self.height() + 1) / 2;
-                    slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * chroma_height) as usize)
-                }
-            }
-        }
-    };
-}
-
-macro_rules! impl_yuv16_buffer {
-    ($x:ty $(, $cast:ident)*) => {
-        impl PlanarYuv16BBuffer for $x {
-            fn data_y(&self) -> &[u16] {
-                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
-                unsafe {
-                    slice::from_raw_parts((*ptr).data_y(), (self.width() * self.height()) as usize)
-                }
-            }
-
-            fn data_u(&self) -> &[u16] {
-                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
-                unsafe {
-                    let chroma_height = (self.height() + 1) / 2;
-                    slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * chroma_height) as usize)
-                }
-            }
-
-            fn data_v(&self) -> &[u16] {
-                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
-                unsafe {
-                    let chroma_height = (self.height() + 1) / 2;
-                    slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * chroma_height) as usize)
-                }
-            }
-        }
-    };
-}
-
-macro_rules! impl_biyuv_buffer {
-    ($x:ty $(, $cast:ident)*) => {
-        impl BiplanarYuvBuffer for $x {
-            fn chroma_width(&self) -> i32 {
-                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
-                unsafe {
-                    (*ptr).chroma_width()
-                }
-            }
-
-            fn chroma_height(&self) -> i32 {
-                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
-                unsafe {
-                    (*ptr).chroma_height()
-                }
-            }
-
-            fn stride_y(&self) -> i32 {
-                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
-                unsafe {
-                    (*ptr).stride_y()
-                }
-            }
-
-            fn stride_uv(&self) -> i32 {
-                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
-                unsafe {
-                    (*ptr).stride_uv()
-                }
-            }
-        }
-    };
-}
-
-macro_rules! impl_biyuv8_buffer {
-    ($x:ty $(, $cast:ident)*) => {
-        impl BiplanarYuv8Buffer for $x {
-            fn data_y(&self) -> &[u8] {
-                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
-                unsafe {
-                    slice::from_raw_parts((*ptr).data_y(), (self.width() * self.height()) as usize)
-                }
-            }
-
-            fn data_uv(&self) -> &[u8] {
-                let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
-                unsafe {
-                    let chroma_height = (self.height() + 1) / 2;
-                    slice::from_raw_parts((*ptr).data_uv(), (self.stride_uv() * chroma_height) as usize)
-                }
-            }
-        }
-    };
-}
-
-pub struct NativeBuffer {
-    cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>,
-}
-
-pub struct I420Buffer {
-    cxx_handle: UniquePtr<vfb_sys::ffi::I420Buffer>,
-}
-
-pub struct I420ABuffer {
-    cxx_handle: UniquePtr<vfb_sys::ffi::I420ABuffer>,
-}
-
-pub struct I422Buffer {
-    cxx_handle: UniquePtr<vfb_sys::ffi::I422Buffer>,
-}
-
-pub struct I444Buffer {
-    cxx_handle: UniquePtr<vfb_sys::ffi::I444Buffer>,
-}
-
-pub struct I010Buffer {
-    cxx_handle: UniquePtr<vfb_sys::ffi::I010Buffer>,
-}
-
-pub struct NV12Buffer {
-    cxx_handle: UniquePtr<vfb_sys::ffi::NV12Buffer>,
-}
-
-impl_video_frame_buffer!(NativeBuffer);
-impl_video_frame_buffer!(I420Buffer, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
-impl_video_frame_buffer!(I420ABuffer, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
-impl_video_frame_buffer!(I422Buffer, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
-impl_video_frame_buffer!(I444Buffer, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
-impl_video_frame_buffer!(I010Buffer, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb);
-impl_video_frame_buffer!(NV12Buffer, nv12_to_biyuv8, biyuv8_to_biyuv, biyuv_to_vfb);
-
-impl_yuv_buffer!(I420Buffer, i420_to_yuv8, yuv8_to_yuv);
-impl_yuv_buffer!(I420ABuffer, i420a_to_yuv8, yuv8_to_yuv);
-impl_yuv_buffer!(I422Buffer, i422_to_yuv8, yuv8_to_yuv);
-impl_yuv_buffer!(I444Buffer, i444_to_yuv8, yuv8_to_yuv);
-impl_yuv_buffer!(I010Buffer, i010_to_yuv16b, yuv16b_to_yuv);
-
-impl_yuv8_buffer!(I420Buffer, i420_to_yuv8);
-impl_yuv8_buffer!(I420ABuffer, i420a_to_yuv8);
-impl_yuv8_buffer!(I422Buffer, i422_to_yuv8);
-impl_yuv8_buffer!(I444Buffer, i444_to_yuv8);
-
-impl_yuv16_buffer!(I010Buffer, i010_to_yuv16b);
-
-impl_biyuv_buffer!(NV12Buffer, nv12_to_biyuv8, biyuv8_to_biyuv);
-
-impl_biyuv8_buffer!(NV12Buffer, nv12_to_biyuv8);
-
-impl NativeBuffer {
-    fn from(cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>) -> Self {
-        Self { cxx_handle }
-    }
-
-    pub(crate) fn release(self) -> UniquePtr<vfb_sys::ffi::VideoFrameBuffer> {
-        self.cxx_handle
-    }
-}
-
-impl I420Buffer {
-    pub fn new(width: u32, height: u32) -> Self {
-        Self::from(vfb_sys::ffi::create_i420_buffer(
-            width as i32,
-            height as i32,
-        ))
-    }
-
-    fn from(cxx_handle: UniquePtr<vfb_sys::ffi::I420Buffer>) -> Self {
-        Self { cxx_handle }
-    }
-
-    pub(crate) fn release(self) -> UniquePtr<vfb_sys::ffi::I420Buffer> {
-        self.cxx_handle
-    }
-}
-
-impl I420ABuffer {
-    fn from(cxx_handle: UniquePtr<vfb_sys::ffi::I420ABuffer>) -> Self {
-        Self { cxx_handle }
-    }
-
-    pub(crate) fn release(self) -> UniquePtr<vfb_sys::ffi::I420ABuffer> {
-        self.cxx_handle
-    }
-}
-
-impl I422Buffer {
-    fn from(cxx_handle: UniquePtr<vfb_sys::ffi::I422Buffer>) -> Self {
-        Self { cxx_handle }
-    }
-
-    pub(crate) fn release(self) -> UniquePtr<vfb_sys::ffi::I422Buffer> {
-        self.cxx_handle
-    }
-}
-
-impl I444Buffer {
-    fn from(cxx_handle: UniquePtr<vfb_sys::ffi::I444Buffer>) -> Self {
-        Self { cxx_handle }
-    }
-
-    pub(crate) fn release(self) -> UniquePtr<vfb_sys::ffi::I444Buffer> {
-        self.cxx_handle
-    }
-}
-
-impl I010Buffer {
-    fn from(cxx_handle: UniquePtr<vfb_sys::ffi::I010Buffer>) -> Self {
-        Self { cxx_handle }
-    }
-
-    pub(crate) fn release(self) -> UniquePtr<vfb_sys::ffi::I010Buffer> {
-        self.cxx_handle
-    }
-}
-
-impl NV12Buffer {
-    fn from(cxx_handle: UniquePtr<vfb_sys::ffi::NV12Buffer>) -> Self {
-        Self { cxx_handle }
-    }
-
-    pub(crate) fn release(self) -> UniquePtr<vfb_sys::ffi::NV12Buffer> {
-        self.cxx_handle
-    }
-}

livekit-webrtc/src/video_source.rs

@@ -0,0 +1,28 @@
+use crate::imp::video_source as vs_imp;
+
+#[cfg(not(target_arch = "wasm32"))]
+pub mod native {
+    use super::vs_imp;
+    use crate::video_frame::{VideoFrame, VideoFrameBuffer};
+    use std::fmt::{Debug, Formatter};
+
+    #[derive(Default, Clone)]
+    pub struct NativeVideoSource {
+        pub(crate) handle: vs_imp::NativeVideoSource,
+    }
+
+    impl Debug for NativeVideoSource {
+        fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
+            f.debug_struct("NativeVideoSource").finish()
+        }
+    }
+
+    impl NativeVideoSource {
+        pub fn capture_frame<T: VideoFrameBuffer>(&self, frame: &VideoFrame<T>) {
+            self.handle.capture_frame(frame)
+        }
+    }
+}
+
+#[cfg(target_arch = "wasm32")]
+pub mod web {}

livekit-webrtc/src/video_stream.rs

@@ -0,0 +1,54 @@
+use crate::imp::video_stream as stream_imp;
+
+// There is no shared sink between native and web platforms.
+// Each platform requires different configuration (e.g: WebGlContext, ..)
+
+#[cfg(not(target_arch = "wasm32"))]
+pub mod native {
+    use super::stream_imp;
+    use crate::media_stream::RtcVideoTrack;
+    use crate::video_frame::BoxVideoFrame;
+    use futures::stream::Stream;
+    use std::fmt::Debug;
+    use std::pin::Pin;
+    use std::task::{Context, Poll};
+
+    pub struct NativeVideoStream {
+        pub(crate) handle: stream_imp::NativeVideoStream,
+    }
+
+    impl Debug for NativeVideoStream {
+        fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+            f.debug_struct("NativeVideoStream")
+                .field("track", &self.track())
+                .finish()
+        }
+    }
+
+    impl NativeVideoStream {
+        pub fn new(video_track: RtcVideoTrack) -> Self {
+            Self {
+                handle: stream_imp::NativeVideoStream::new(video_track),
+            }
+        }
+
+        pub fn track(&self) -> RtcVideoTrack {
+            self.handle.track()
+        }
+
+        pub fn close(&mut self) {
+            self.handle.close();
+        }
+    }
+
+    impl Stream for NativeVideoStream {
+        type Item = BoxVideoFrame;
+
+        fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
+            Pin::new(&mut self.get_mut().handle).poll_next(cx)
+        }
+    }
+}
+
+#[cfg(target_arch = "wasm32")]
+pub mod web {}

livekit-webrtc/src/web/data_channel.rs

@@ -0,0 +1,103 @@
+use core::str;
+use std::cell::RefCell;
+use std::rc::Rc;
+use wasm_bindgen::prelude::*;
+use web_sys::{MessageEvent, RtcDataChannelEvent, RtcDataChannelState};
+
+use crate::data_channel::{
+    DataChannelError, DataChannelTrait, DataState, OnBufferedAmountChange, OnMessage, OnStateChange,
+};
+
+impl From<RtcDataChannelState> for DataState {
+    fn from(value: RtcDataChannelState) -> Self {
+        match value {
+            RtcDataChannelState::Connecting => Self::Connecting,
+            RtcDataChannelState::Open => Self::Open,
+            RtcDataChannelState::Closing => Self::Closing,
+            RtcDataChannelState::Closed => Self::Closed,
+            _ => panic!("unknown data channel state"),
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct DataChannel {
+    sys_handle: web_sys::RtcDataChannel,
+    on_closing: Rc<RefCell<Option<JsValue>>>,
+}
+
+impl DataChannelTrait for DataChannel {
+    fn send(&self, data: &[u8], binary: bool) -> Result<(), DataChannelError> {
+        if binary {
+            self.sys_handle
+                .send_with_u8_array(data)
+                .map_err(|_| DataChannelError::Send)
+        } else {
+            let utf8 = str::from_utf8(data)?;
+            self.sys_handle
+                .send_with_str(utf8)
+                .map_err(|_| DataChannelError::Send)
+        }
+    }
+
+    fn label(&self) -> String {
+        self.sys_handle.label()
+    }
+
+    fn state(&self) -> DataState {
+        self.sys_handle.ready_state().into()
+    }
+
+    fn close(&self) {
+        self.sys_handle.close();
+    }
+
+    fn on_state_change(&self, callback: Option<OnStateChange>) {
+        if let Some(mut callback) = callback {
+            let dc = self.clone();
+            let js_callback = Closure::new(move |_: RtcDataChannelEvent| {
+                callback(dc.state());
+            });
+            let js_callback = js_callback.into_js_value();
+            self.sys_handle
+                .set_onopen(Some(js_callback.unchecked_ref()));
+            self.sys_handle
+                .set_onclose(Some(js_callback.unchecked_ref()));
+            self.sys_handle
+                .add_event_listener_with_callback("closing", js_callback.unchecked_ref())
+                .unwrap();
+
+            self.on_closing.replace(Some(js_callback));
+        } else {
+            self.sys_handle.set_onopen(None);
+            self.sys_handle.set_onclose(None);
+            if let Some(on_closing) = self.on_closing.take() {
+                self.sys_handle
+                    .remove_event_listener_with_callback("closing", on_closing.unchecked_ref())
+                    .unwrap();
+            }
+            self.on_closing.replace(None);
+        }
+    }
+
+    fn on_message(&self, callback: Option<OnMessage>) {
+        let js_callback = callback.map(|mut callback| {
+            Closure::new(move |event: MessageEvent| {
+                if let Some(str) = event.as_string() {
+                    callback(str.as_bytes(), false);
+                }
+            })
+            .into_js_value()
+        });
+
+        self.sys_handle.set_onmessage(
+            js_callback
+                .as_ref()
+                .map(|callback| callback.unchecked_ref()),
+        );
+    }
+
+    fn on_buffered_amount_change(&self, _callback: Option<OnBufferedAmountChange>) {
+        todo!("onbufferedamountlow instead?")
+    }
+}

livekit-webrtc/src/web/media_stream.rs

@@ -0,0 +1,358 @@
+use super::VideoTrack;
+use crate::{
+    media_stream::{
+        BiplanarYuv8Buffer, BiplanarYuvBuffer, PlanarYuv16BBuffer, PlanarYuv8Buffer,
+        PlanarYuvBuffer, VideoFrameBuffer,
+    },
+    video_frame::{BiplanarYuv8Buffer, I420Buffer, SinkError, VideoFrame, VideoFrameBuffer},
+    I010Buffer, I420ABuffer, I422Buffer, I444Buffer, NV12Buffer,
+};
+use std::sync::mpsc;
+use web_sys::{WebGlRenderingContext, WebGlTexture};
+
+#[derive(Debug)]
+pub struct WebGlVideoSink {
+    track: Arc<VideoTrack>,
+    gl_ctx: WebGlRenderingContext,
+    tex: WebGlTexture,
+}
+
+/// Create a new WebGL texture and update it inside requestAnimationFrame
+impl WebGlVideoSink {
+    pub fn new(
+        track: Arc<VideoTrack>,
+        gl_ctx: WebGlRenderingContext,
+    ) -> Result<(Self, mpsc::Receiver<VideoFrame<WebGlBuffer>>), SinkError> {
+        let (sender, receiver) = mpsc::channel();
+        let tex = gl_ctx.create_texture()?;
+        Ok((Self { track, gl_ctx, tex }, receiver))
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct WebGlBuffer {
+    width: i32,
+    height: i32,
+    tex: WebGlTexture,
+}
+
+impl VideoFrameBuffer for WebGlBuffer {
+    fn width(&self) -> i32 {
+        self.width
+    }
+
+    fn height(&self) -> i32 {
+        self.height
+    }
+}
+
+/// The following types could be implemented if we want
+/// to support VideoFrame with WebCodecs
+
+#[derive(Debug)]
+pub struct I420Buffer {}
+
+#[derive(Debug)]
+pub struct I420ABuffer {}
+
+#[derive(Debug)]
+pub struct I422Buffer {}
+
+#[derive(Debug)]
+pub struct I444Buffer {}
+
+#[derive(Debug)]
+pub struct I010Buffer {}
+
+#[derive(Debug)]
+pub struct NV12Buffer {}
+
+impl VideoFrameBuffer for I420Buffer {
+    fn width(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn height(&self) -> i32 {
+        unimplemented!()
+    }
+}
+
+impl VideoFrameBuffer for I420ABuffer {
+    fn width(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn height(&self) -> i32 {
+        unimplemented!()
+    }
+}
+
+impl VideoFrameBuffer for I422Buffer {
+    fn width(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn height(&self) -> i32 {
+        unimplemented!()
+    }
+}
+
+impl VideoFrameBuffer for I444Buffer {
+    fn width(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn height(&self) -> i32 {
+        unimplemented!()
+    }
+}
+
+impl VideoFrameBuffer for I010Buffer {
+    fn width(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn height(&self) -> i32 {
+        unimplemented!()
+    }
+}
+
+impl VideoFrameBuffer for NV12Buffer {
+    fn width(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn height(&self) -> i32 {
+        unimplemented!()
+    }
+}
+
+impl PlanarYuvBuffer for I420Buffer {
+    fn chroma_width(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn chroma_height(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn stride_y(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn stride_u(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn stride_v(&self) -> i32 {
+        unimplemented!()
+    }
+}
+
+impl PlanarYuvBuffer for I420ABuffer {
+    fn chroma_width(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn chroma_height(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn stride_y(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn stride_u(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn stride_v(&self) -> i32 {
+        unimplemented!()
+    }
+}
+
+impl PlanarYuvBuffer for I422Buffer {
+    fn chroma_width(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn chroma_height(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn stride_y(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn stride_u(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn stride_v(&self) -> i32 {
+        unimplemented!()
+    }
+}
+
+impl PlanarYuvBuffer for I444Buffer {
+    fn chroma_width(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn chroma_height(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn stride_y(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn stride_u(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn stride_v(&self) -> i32 {
+        unimplemented!()
+    }
+}
+
+impl PlanarYuvBuffer for I010Buffer {
+    fn chroma_width(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn chroma_height(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn stride_y(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn stride_u(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn stride_v(&self) -> i32 {
+        unimplemented!()
+    }
+}
+
+impl PlanarYuvBuffer for NV12Buffer {
+    fn chroma_width(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn chroma_height(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn stride_y(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn stride_u(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn stride_v(&self) -> i32 {
+        unimplemented!()
+    }
+}
+
+impl PlanarYuv8Buffer for I420Buffer {
+    fn data_y(&self) -> &[u8] {
+        unimplemented!()
+    }
+
+    fn data_u(&self) -> &[u8] {
+        unimplemented!()
+    }
+
+    fn data_v(&self) -> &[u8] {
+        unimplemented!()
+    }
+}
+
+impl PlanarYuv8Buffer for I420ABuffer {
+    fn data_y(&self) -> &[u8] {
+        unimplemented!()
+    }
+
+    fn data_u(&self) -> &[u8] {
+        unimplemented!()
+    }
+
+    fn data_v(&self) -> &[u8] {
+        unimplemented!()
+    }
+}
+
+impl PlanarYuv8Buffer for I422Buffer {
+    fn data_y(&self) -> &[u8] {
+        unimplemented!()
+    }
+
+    fn data_u(&self) -> &[u8] {
+        unimplemented!()
+    }
+
+    fn data_v(&self) -> &[u8] {
+        unimplemented!()
+    }
+}
+
+impl PlanarYuv8Buffer for I444Buffer {
+    fn data_y(&self) -> &[u8] {
+        unimplemented!()
+    }
+
+    fn data_u(&self) -> &[u8] {
+        unimplemented!()
+    }
+
+    fn data_v(&self) -> &[u8] {
+        unimplemented!()
+    }
+}
+
+impl PlanarYuv16BBuffer for I010Buffer {
+    fn data_y(&self) -> &[u16] {
+        unimplemented!()
+    }
+
+    fn data_u(&self) -> &[u16] {
+        unimplemented!()
+    }
+
+    fn data_v(&self) -> &[u16] {
+        unimplemented!()
+    }
+}
+
+impl BiplanarYuvBuffer for NV12Buffer {
+    fn chroma_width(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn chroma_height(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn stride_y(&self) -> i32 {
+        unimplemented!()
+    }
+
+    fn stride_uv(&self) -> i32 {
+        unimplemented!()
+    }
+}
+
+impl BiplanarYuv8Buffer for NV12Buffer {
+    fn data_y(&self) -> &[u8] {
+        unimplemented!()
+    }
+
+    fn data_uv(&self) -> &[u8] {
+        unimplemented!()
+    }
+}

livekit-webrtc/src/web/mod.rs

@@ -0,0 +1 @@
+unimplemented!()

livekit-webrtc/src/webrtc.rs

@@ -1,24 +0,0 @@
-use cxx::SharedPtr;
-
-use webrtc_sys::webrtc as sys_rtc;
-
-pub use sys_rtc::ffi::MediaType;
-pub use sys_rtc::ffi::Priority;
-pub use sys_rtc::ffi::RtpTransceiverDirection;
-
-#[derive(Clone)]
-pub struct RTCRuntime {
-    cxx_handle: SharedPtr<sys_rtc::ffi::RTCRuntime>,
-}
-
-impl RTCRuntime {
-    pub fn new() -> Self {
-        Self {
-            cxx_handle: sys_rtc::ffi::create_rtc_runtime(),
-        }
-    }
-
-    pub(crate) fn release(self) -> SharedPtr<sys_rtc::ffi::RTCRuntime> {
-        self.cxx_handle
-    }
-}

livekit-webrtc/src/yuv_helper.rs

@@ -1,98 +0,0 @@
-use thiserror::Error;
-
-use webrtc_sys::yuv_helper as yuv_sys;
-
-#[derive(Error, Debug)]
-pub enum ConvertError {
-    #[error("conversion failed: {0}")]
-    Convert(&'static str),
-}
-
-fn i420_safety(
-    src_y: &[u8],
-    src_stride_y: i32,
-    src_u: &[u8],
-    src_stride_u: i32,
-    src_v: &[u8],
-    src_stride_v: i32,
-    dst: &mut [u8],
-    dst_stride: i32,
-    _width: i32,
-    height: i32,
-) -> Result<(), ConvertError> {
-    let chroma_height = (height + 1) / 2;
-    let min_y = (src_stride_y * height) as usize;
-    let min_u = (src_stride_u * chroma_height) as usize;
-    let min_v = (src_stride_v * chroma_height) as usize;
-    let min_dst = (dst_stride * height) as usize;
-
-    if src_y.len() < min_y {
-        return Err(ConvertError::Convert("src_y isn't large enough"));
-    }
-
-    if src_u.len() < min_u {
-        return Err(ConvertError::Convert("src_u isn't large enough"));
-    }
-
-    if src_v.len() < min_v {
-        return Err(ConvertError::Convert("src_v isn't large enough"));
-    }
-
-    if dst.len() < min_dst {
-        return Err(ConvertError::Convert("dst isn't large enough"));
-    }
-
-    Ok(())
-}
-
-macro_rules! i420_to_x {
-    ($x:ident) => {
-        pub fn $x(
-            src_y: &[u8],
-            src_stride_y: i32,
-            src_u: &[u8],
-            src_stride_u: i32,
-            src_v: &[u8],
-            src_stride_v: i32,
-            dst: &mut [u8],
-            dst_stride: i32,
-            width: i32,
-            height: i32,
-        ) -> Result<(), ConvertError> {
-            i420_safety(
-                src_y,
-                src_stride_y,
-                src_u,
-                src_stride_u,
-                src_v,
-                src_stride_v,
-                dst,
-                dst_stride,
-                width,
-                height,
-            )?;
-
-            unsafe {
-                yuv_sys::ffi::$x(
-                    src_y.as_ptr(),
-                    src_stride_y,
-                    src_u.as_ptr(),
-                    src_stride_u,
-                    src_v.as_ptr(),
-                    src_stride_v,
-                    dst.as_mut_ptr(),
-                    dst_stride,
-                    width,
-                    height,
-                );
-            }
-
-            Ok(())
-        }
-    };
-}
-
-i420_to_x!(i420_to_argb);
-i420_to_x!(i420_to_bgra);
-i420_to_x!(i420_to_abgr);
-i420_to_x!(i420_to_rgba);

livekit/Cargo.toml

@@ -11,9 +11,9 @@ livekit-webrtc = { path = "../livekit-webrtc", version = "0.1.1" }
 livekit-utils = { path = "../livekit-utils", version = "0.1.1" }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1.0"
-tokio-tungstenite = { version = "0.17.2", features = ["native-tls"] }
+tokio-tungstenite = { version = "0.18", features = ["native-tls"] }
 tokio = { version = "1", features = ["full"] }
-futures = "0.3"
+tokio-stream = "0.1"
 parking_lot = { version = "0.12.1", features = ["send_guard"] }
 url = "2.2.2"
 futures-util = "0.3.23"

livekit/src/lib.rs

@@ -1,12 +1,9 @@
 extern crate core;
 
-pub mod proto {
+pub mod proto;
-    include!(concat!(env!("OUT_DIR"), "/livekit.rs"));
+mod room;
-}
-
 mod rtc_engine;
 mod signal_client;
-mod room;
 
 pub mod webrtc {
     pub use livekit_webrtc::*;

livekit/src/prelude.rs

@@ -1,31 +1,12 @@
-pub use crate::participant::{
+pub use crate::participant::{LocalParticipant, Participant, ParticipantEvent, RemoteParticipant};
-    LocalParticipant, Participant, ParticipantEvent, ParticipantTrait, RemoteParticipant,
-};
 
-pub use crate::{ConnectionState, Room, RoomError, RoomEvent, RoomSession};
+pub use crate::{ConnectionState, Room, RoomError, RoomEvent, RoomResult, RoomSession};
 
-pub use crate::publication::{
+pub use crate::publication::{LocalTrackPublication, RemoteTrackPublication, TrackPublication};
-    LocalTrackPublication, RemoteTrackPublication, TrackPublication, TrackPublicationTrait,
-};
 
 pub use crate::track::{
-    AudioTrackHandle, LocalAudioTrack, LocalTrackHandle, LocalVideoTrack, RemoteAudioTrack,
+    AudioTrack, LocalAudioTrack, LocalTrack, LocalVideoTrack, RemoteAudioTrack, RemoteTrack,
-    RemoteTrackHandle, RemoteVideoTrack, StreamState, TrackEvent, TrackHandle, TrackKind,
+    RemoteVideoTrack, StreamState, Track, TrackEvent, TrackKind, TrackSource, VideoTrack,
-    TrackSource, TrackTrait, VideoTrackHandle,
 };
 
 pub use crate::id::*;
-
-pub use crate::webrtc::{
-    data_channel::DataChannel,
-    media_stream::{
-        AudioTrack, MediaStream, MediaStreamTrackHandle, MediaStreamTrackTrait,
-        OnConstraintsChangedHandler, OnDiscardedFrameHandler, OnFrameHandler, VideoTrack,
-    },
-    rtp_receiver::RtpReceiver,
-    rtp_transceiver::RtpTransceiver,
-    video_frame::{VideoFrame, VideoRotation},
-    video_frame_buffer::{
-        VideoFormatType, VideoFrameBuffer, VideoFrameBufferTrait, VideoFrameBufferType,
-    },
-};

livekit/src/proto.rs

@@ -0,0 +1,49 @@
+use crate::track;
+
+include!(concat!(env!("OUT_DIR"), "/livekit.rs"));
+
+// Conversions
+impl TryFrom<TrackType> for track::TrackKind {
+    type Error = &'static str;
+
+    fn try_from(r#type: TrackType) -> Result<Self, Self::Error> {
+        match r#type {
+            TrackType::Audio => Ok(Self::Audio),
+            TrackType::Video => Ok(Self::Video),
+            TrackType::Data => Err("data tracks are not implemented yet"),
+        }
+    }
+}
+
+impl From<track::TrackKind> for TrackType {
+    fn from(kind: track::TrackKind) -> Self {
+        match kind {
+            track::TrackKind::Audio => Self::Audio,
+            track::TrackKind::Video => Self::Video,
+        }
+    }
+}
+
+impl From<TrackSource> for track::TrackSource {
+    fn from(source: TrackSource) -> Self {
+        match source {
+            TrackSource::Camera => Self::Camera,
+            TrackSource::Microphone => Self::Microphone,
+            TrackSource::ScreenShare => Self::Screenshare,
+            TrackSource::ScreenShareAudio => Self::ScreenshareAudio,
+            TrackSource::Unknown => Self::Unknown,
+        }
+    }
+}
+
+impl From<track::TrackSource> for TrackSource {
+    fn from(source: track::TrackSource) -> Self {
+        match source {
+            track::TrackSource::Camera => Self::Camera,
+            track::TrackSource::Microphone => Self::Microphone,
+            track::TrackSource::Screenshare => Self::ScreenShare,
+            track::TrackSource::ScreenshareAudio => Self::ScreenShareAudio,
+            track::TrackSource::Unknown => Self::Unknown,
+        }
+    }
+}

livekit/src/room/mod.rs

@@ -1,3 +1,4 @@
+use self::track::RemoteTrack;
 use crate::participant::ConnectionQuality;
 use crate::prelude::*;
 use crate::proto;
@@ -10,6 +11,7 @@ use tokio::sync::mpsc;
 pub use crate::rtc_engine::SimulateScenario;
 
 pub mod id;
+pub mod options;
 pub mod participant;
 pub mod publication;
 pub mod room_session;
@@ -25,34 +27,36 @@ pub enum RoomError {
     Engine(#[from] EngineError),
     #[error("room failure: {0}")]
     Internal(String),
+    #[error("this track or a track of the same source is already published")]
+    TrackAlreadyPublished,
 }
 
 #[derive(Clone, Debug)]
 pub enum RoomEvent {
-    ParticipantConnected(Arc<RemoteParticipant>),
+    ParticipantConnected(RemoteParticipant),
-    ParticipantDisconnected(Arc<RemoteParticipant>),
+    ParticipantDisconnected(RemoteParticipant),
     TrackSubscribed {
-        track: RemoteTrackHandle,
+        track: RemoteTrack,
         publication: RemoteTrackPublication,
-        participant: Arc<RemoteParticipant>,
+        participant: RemoteParticipant,
     },
     TrackPublished {
         publication: RemoteTrackPublication,
-        participant: Arc<RemoteParticipant>,
+        participant: RemoteParticipant,
     },
     TrackUnpublished {
         publication: RemoteTrackPublication,
-        participant: Arc<RemoteParticipant>,
+        participant: RemoteParticipant,
     },
     TrackUnsubscribed {
-        track: RemoteTrackHandle,
+        track: RemoteTrack,
         publication: RemoteTrackPublication,
-        participant: Arc<RemoteParticipant>,
+        participant: RemoteParticipant,
     },
     TrackSubscriptionFailed {
         error: track::TrackError,
         sid: TrackSid,
-        participant: Arc<RemoteParticipant>,
+        participant: RemoteParticipant,
     },
     TrackMuted {
         participant: Participant,
@@ -72,7 +76,7 @@ pub enum RoomEvent {
     DataReceived {
         payload: Arc<Vec<u8>>,
         kind: proto::data_packet::Kind,
-        participant: Arc<RemoteParticipant>,
+        participant: RemoteParticipant,
     },
     ConnectionStateChanged(ConnectionState),
     Connected,

livekit/src/room/observer.rs

@@ -0,0 +1 @@
+

livekit/src/room/options.rs

@@ -0,0 +1,372 @@
+use crate::prelude::*;
+use crate::proto;
+use livekit_webrtc::prelude::*;
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub enum VideoCodec {
+    VP8,
+    H264,
+    AV1,
+}
+
+impl VideoCodec {
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            VideoCodec::VP8 => "vp8",
+            VideoCodec::H264 => "h264",
+            VideoCodec::AV1 => "av1",
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct VideoResolution {
+    pub width: u32,
+    pub height: u32,
+    pub frame_rate: f64,
+    pub aspect_ratio: f32,
+}
+
+#[derive(Debug, Clone)]
+pub struct VideoEncoding {
+    pub max_bitrate: u64,
+    pub max_framerate: f64,
+}
+
+#[derive(Debug, Clone)]
+pub struct VideoPreset {
+    pub encoding: VideoEncoding,
+    pub width: u32,
+    pub height: u32,
+}
+
+#[derive(Debug, Clone)]
+pub struct AudioPreset {
+    pub max_bitrate: u32,
+}
+
+impl AudioPreset {
+    pub const fn new(max_bitrate: u32) -> Self {
+        Self { max_bitrate }
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct VideoCaptureOptions {
+    pub preset: VideoPreset,
+}
+
+impl Default for VideoCaptureOptions {
+    fn default() -> Self {
+        Self {
+            preset: video::H720,
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct TrackPublishOptions {
+    pub dynacast: bool,
+    pub video_codec: VideoCodec,
+    pub dtx: bool,
+    pub red: bool,
+    pub simulcast: bool,
+    pub screenshare: bool,
+    pub name: String,
+    pub source: TrackSource,
+}
+
+impl Default for TrackPublishOptions {
+    fn default() -> Self {
+        Self {
+            dynacast: false,
+            video_codec: VideoCodec::VP8,
+            dtx: true,
+            red: true,
+            simulcast: true,
+            screenshare: false,
+            name: "unnamed track".to_owned(),
+            source: TrackSource::Unknown,
+        }
+    }
+}
+
+impl VideoPreset {
+    pub const fn new(width: u32, height: u32, max_bitrate: u64, max_framerate: f64) -> Self {
+        Self {
+            width,
+            height,
+            encoding: VideoEncoding {
+                max_bitrate,
+                max_framerate,
+            },
+        }
+    }
+
+    pub fn resolution(&self) -> VideoResolution {
+        VideoResolution {
+            width: self.width,
+            height: self.height,
+            frame_rate: self.encoding.max_framerate,
+            aspect_ratio: self.width as f32 / self.height as f32,
+        }
+    }
+}
+
+/// Compute appropriate RtpEncodingParameters from the video resolution.
+/// TrackPublishOptions helps to find the most appropriate encodings
+pub fn compute_video_encodings(
+    width: u32,
+    height: u32,
+    options: &TrackPublishOptions,
+) -> Vec<RtpEncodingParameters> {
+    let encoding = compute_appropriate_encoding(options.screenshare, width, height);
+
+    let initial_preset = VideoPreset {
+        width,
+        height,
+        encoding: VideoEncoding {
+            max_bitrate: encoding.max_bitrate,
+            max_framerate: encoding.max_framerate,
+        },
+    };
+
+    if !options.simulcast {
+        return into_rtp_encodings(width, height, &[initial_preset]);
+    }
+
+    let mut simulcast_presets =
+        compute_default_simulcast_presets(options.screenshare, &initial_preset);
+
+    let mid_preset = simulcast_presets.pop();
+    let low_preset = simulcast_presets.pop();
+
+    let size = u32::max(width, height);
+    if size >= 960 && low_preset.is_some() {
+        return into_rtp_encodings(
+            width,
+            height,
+            &[low_preset.unwrap(), mid_preset.unwrap(), initial_preset],
+        );
+    } else if size >= 480 {
+        return into_rtp_encodings(width, height, &[mid_preset.unwrap(), initial_preset]);
+    }
+
+    // Other layers not needed
+    into_rtp_encodings(width, height, &[initial_preset])
+}
+
+/// Return an appropriate VideoEncdoding for the specified resolution based on our presets
+pub fn compute_appropriate_encoding(
+    is_screenshare: bool,
+    width: u32,
+    height: u32,
+) -> VideoEncoding {
+    let presets = compute_presets_for_resolution(is_screenshare, width, height);
+    let size = u32::max(width, height);
+
+    for preset in presets {
+        if preset.width >= size {
+            return preset.encoding.clone();
+        }
+    }
+
+    unreachable!()
+}
+
+pub fn compute_presets_for_resolution(
+    is_screenshare: bool,
+    width: u32,
+    height: u32,
+) -> &'static [VideoPreset] {
+    if is_screenshare {
+        return screenshare::PRESETS;
+    }
+
+    // Check how close width & height are from 16/9 or 4/3
+    let ar = landscape_aspect_ratio(width, height);
+    if f32::abs(ar - 16.0 / 9.0) < f32::abs(ar - 4.0 / 3.0) {
+        return video::PRESETS;
+    }
+
+    video43::PRESETS
+}
+
+/// Returns our most appropriate default presets
+pub fn compute_default_simulcast_presets(
+    is_screenshare: bool,
+    initial: &VideoPreset,
+) -> Vec<VideoPreset> {
+    if is_screenshare {
+        return vec![screenshare::compute_default_simulcast_preset(initial)];
+    }
+
+    let ar = landscape_aspect_ratio(initial.width, initial.height);
+    if f32::abs(ar - 16.0 / 9.0) < f32::abs(ar - 4.0 / 3.0) {
+        return video::DEFAULT_SIMULCAST_PRESETS.to_owned();
+    }
+
+    video43::DEFAULT_SIMULCAST_PRESETS.to_owned()
+}
+
+pub fn landscape_aspect_ratio(width: u32, height: u32) -> f32 {
+    if width > height {
+        width as f32 / height as f32
+    } else {
+        height as f32 / width as f32
+    }
+}
+
+/// Presets must be ordered
+pub fn into_rtp_encodings(
+    initial_width: u32,
+    initial_height: u32,
+    presets: &[VideoPreset],
+) -> Vec<RtpEncodingParameters> {
+    let mut encodings = Vec::with_capacity(presets.len());
+    let size = u32::min(initial_width, initial_height);
+    for (i, preset) in presets.iter().enumerate() {
+        encodings.push(RtpEncodingParameters {
+            rid: VIDEO_RIDS[i].to_string(),
+            scale_resolution_down_by: Some(f64::max(
+                1.0,
+                size as f64 / u32::min(preset.width, preset.height) as f64,
+            )),
+            max_bitrate: Some(preset.encoding.max_bitrate),
+            max_framerate: Some(preset.encoding.max_framerate),
+            ..Default::default()
+        })
+    }
+
+    encodings
+}
+
+pub fn video_quality_for_rid(rid: &str) -> Option<proto::VideoQuality> {
+    match rid {
+        "f" => Some(proto::VideoQuality::High),
+        "h" => Some(proto::VideoQuality::Medium),
+        "q" => Some(proto::VideoQuality::Low),
+        _ => None,
+    }
+}
+
+pub fn video_layers_from_encodings(
+    width: u32,
+    height: u32,
+    encodings: &[RtpEncodingParameters],
+) -> Vec<proto::VideoLayer> {
+    if encodings.is_empty() {
+        return vec![proto::VideoLayer {
+            quality: proto::VideoQuality::High as i32,
+            width,
+            height,
+            bitrate: 0,
+            ssrc: 0,
+        }];
+    }
+
+    let mut layers = Vec::with_capacity(encodings.len());
+    for encoding in encodings {
+        let scale = encoding.scale_resolution_down_by.unwrap_or(1.0);
+        let quality = video_quality_for_rid(&encoding.rid).unwrap_or(proto::VideoQuality::High);
+
+        layers.push(proto::VideoLayer {
+            quality: quality as i32,
+            width: (width as f64 / scale) as u32,
+            height: (height as f64 / scale) as u32,
+            bitrate: encoding.max_bitrate.unwrap_or(0) as u32,
+            ssrc: 0,
+        });
+    }
+
+    layers
+}
+
+const VIDEO_RIDS: &[char] = &['q', 'h', 'f'];
+
+pub mod audio {
+    use super::AudioPreset;
+
+    pub const TELEPHONE: AudioPreset = AudioPreset::new(12_000);
+    pub const SPEECH: AudioPreset = AudioPreset::new(20_000);
+    pub const MUSIC: AudioPreset = AudioPreset::new(32_000);
+    pub const MUSIC_STEREO: AudioPreset = AudioPreset::new(48_000);
+    pub const MUSIC_HIGH_QUALITY: AudioPreset = AudioPreset::new(64_000);
+    pub const MUSIC_HIGH_QUALITY_STEREO: AudioPreset = AudioPreset::new(96_000);
+
+    pub const PRESETS: &[AudioPreset] = &[
+        TELEPHONE,
+        SPEECH,
+        MUSIC,
+        MUSIC_STEREO,
+        MUSIC_HIGH_QUALITY,
+        MUSIC_HIGH_QUALITY_STEREO,
+    ];
+}
+
+pub mod video {
+    use super::VideoPreset;
+
+    pub const H90: VideoPreset = VideoPreset::new(160, 90, 60_000, 15.0);
+    pub const H180: VideoPreset = VideoPreset::new(320, 180, 120_000, 15.0);
+    pub const H216: VideoPreset = VideoPreset::new(384, 216, 180_000, 15.0);
+    pub const H360: VideoPreset = VideoPreset::new(640, 360, 300_000, 20.0);
+    pub const H540: VideoPreset = VideoPreset::new(960, 540, 600_000, 25.0);
+    pub const H720: VideoPreset = VideoPreset::new(1280, 720, 1_700_000, 30.0);
+    pub const H1080: VideoPreset = VideoPreset::new(1920, 1080, 3_000_000, 30.0);
+    pub const H1440: VideoPreset = VideoPreset::new(2560, 1440, 5_000_000, 30.0);
+    pub const H2160: VideoPreset = VideoPreset::new(3840, 2160, 8_000_000, 30.0);
+
+    pub const PRESETS: &[VideoPreset] = &[H90, H180, H216, H360, H540, H720, H1080, H1440, H2160];
+    pub const DEFAULT_SIMULCAST_PRESETS: &[VideoPreset] = &[H180, H360];
+}
+
+pub mod video43 {
+    use super::VideoPreset;
+
+    pub const H120: VideoPreset = VideoPreset::new(160, 120, 80_000, 15.0);
+    pub const H180: VideoPreset = VideoPreset::new(240, 180, 100_000, 15.0);
+    pub const H240: VideoPreset = VideoPreset::new(320, 240, 150_000, 15.0);
+    pub const H360: VideoPreset = VideoPreset::new(480, 360, 225_000, 20.0);
+    pub const H480: VideoPreset = VideoPreset::new(640, 480, 300_000, 20.0);
+    pub const H540: VideoPreset = VideoPreset::new(720, 540, 450_000, 25.0);
+    pub const H720: VideoPreset = VideoPreset::new(960, 720, 1_500_000, 30.0);
+    pub const H1080: VideoPreset = VideoPreset::new(1440, 1080, 2_500_000, 30.0);
+    pub const H1440: VideoPreset = VideoPreset::new(1920, 1440, 3_500_000, 30.0);
+
+    pub const PRESETS: &[VideoPreset] = &[H120, H180, H240, H360, H480, H540, H720, H1080, H1440];
+    pub const DEFAULT_SIMULCAST_PRESETS: &[VideoPreset] = &[H180, H360];
+}
+
+pub mod screenshare {
+    /// The screenshare presets are optimized for quality.
+    /// When simulcasting, we prefer to reduce the FPS.
+    use super::VideoPreset;
+
+    pub const H360_FPS3: VideoPreset = VideoPreset::new(640, 360, 200_000, 3.0);
+    pub const H720_FPS5: VideoPreset = VideoPreset::new(1280, 720, 400_000, 5.0);
+    pub const H720_FPS15: VideoPreset = VideoPreset::new(1280, 720, 1_000_000, 15.0);
+    pub const H1080_FPS15: VideoPreset = VideoPreset::new(1920, 1080, 1_500_000, 15.0);
+    pub const H1080_FPS30: VideoPreset = VideoPreset::new(1920, 1080, 3_000_000, 30.0);
+
+    pub const PRESETS: &[VideoPreset] =
+        &[H360_FPS3, H720_FPS5, H720_FPS15, H1080_FPS15, H1080_FPS30];
+
+    /// Only one additional layer for screenshares. (Prioritize quality)
+    pub fn compute_default_simulcast_preset(initial: &VideoPreset) -> VideoPreset {
+        const SCALE_DOWN_FACTOR: u32 = 2;
+        const FPS: f64 = 3.0;
+
+        VideoPreset::new(
+            initial.width / SCALE_DOWN_FACTOR,
+            initial.height / SCALE_DOWN_FACTOR,
+            u64::max(
+                150_000,
+                initial.encoding.max_bitrate as u64
+                    / (SCALE_DOWN_FACTOR.pow(2) as u64
+                        * (initial.encoding.max_framerate / FPS) as u64),
+            ),
+            FPS,
+        )
+    }
+}

livekit/src/room/participant/local_participant.rs

@@ -1,37 +1,130 @@
-use super::{
+use super::{ConnectionQuality, ParticipantInner};
-    impl_participant_trait, ConnectionQuality, ParticipantInternalTrait, ParticipantShared,
+use crate::options::compute_video_encodings;
-};
+use crate::options::video_layers_from_encodings;
+use crate::options::TrackPublishOptions;
 use crate::prelude::*;
 use crate::proto;
-use crate::publication::TrackPublication;
+use crate::rtc_engine::RtcEngine;
-use crate::rtc_engine::RTCEngine;
 use parking_lot::RwLockReadGuard;
 use std::collections::HashMap;
-use std::sync::atomic::Ordering;
 use std::sync::Arc;
 use tokio::sync::mpsc;
+use tracing::{debug, trace};
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct LocalParticipant {
-    shared: ParticipantShared,
+    inner: Arc<ParticipantInner>,
-    rtc_engine: Arc<RTCEngine>,
+    rtc_engine: Arc<RtcEngine>,
 }
 
 impl LocalParticipant {
     pub(crate) fn new(
-        rtc_engine: Arc<RTCEngine>,
+        rtc_engine: Arc<RtcEngine>,
         sid: ParticipantSid,
         identity: ParticipantIdentity,
         name: String,
         metadata: String,
     ) -> Self {
         Self {
-            shared: ParticipantShared::new(sid, identity, name, metadata),
+            inner: Arc::new(ParticipantInner::new(sid, identity, name, metadata)),
             rtc_engine,
         }
     }
 
-    pub async fn publish_track(&self) {}
+    pub async fn publish_track(
+        &self,
+        track: LocalTrack,
+        options: TrackPublishOptions,
+    ) -> RoomResult<LocalTrackPublication> {
+        let mut req = proto::AddTrackRequest {
+            cid: track.rtc_track().id(),
+            name: options.name.clone(),
+            r#type: proto::TrackType::from(track.kind()) as i32,
+            muted: track.muted(),
+            source: proto::TrackSource::from(options.source) as i32,
+            disable_dtx: !options.dtx,
+            disable_red: !options.red,
+            ..Default::default()
+        };
+
+        let mut encodings = Vec::default();
+        match &track {
+            LocalTrack::Video(video_track) => {
+                // Get the video dimension
+                // TODO(theomonnom): Use MediaStreamTrack::getSettings() on web
+                let capture_options = video_track.capture_options();
+                req.width = capture_options.preset.width;
+                req.height = capture_options.preset.height;
+
+                encodings = compute_video_encodings(req.width, req.height, &options);
+                req.layers = video_layers_from_encodings(req.width, req.height, &encodings);
+            }
+            LocalTrack::Audio(_audio_track) => {}
+        }
+
+        let track_info = self.rtc_engine.add_track(req).await?;
+        let publication =
+            LocalTrackPublication::new(track_info.clone(), track.clone(), options.clone());
+        track.update_info(track_info); // Update SID + Source
+        debug!("publishing track with cid {:?}", track.rtc_track().id());
+        let transceiver = self
+            .rtc_engine
+            .create_sender(track.clone(), options, encodings)
+            .await?;
+
+        track.update_transceiver(Some(transceiver));
+        track.start();
+
+        tokio::spawn({
+            let rtc_engine = self.rtc_engine.clone();
+            async move {
+                let _ = rtc_engine.negotiate_publisher().await;
+            }
+        });
+
+        self.inner
+            .add_track_publication(TrackPublication::Local(publication.clone()));
+
+        self.inner
+            .dispatcher
+            .dispatch(&ParticipantEvent::LocalTrackPublished {
+                publication: publication.clone(),
+            });
+
+        Ok(publication)
+    }
+
+    pub async fn unpublish_track(
+        &self,
+        track: TrackSid,
+        stop_on_unpublish: bool,
+    ) -> RoomResult<LocalTrackPublication> {
+        let mut tracks = self.inner.tracks.write();
+        if let Some(TrackPublication::Local(publication)) = tracks.remove(&track) {
+            let track = publication.track().unwrap();
+            let sender = track.transceiver().unwrap().sender();
+            self.rtc_engine.remove_track(sender).await?;
+            track.update_transceiver(None);
+
+            self.inner
+                .dispatcher
+                .dispatch(&ParticipantEvent::LocalTrackUnpublished {
+                    publication: publication.clone(),
+                });
+            publication.update_track(None);
+
+            tokio::spawn({
+                let rtc_engine = self.rtc_engine.clone();
+                async move {
+                    let _ = rtc_engine.negotiate_publisher().await;
+                }
+            });
+
+            Ok(publication)
+        } else {
+            Err(RoomError::Internal("track not found".to_string()))
+        }
+    }
 
     pub async fn publish_data(
         &self,
@@ -52,24 +145,80 @@ impl LocalParticipant {
             .await
             .map_err(Into::into)
     }
-}
 
-impl ParticipantInternalTrait for LocalParticipant {
+    #[inline]
-    fn update_info(self: &Arc<Self>, info: proto::ParticipantInfo, _emit_events: bool) {
+    pub fn get_track_publication(&self, sid: &TrackSid) -> Option<LocalTrackPublication> {
-        self.shared.update_info(info);
+        self.inner.tracks.read().get(sid).map(|track| {
+            if let TrackPublication::Local(local) = track {
+                return local.clone();
             }
 
-    fn set_speaking(&self, speaking: bool) {
+            unreachable!()
-        self.shared.set_speaking(speaking);
+        })
     }
 
-    fn set_audio_level(&self, level: f32) {
+    #[inline]
-        self.shared.set_audio_level(level);
+    pub fn sid(&self) -> ParticipantSid {
+        self.inner.sid()
     }
 
-    fn set_connection_quality(&self, quality: ConnectionQuality) {
+    #[inline]
-        self.shared.set_connection_quality(quality);
+    pub fn identity(&self) -> ParticipantIdentity {
+        self.inner.identity()
+    }
+
+    #[inline]
+    pub fn name(&self) -> String {
+        self.inner.name()
+    }
+
+    #[inline]
+    pub fn metadata(&self) -> String {
+        self.inner.metadata()
+    }
+
+    #[inline]
+    pub fn is_speaking(&self) -> bool {
+        self.inner.is_speaking()
+    }
+
+    #[inline]
+    pub fn tracks(&self) -> RwLockReadGuard<HashMap<TrackSid, TrackPublication>> {
+        self.inner.tracks()
+    }
+
+    #[inline]
+    pub fn audio_level(&self) -> f32 {
+        self.inner.audio_level()
+    }
+
+    #[inline]
+    pub fn connection_quality(&self) -> ConnectionQuality {
+        self.inner.connection_quality()
+    }
+
+    #[inline]
+    pub fn register_observer(&self) -> mpsc::UnboundedReceiver<ParticipantEvent> {
+        self.inner.register_observer()
+    }
+
+    #[inline]
+    pub(crate) fn update_info(self: &Self, info: proto::ParticipantInfo) {
+        self.inner.update_info(info);
+    }
+
+    #[inline]
+    pub(crate) fn set_speaking(&self, speaking: bool) {
+        self.inner.set_speaking(speaking);
+    }
+
+    #[inline]
+    pub(crate) fn set_audio_level(&self, level: f32) {
+        self.inner.set_audio_level(level);
+    }
+
+    #[inline]
+    pub(crate) fn set_connection_quality(&self, quality: ConnectionQuality) {
+        self.inner.set_connection_quality(quality);
     }
 }
-
-impl_participant_trait!(LocalParticipant);

livekit/src/room/participant/mod.rs

@@ -24,11 +24,11 @@ pub enum ParticipantEvent {
         publication: RemoteTrackPublication,
     },
     TrackSubscribed {
-        track: RemoteTrackHandle,
+        track: RemoteTrack,
         publication: RemoteTrackPublication,
     },
     TrackUnsubscribed {
-        track: RemoteTrackHandle,
+        track: RemoteTrack,
         publication: RemoteTrackPublication,
     },
     TrackSubscriptionFailed {
@@ -51,6 +51,12 @@ pub enum ParticipantEvent {
     ConnectionQualityChanged {
         quality: ConnectionQuality,
     },
+    LocalTrackPublished {
+        publication: LocalTrackPublication,
+    },
+    LocalTrackUnpublished {
+        publication: LocalTrackPublication,
+    },
 }
 
 #[derive(Debug, Clone, Copy, Eq, PartialEq)]
@@ -83,21 +89,48 @@ impl From<proto::ConnectionQuality> for ConnectionQuality {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
-pub(super) struct ParticipantShared {
+pub enum Participant {
-    pub(super) sid: Mutex<ParticipantSid>,
+    Local(LocalParticipant),
-    pub(super) identity: Mutex<ParticipantIdentity>,
+    Remote(RemoteParticipant),
-    pub(super) name: Mutex<String>,
-    pub(super) metadata: Mutex<String>,
-    pub(super) tracks: RwLock<HashMap<TrackSid, TrackPublication>>,
-    pub(super) speaking: AtomicBool,
-    pub(super) audio_level: AtomicU32,
-    pub(super) connection_quality: AtomicU8,
-    pub(super) dispatcher: Mutex<Dispatcher<ParticipantEvent>>,
 }
 
-impl ParticipantShared {
+impl Participant {
-    pub(super) fn new(
+    enum_dispatch!(
+        [Local, Remote];
+        pub fn sid(self: &Self) -> ParticipantSid;
+        pub fn identity(self: &Self) -> ParticipantIdentity;
+        pub fn name(self: &Self) -> String;
+        pub fn metadata(self: &Self) -> String;
+        pub fn is_speaking(self: &Self) -> bool;
+        pub fn audio_level(self: &Self) -> f32;
+        pub fn connection_quality(self: &Self) -> ConnectionQuality;
+        pub fn tracks(self: &Self) -> RwLockReadGuard<HashMap<TrackSid, TrackPublication>>;
+        pub fn register_observer(self: &Self) -> mpsc::UnboundedReceiver<ParticipantEvent>;
+
+        // Internal functions
+        pub(crate) fn set_speaking(self: &Self, speaking: bool) -> ();
+        pub(crate) fn set_audio_level(self: &Self, level: f32) -> ();
+        pub(crate) fn set_connection_quality(self: &Self, quality: ConnectionQuality) -> ();
+        pub(crate) fn update_info(self: &Self, info: proto::ParticipantInfo) -> ();
+    );
+}
+
+#[derive(Debug)]
+pub(crate) struct ParticipantInner {
+    pub sid: Mutex<ParticipantSid>,
+    pub identity: Mutex<ParticipantIdentity>,
+    pub name: Mutex<String>,
+    pub metadata: Mutex<String>,
+    pub speaking: AtomicBool,
+    pub tracks: RwLock<HashMap<TrackSid, TrackPublication>>,
+    pub audio_level: AtomicU32,
+    pub connection_quality: AtomicU8,
+    pub dispatcher: Dispatcher<ParticipantEvent>,
+}
+
+impl ParticipantInner {
+    pub fn new(
         sid: ParticipantSid,
         identity: ParticipantIdentity,
         name: String,
@@ -116,128 +149,64 @@ impl ParticipantShared {
         }
     }
 
-    pub(crate) fn update_info(&self, info: proto::ParticipantInfo) {
+    pub fn sid(&self) -> ParticipantSid {
+        self.sid.lock().clone()
+    }
+
+    pub fn identity(&self) -> ParticipantIdentity {
+        self.identity.lock().clone()
+    }
+
+    pub fn name(&self) -> String {
+        self.name.lock().clone()
+    }
+
+    pub fn metadata(&self) -> String {
+        self.metadata.lock().clone()
+    }
+
+    pub fn is_speaking(&self) -> bool {
+        self.speaking.load(Ordering::SeqCst)
+    }
+
+    pub fn tracks(&self) -> RwLockReadGuard<HashMap<TrackSid, TrackPublication>> {
+        self.tracks.read()
+    }
+
+    pub fn audio_level(&self) -> f32 {
+        f32::from_bits(self.audio_level.load(Ordering::SeqCst))
+    }
+
+    pub fn connection_quality(&self) -> ConnectionQuality {
+        self.connection_quality.load(Ordering::SeqCst).into()
+    }
+
+    pub fn register_observer(&self) -> mpsc::UnboundedReceiver<ParticipantEvent> {
+        self.dispatcher.register()
+    }
+
+    pub fn update_info(&self, info: proto::ParticipantInfo) {
         *self.sid.lock() = info.sid.into();
         *self.identity.lock() = info.identity.into();
         *self.name.lock() = info.name;
         *self.metadata.lock() = info.metadata; // TODO(theomonnom): callback MetadataChanged
     }
 
-    pub(crate) fn set_speaking(&self, speaking: bool) {
+    pub fn set_speaking(&self, speaking: bool) {
         self.speaking.store(speaking, Ordering::SeqCst);
     }
 
-    pub(crate) fn set_audio_level(&self, audio_level: f32) {
+    pub fn set_audio_level(&self, audio_level: f32) {
         self.audio_level
             .store(audio_level.to_bits(), Ordering::SeqCst)
     }
 
-    pub(crate) fn register_observer(&self) -> mpsc::UnboundedReceiver<ParticipantEvent> {
+    pub fn set_connection_quality(&self, quality: ConnectionQuality) {
-        self.dispatcher.lock().register()
-    }
-
-    pub(crate) fn set_connection_quality(&self, quality: ConnectionQuality) {
         self.connection_quality
             .store(quality as u8, Ordering::SeqCst);
     }
 
-    pub(crate) fn add_track_publication(&self, publication: TrackPublication) {
+    pub fn add_track_publication(&self, publication: TrackPublication) {
         self.tracks.write().insert(publication.sid(), publication);
     }
 }
-
-pub(crate) trait ParticipantInternalTrait {
-    fn set_speaking(&self, speaking: bool);
-    fn set_audio_level(&self, level: f32);
-    fn set_connection_quality(&self, quality: ConnectionQuality);
-    fn update_info(self: &Arc<Self>, info: proto::ParticipantInfo, emit_events: bool);
-}
-
-pub trait ParticipantTrait {
-    fn sid(&self) -> ParticipantSid;
-    fn identity(&self) -> ParticipantIdentity;
-    fn name(&self) -> String;
-    fn metadata(&self) -> String;
-    fn is_speaking(&self) -> bool;
-    fn audio_level(&self) -> f32;
-    fn connection_quality(&self) -> ConnectionQuality;
-    fn tracks(&self) -> RwLockReadGuard<HashMap<TrackSid, TrackPublication>>;
-    fn register_observer(&self) -> mpsc::UnboundedReceiver<ParticipantEvent>;
-}
-
-#[derive(Debug, Clone)]
-pub enum Participant {
-    Local(Arc<LocalParticipant>),
-    Remote(Arc<RemoteParticipant>),
-}
-
-// TODO(theomonnom): Should I provide a WeakParticipant here ?
-
-impl Participant {
-    enum_dispatch!(
-        [Local, Remote]
-        fnc!(pub(crate), update_info, &Self, [info: proto::ParticipantInfo, emit_events: bool], ());
-        fnc!(pub(crate), set_speaking, &Self, [speaking: bool], ());
-        fnc!(pub(crate), set_audio_level, &Self, [audio_level: f32], ());
-        fnc!(pub(crate), set_connection_quality, &Self, [quality: ConnectionQuality], ());
-    );
-}
-
-impl ParticipantTrait for Participant {
-    enum_dispatch!(
-        [Local, Remote]
-        fnc!(sid, &Self, [], ParticipantSid);
-        fnc!(identity, &Self, [], ParticipantIdentity);
-        fnc!(name, &Self, [], String);
-        fnc!(metadata, &Self, [], String);
-        fnc!(is_speaking, &Self, [], bool);
-        fnc!(audio_level, &Self, [], f32);
-        fnc!(connection_quality, &Self, [], ConnectionQuality);
-        fnc!(tracks, &Self, [], RwLockReadGuard<HashMap<TrackSid, TrackPublication>>);
-        fnc!(register_observer, &Self, [], mpsc::UnboundedReceiver<ParticipantEvent>);
-    );
-}
-
-macro_rules! impl_participant_trait {
-    ($x:ty) => {
-        impl crate::room::participant::ParticipantTrait for $x {
-            fn sid(&self) -> ParticipantSid {
-                self.shared.sid.lock().clone()
-            }
-
-            fn identity(&self) -> ParticipantIdentity {
-                self.shared.identity.lock().clone()
-            }
-
-            fn name(&self) -> String {
-                self.shared.name.lock().clone()
-            }
-
-            fn metadata(&self) -> String {
-                self.shared.metadata.lock().clone()
-            }
-
-            fn is_speaking(&self) -> bool {
-                self.shared.speaking.load(Ordering::SeqCst)
-            }
-
-            fn audio_level(&self) -> f32 {
-                f32::from_bits(self.shared.audio_level.load(Ordering::SeqCst))
-            }
-
-            fn connection_quality(&self) -> ConnectionQuality {
-                self.shared.connection_quality.load(Ordering::SeqCst).into()
-            }
-
-            fn tracks(&self) -> RwLockReadGuard<HashMap<TrackSid, TrackPublication>> {
-                self.shared.tracks.read()
-            }
-
-            fn register_observer(&self) -> mpsc::UnboundedReceiver<ParticipantEvent> {
-                self.shared.register_observer()
-            }
-        }
-    };
-}
-
-pub(super) use impl_participant_trait;

livekit/src/room/participant/remote_participant.rs

@@ -1,15 +1,11 @@
-use super::{
+use super::{ConnectionQuality, ParticipantInner};
-    impl_participant_trait, ConnectionQuality, ParticipantInternalTrait, ParticipantShared,
-};
 use crate::prelude::*;
 use crate::proto;
-use crate::publication::TrackPublicationInternalTrait;
 use crate::track::TrackError;
-use livekit_webrtc::prelude::*;
+use livekit_webrtc as rtc;
 use parking_lot::RwLockReadGuard;
-use std::collections::HashMap;
+use rtc::prelude::MediaStreamTrack;
-use std::collections::HashSet;
+use std::collections::{HashMap, HashSet};
-use std::sync::atomic::Ordering;
 use std::sync::Arc;
 use std::time::Duration;
 use tokio::sync::mpsc;
@@ -18,9 +14,9 @@ use tracing::{debug, error, instrument, Level};
 
 const ADD_TRACK_TIMEOUT: Duration = Duration::from_secs(5);
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct RemoteParticipant {
-    shared: ParticipantShared,
+    inner: Arc<ParticipantInner>,
 }
 
 impl RemoteParticipant {
@@ -31,26 +27,25 @@ impl RemoteParticipant {
         metadata: String,
     ) -> Self {
         Self {
-            shared: ParticipantShared::new(sid, identity, name, metadata),
+            inner: Arc::new(ParticipantInner::new(sid, identity, name, metadata)),
         }
     }
 
-    fn get_track_publication(&self, sid: &TrackSid) -> Option<RemoteTrackPublication> {
+    #[inline]
-        self.shared.tracks.read().get(sid).map(|track| {
+    pub fn get_track_publication(&self, sid: &TrackSid) -> Option<RemoteTrackPublication> {
+        self.inner.tracks.read().get(sid).map(|track| {
             if let TrackPublication::Remote(remote) = track {
-                remote.clone()
+                return remote.clone();
-            } else {
-                unreachable!()
             }
+            unreachable!()
         })
     }
 
-    /// Called by the RoomSession when receiving data by the RTCSession
+    /// Called by the RoomSession when receiving data from the RrcSession
     /// It is just used to emit the Data event on the participant dispatcher.
     pub(crate) fn on_data_received(&self, data: Arc<Vec<u8>>, kind: proto::data_packet::Kind) {
-        self.shared
+        self.inner
             .dispatcher
-            .lock()
             .dispatch(&ParticipantEvent::DataReceived {
                 payload: data,
                 kind,
@@ -59,9 +54,9 @@ impl RemoteParticipant {
 
     #[instrument(level = Level::DEBUG)]
     pub(crate) async fn add_subscribed_media_track(
-        self: Arc<Self>,
+        &self,
         sid: TrackSid,
-        media_track: MediaStreamTrackHandle,
+        media_track: rtc::media_stream::MediaStreamTrack,
     ) {
         let wait_publication = {
             let participant = self.clone();
@@ -73,7 +68,7 @@ impl RemoteParticipant {
                         return publication;
                     }
 
-                    tokio::task::yield_now().await;
+                    tokio::task::yield_now().await; // Remove yield
                 }
             }
         };
@@ -81,25 +76,25 @@ impl RemoteParticipant {
         if let Ok(remote_publication) = timeout(ADD_TRACK_TIMEOUT, wait_publication).await {
             let track = match remote_publication.kind() {
                 TrackKind::Audio => {
-                    if let MediaStreamTrackHandle::Audio(rtc_track) = media_track {
+                    if let MediaStreamTrack::Audio(rtc_track) = media_track {
                         let audio_track = RemoteAudioTrack::new(
                             remote_publication.sid().into(),
                             remote_publication.name(),
                             rtc_track,
                         );
-                        RemoteTrackHandle::Audio(Arc::new(audio_track))
+                        RemoteTrack::Audio(audio_track)
                     } else {
                         unreachable!();
                     }
                 }
                 TrackKind::Video => {
-                    if let MediaStreamTrackHandle::Video(rtc_track) = media_track {
+                    if let MediaStreamTrack::Video(rtc_track) = media_track {
                         let video_track = RemoteVideoTrack::new(
                             remote_publication.sid().into(),
                             remote_publication.name(),
                             rtc_track,
                         );
-                        RemoteTrackHandle::Video(Arc::new(video_track))
+                        RemoteTrack::Video(video_track)
                     } else {
                         unreachable!()
                     }
@@ -110,23 +105,30 @@ impl RemoteParticipant {
             debug!("starting track: {:?}", sid);
 
             remote_publication.update_track(Some(track.clone().into()));
-            self.shared
+            track.set_muted(remote_publication.muted());
+            track.update_info(proto::TrackInfo {
+                sid: remote_publication.sid().to_string(),
+                name: remote_publication.name().to_string(),
+                r#type: proto::TrackType::from(remote_publication.kind()) as i32,
+                source: proto::TrackSource::from(remote_publication.source()) as i32,
+                ..Default::default()
+            });
+
+            self.inner
                 .add_track_publication(TrackPublication::Remote(remote_publication.clone()));
             track.start();
 
-            self.shared
+            self.inner
                 .dispatcher
-                .lock()
                 .dispatch(&ParticipantEvent::TrackSubscribed {
-                    track: track,
+                    track,
                     publication: remote_publication,
                 });
         } else {
             error!("could not find published track with sid: {:?}", sid);
 
-            self.shared
+            self.inner
                 .dispatcher
-                .lock()
                 .dispatch(&ParticipantEvent::TrackSubscriptionFailed {
                     sid: sid.clone(),
                     error: TrackError::TrackNotFound(sid.clone().to_string()),
@@ -134,81 +136,118 @@ impl RemoteParticipant {
         }
     }
 
-    pub(crate) fn unpublish_track(self: &Arc<Self>, sid: &TrackSid, emit_events: bool) {
+    pub(crate) fn unpublish_track(&self, sid: &TrackSid) {
         if let Some(publication) = self.get_track_publication(sid) {
             // Unsubscribe to the track if needed
             if let Some(track) = publication.track() {
                 track.stop();
 
-                self.shared
+                self.inner
                     .dispatcher
-                    .lock()
                     .dispatch(&ParticipantEvent::TrackUnsubscribed {
                         track: track.clone(),
                         publication: publication.clone(),
                     });
             }
 
-            if emit_events {
+            self.inner
-                self.shared
                 .dispatcher
-                    .lock()
                 .dispatch(&ParticipantEvent::TrackUnpublished {
                     publication: publication.clone(),
                 });
-            }
 
             publication.update_track(None);
         }
     }
-}
 
-impl ParticipantInternalTrait for RemoteParticipant {
+    pub(crate) fn update_info(&self, info: proto::ParticipantInfo) {
-    fn update_info(self: &Arc<Self>, info: proto::ParticipantInfo, emit_events: bool) {
+        self.inner.update_info(info.clone());
-        self.shared.update_info(info.clone());
 
         let mut valid_tracks = HashSet::<TrackSid>::new();
         for track in info.tracks {
             if let Some(publication) = self.get_track_publication(&track.sid.clone().into()) {
                 publication.update_info(track.clone());
             } else {
-                let publication = RemoteTrackPublication::new(track.clone(), self.sid(), None);
+                let publication = RemoteTrackPublication::new(track.clone(), None);
-                self.shared
+                self.inner
                     .add_track_publication(TrackPublication::Remote(publication.clone()));
 
                 // This is a new track, dispatch publish event
-                if emit_events {
+                self.inner
-                    self.shared
                     .dispatcher
-                        .lock()
                     .dispatch(&ParticipantEvent::TrackPublished { publication });
             }
-            }
 
             valid_tracks.insert(track.sid.into());
         }
 
         // remove tracks that are no longer valid
-        for (sid, _) in self.shared.tracks.read().iter() {
+        for (sid, _) in self.inner.tracks.read().iter() {
             if valid_tracks.contains(sid) {
                 continue;
             }
 
-            self.unpublish_track(sid, emit_events);
+            self.unpublish_track(sid);
         }
     }
 
-    fn set_speaking(&self, speaking: bool) {
+    #[inline]
-        self.shared.set_speaking(speaking);
+    pub fn sid(&self) -> ParticipantSid {
+        self.inner.sid()
     }
 
-    fn set_audio_level(&self, level: f32) {
+    #[inline]
-        self.shared.set_audio_level(level);
+    pub fn identity(&self) -> ParticipantIdentity {
+        self.inner.identity()
     }
 
-    fn set_connection_quality(&self, quality: ConnectionQuality) {
+    #[inline]
-        self.shared.set_connection_quality(quality);
+    pub fn name(&self) -> String {
+        self.inner.name()
+    }
+
+    #[inline]
+    pub fn metadata(&self) -> String {
+        self.inner.metadata()
+    }
+
+    #[inline]
+    pub fn is_speaking(&self) -> bool {
+        self.inner.is_speaking()
+    }
+
+    #[inline]
+    pub fn tracks(&self) -> RwLockReadGuard<HashMap<TrackSid, TrackPublication>> {
+        self.inner.tracks()
+    }
+
+    #[inline]
+    pub fn audio_level(&self) -> f32 {
+        self.inner.audio_level()
+    }
+
+    #[inline]
+    pub fn connection_quality(&self) -> ConnectionQuality {
+        self.inner.connection_quality()
+    }
+
+    #[inline]
+    pub fn register_observer(&self) -> mpsc::UnboundedReceiver<ParticipantEvent> {
+        self.inner.register_observer()
+    }
+
+    #[inline]
+    pub(crate) fn set_speaking(&self, speaking: bool) {
+        self.inner.set_speaking(speaking);
+    }
+
+    #[inline]
+    pub(crate) fn set_audio_level(&self, level: f32) {
+        self.inner.set_audio_level(level);
+    }
+
+    #[inline]
+    pub(crate) fn set_connection_quality(&self, quality: ConnectionQuality) {
+        self.inner.set_connection_quality(quality);
     }
 }
-
-impl_participant_trait!(RemoteParticipant);

livekit/src/room/publication/local.rs

@@ -0,0 +1,91 @@
+use super::TrackPublicationInner;
+use crate::id::TrackSid;
+use crate::options::TrackPublishOptions;
+use crate::proto;
+use crate::track::{LocalTrack, Track, TrackDimension, TrackKind, TrackSource};
+use parking_lot::Mutex;
+use std::sync::Arc;
+
+#[derive(Debug)]
+struct LocalTrackPublicationInner {
+    publication_inner: TrackPublicationInner,
+    options: Mutex<TrackPublishOptions>,
+}
+
+#[derive(Clone, Debug)]
+pub struct LocalTrackPublication {
+    inner: Arc<LocalTrackPublicationInner>,
+}
+
+impl LocalTrackPublication {
+    pub(crate) fn new(
+        info: proto::TrackInfo,
+        track: LocalTrack,
+        options: TrackPublishOptions,
+    ) -> Self {
+        Self {
+            inner: Arc::new(LocalTrackPublicationInner {
+                publication_inner: TrackPublicationInner::new(info, Some(track.into())),
+                options: Mutex::new(options),
+            }),
+        }
+    }
+
+    #[inline]
+    pub fn sid(&self) -> TrackSid {
+        self.inner.publication_inner.sid()
+    }
+
+    #[inline]
+    pub fn name(&self) -> String {
+        self.inner.publication_inner.name()
+    }
+
+    #[inline]
+    pub fn kind(&self) -> TrackKind {
+        self.inner.publication_inner.kind()
+    }
+
+    #[inline]
+    pub fn source(&self) -> TrackSource {
+        self.inner.publication_inner.source()
+    }
+
+    #[inline]
+    pub fn simulcasted(&self) -> bool {
+        self.inner.publication_inner.simulcasted()
+    }
+
+    #[inline]
+    pub fn dimension(&self) -> TrackDimension {
+        self.inner.publication_inner.dimension()
+    }
+
+    #[inline]
+    pub fn track(&self) -> Option<LocalTrack> {
+        self.inner
+            .publication_inner
+            .track()
+            .map(|track| track.try_into().unwrap())
+    }
+
+    #[inline]
+    pub fn mime_type(&self) -> String {
+        self.inner.publication_inner.mime_type()
+    }
+
+    #[inline]
+    pub fn muted(&self) -> bool {
+        self.inner.publication_inner.muted()
+    }
+
+    #[inline]
+    pub(crate) fn update_track(&self, track: Option<Track>) {
+        self.inner.publication_inner.update_track(track);
+    }
+
+    #[inline]
+    pub(crate) fn update_info(&self, info: proto::TrackInfo) {
+        self.inner.publication_inner.update_info(info);
+    }
+}

livekit/src/room/publication/mod.rs

@@ -1,58 +1,46 @@
+use super::track::{TrackDimension, TrackEvent};
 use crate::prelude::*;
 use crate::proto;
+use crate::track::Track;
+use futures_util::stream::StreamExt;
 use livekit_utils::enum_dispatch;
 use livekit_utils::observer::Dispatcher;
 use parking_lot::Mutex;
 use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
 use std::sync::Arc;
-use tokio::sync::{mpsc, oneshot};
+use tokio::sync::Notify;
+use tokio_stream::wrappers::UnboundedReceiverStream;
 
-use super::track::{TrackDimension, TrackEvent};
+mod local;
+pub use local::*;
 
-pub(crate) trait TrackPublicationInternalTrait {
+mod remote;
-    fn update_track(&self, track: Option<TrackHandle>);
+pub use remote::*;
-    fn update_info(&self, info: proto::TrackInfo);
-}
-
-pub trait TrackPublicationTrait {
-    fn name(&self) -> String;
-    fn sid(&self) -> TrackSid;
-    fn kind(&self) -> TrackKind;
-    fn source(&self) -> TrackSource;
-    fn simulcasted(&self) -> bool;
-    fn dimension(&self) -> TrackDimension;
-    fn mime_type(&self) -> String;
-    fn muted(&self) -> bool;
-}
 
 #[derive(Debug)]
-pub(super) struct TrackPublicationShared {
+pub(crate) struct TrackPublicationInner {
-    pub(super) track: Mutex<Option<TrackHandle>>,
+    track: Mutex<Option<Track>>,
-    pub(super) name: Mutex<String>,
+    name: Mutex<String>,
-    pub(super) sid: Mutex<TrackSid>,
+    sid: Mutex<TrackSid>,
-    pub(super) kind: AtomicU8,   // Casted to TrackKind
+    kind: AtomicU8,   // Casted to TrackKind
-    pub(super) source: AtomicU8, // Casted to TrackSource
+    source: AtomicU8, // Casted to TrackSource
-    pub(super) simulcasted: AtomicBool,
+    simulcasted: AtomicBool,
-    pub(super) dimension: Mutex<TrackDimension>,
+    dimension: Mutex<TrackDimension>,
-    pub(super) mime_type: Mutex<String>,
+    mime_type: Mutex<String>,
-    pub(super) muted: AtomicBool,
+    muted: AtomicBool,
-    pub(super) participant: ParticipantSid,
+    dispatcher: Dispatcher<TrackEvent>,
-    pub(super) dispatcher: Mutex<Dispatcher<TrackEvent>>,
+    close_notifier: Arc<Notify>,
-    pub(super) close_sender: Mutex<Option<oneshot::Sender<()>>>,
 }
 
-impl TrackPublicationShared {
+impl TrackPublicationInner {
-    pub fn new(
+    pub fn new(info: proto::TrackInfo, track: Option<Track>) -> Self {
-        info: proto::TrackInfo,
+        Self {
-        participant: ParticipantSid,
-        track: Option<TrackHandle>,
-    ) -> Arc<Self> {
-        Arc::new(Self {
             track: Mutex::new(track),
             name: Mutex::new(info.name),
             sid: Mutex::new(info.sid.into()),
             kind: AtomicU8::new(
-                TrackKind::from(proto::TrackType::from_i32(info.r#type).unwrap()) as u8,
+                TrackKind::try_from(proto::TrackType::from_i32(info.r#type).unwrap()).unwrap()
+                    as u8,
             ),
             source: AtomicU8::new(TrackSource::from(
                 proto::TrackSource::from_i32(info.source).unwrap(),
@@ -62,46 +50,32 @@ impl TrackPublicationShared {
             mime_type: Mutex::new(info.mime_type),
             muted: AtomicBool::new(info.muted),
             dispatcher: Default::default(),
-            close_sender: Default::default(),
+            close_notifier: Default::default(),
-            participant,
+        }
-        })
     }
 
-    pub fn update_track(self: &Arc<Self>, track: Option<TrackHandle>) {
+    pub fn update_track(&self, track: Option<Track>) {
         let mut old_track = self.track.lock();
-
-        if let Some(close_sender) = self.close_sender.lock().take() {
-            let _ = close_sender.send(());
-        }
-
         *old_track = track.clone();
-        if let Some(track) = track {
-            let (close_sender, close_receiver) = oneshot::channel();
-            self.close_sender.lock().replace(close_sender);
 
-            let track_receiver = track.register_observer();
+        self.close_notifier.notify_waiters();
-            tokio::spawn(
-                self.clone()
-                    .publication_task(close_receiver, track_receiver),
-            );
-        }
-    }
 
-    /// Task used to forward TrackHandle's events to the TrackPublications's dispatcher
+        if let Some(track) = track.as_ref() {
-    async fn publication_task(
+            let track_stream = UnboundedReceiverStream::new(track.register_observer());
-        self: Arc<Self>,
+            tokio::spawn({
-        mut close_receiver: oneshot::Receiver<()>,
+                let dispatcher = self.dispatcher.clone();
-        mut track_receiver: mpsc::UnboundedReceiver<TrackEvent>,
+                let notifier = self.close_notifier.clone();
-    ) {
+
-        loop {
+                async move {
-            tokio::select! {
+                    let notified = notifier.notified();
-                Some(event) = track_receiver.recv() => {
+                    futures_util::pin_mut!(notified);
-                    self.dispatcher.lock().dispatch(&event);
+                    futures_util::future::select(
-                }
+                        track_stream.map(Ok).forward(dispatcher),
-                _ = &mut close_receiver => {
+                        notified,
-                    break;
+                    )
-                }
+                    .await;
                 }
+            });
         }
     }
 
@@ -111,7 +85,7 @@ impl TrackPublicationShared {
         *self.dimension.lock() = TrackDimension(info.width, info.height);
         *self.mime_type.lock() = info.mime_type;
         self.kind.store(
-            TrackKind::from(proto::TrackType::from_i32(info.r#type).unwrap()) as u8,
+            TrackKind::try_from(proto::TrackType::from_i32(info.r#type).unwrap()).unwrap() as u8,
             Ordering::SeqCst,
         );
         self.source.store(
@@ -125,13 +99,41 @@ impl TrackPublicationShared {
             track.set_muted(info.muted);
         }
     }
-}
 
-impl Drop for TrackPublicationShared {
+    pub fn sid(&self) -> TrackSid {
-    fn drop(&mut self) {
+        self.sid.lock().clone()
-        if let Some(close_sender) = self.close_sender.lock().take() {
-            let _ = close_sender.send(());
     }
+
+    pub fn name(&self) -> String {
+        self.name.lock().clone()
+    }
+
+    pub fn kind(&self) -> TrackKind {
+        self.kind.load(Ordering::SeqCst).try_into().unwrap()
+    }
+
+    pub fn source(&self) -> TrackSource {
+        self.source.load(Ordering::SeqCst).into()
+    }
+
+    pub fn simulcasted(&self) -> bool {
+        self.simulcasted.load(Ordering::Relaxed)
+    }
+
+    pub fn dimension(&self) -> TrackDimension {
+        self.dimension.lock().clone()
+    }
+
+    pub fn mime_type(&self) -> String {
+        self.mime_type.lock().clone()
+    }
+
+    pub fn track(&self) -> Option<Track> {
+        self.track.lock().clone()
+    }
+
+    pub fn muted(&self) -> bool {
+        self.muted.load(Ordering::Relaxed)
     }
 }
 
@@ -142,134 +144,22 @@ pub enum TrackPublication {
 }
 
 impl TrackPublication {
-    pub fn track(&self) -> Option<TrackHandle> {
+    enum_dispatch!(
-        // Not calling Local/Remote function here, we don't need "cast"
+        [Local, Remote];
+        pub fn sid(self: &Self) -> TrackSid;
+        pub fn name(self: &Self) -> String;
+        pub fn kind(self: &Self) -> TrackKind;
+        pub fn source(self: &Self) -> TrackSource;
+        pub fn simulcasted(self: &Self) -> bool;
+        pub fn dimension(self: &Self) -> TrackDimension;
+        pub fn mime_type(self: &Self) -> String;
+        pub fn muted(self: &Self) -> bool;
+    );
+
+    pub fn track(&self) -> Option<Track> {
         match self {
-            TrackPublication::Local(p) => p.shared.track.lock().clone(),
+            TrackPublication::Local(p) => p.track().map(Into::into),
-            TrackPublication::Remote(p) => p.shared.track.lock().clone(),
+            TrackPublication::Remote(p) => p.track().map(Into::into),
         }
     }
 }
-
-impl TrackPublicationInternalTrait for TrackPublication {
-    enum_dispatch!(
-        [Local, Remote]
-        fnc!(update_track, &Self, [track: Option<TrackHandle>], ());
-        fnc!(update_info, &Self, [info: proto::TrackInfo], ());
-    );
-}
-
-impl TrackPublicationTrait for TrackPublication {
-    enum_dispatch!(
-        [Local, Remote]
-        fnc!(sid, &Self, [], TrackSid);
-        fnc!(name, &Self, [], String);
-        fnc!(kind, &Self, [], TrackKind);
-        fnc!(source, &Self, [], TrackSource);
-        fnc!(simulcasted, &Self, [], bool);
-        fnc!(dimension, &Self, [], TrackDimension);
-        fnc!(mime_type, &Self, [], String);
-        fnc!(muted, &Self, [], bool);
-    );
-}
-
-macro_rules! impl_publication_trait {
-    ($x:ident) => {
-        impl TrackPublicationTrait for $x {
-            fn name(&self) -> String {
-                self.shared.name.lock().clone()
-            }
-
-            fn sid(&self) -> TrackSid {
-                self.shared.sid.lock().clone()
-            }
-
-            fn kind(&self) -> TrackKind {
-                self.shared.kind.load(Ordering::SeqCst).into()
-            }
-
-            fn source(&self) -> TrackSource {
-                self.shared.source.load(Ordering::SeqCst).into()
-            }
-
-            fn simulcasted(&self) -> bool {
-                self.shared.simulcasted.load(Ordering::SeqCst)
-            }
-
-            fn dimension(&self) -> TrackDimension {
-                self.shared.dimension.lock().clone()
-            }
-
-            fn mime_type(&self) -> String {
-                self.shared.mime_type.lock().clone()
-            }
-
-            fn muted(&self) -> bool {
-                self.shared.muted.load(Ordering::SeqCst)
-            }
-        }
-    };
-}
-
-#[derive(Clone, Debug)]
-pub struct LocalTrackPublication {
-    shared: Arc<TrackPublicationShared>,
-}
-
-impl LocalTrackPublication {
-    pub fn track(&self) -> Option<LocalTrackHandle> {
-        self.shared
-            .track
-            .lock()
-            .clone()
-            .map(|local_track| local_track.try_into().unwrap())
-    }
-}
-
-impl TrackPublicationInternalTrait for LocalTrackPublication {
-    fn update_track(&self, track: Option<TrackHandle>) {
-        self.shared.update_track(track);
-    }
-
-    fn update_info(&self, info: proto::TrackInfo) {
-        self.shared.update_info(info);
-    }
-}
-
-#[derive(Clone, Debug)]
-pub struct RemoteTrackPublication {
-    shared: Arc<TrackPublicationShared>,
-}
-
-impl RemoteTrackPublication {
-    pub fn new(
-        info: proto::TrackInfo,
-        participant: ParticipantSid,
-        track: Option<TrackHandle>,
-    ) -> Self {
-        Self {
-            shared: TrackPublicationShared::new(info, participant, track),
-        }
-    }
-
-    pub fn track(&self) -> Option<RemoteTrackHandle> {
-        self.shared
-            .track
-            .lock()
-            .clone()
-            .map(|track| track.try_into().unwrap())
-    }
-}
-
-impl TrackPublicationInternalTrait for RemoteTrackPublication {
-    fn update_track(&self, track: Option<TrackHandle>) {
-        self.shared.update_track(track);
-    }
-
-    fn update_info(&self, info: proto::TrackInfo) {
-        self.shared.update_info(info);
-    }
-}
-
-impl_publication_trait!(LocalTrackPublication);
-impl_publication_trait!(RemoteTrackPublication);

livekit/src/room/publication/remote.rs

@@ -0,0 +1,73 @@
+use super::TrackPublicationInner;
+use crate::id::TrackSid;
+use crate::proto;
+use crate::track::{RemoteTrack, Track, TrackDimension, TrackKind, TrackSource};
+use std::sync::Arc;
+
+#[derive(Clone, Debug)]
+pub struct RemoteTrackPublication {
+    inner: Arc<TrackPublicationInner>,
+}
+
+impl RemoteTrackPublication {
+    pub(crate) fn new(info: proto::TrackInfo, track: Option<RemoteTrack>) -> Self {
+        Self {
+            inner: Arc::new(TrackPublicationInner::new(info, track.map(Into::into))),
+        }
+    }
+
+    #[inline]
+    pub fn sid(&self) -> TrackSid {
+        self.inner.sid()
+    }
+
+    #[inline]
+    pub fn name(&self) -> String {
+        self.inner.name()
+    }
+
+    #[inline]
+    pub fn kind(&self) -> TrackKind {
+        self.inner.kind()
+    }
+
+    #[inline]
+    pub fn source(&self) -> TrackSource {
+        self.inner.source()
+    }
+
+    #[inline]
+    pub fn simulcasted(&self) -> bool {
+        self.inner.simulcasted()
+    }
+
+    #[inline]
+    pub fn dimension(&self) -> TrackDimension {
+        self.inner.dimension()
+    }
+
+    #[inline]
+    pub fn track(&self) -> Option<RemoteTrack> {
+        self.inner.track().map(|track| track.try_into().unwrap())
+    }
+
+    #[inline]
+    pub fn mime_type(&self) -> String {
+        self.inner.mime_type()
+    }
+
+    #[inline]
+    pub fn muted(&self) -> bool {
+        self.inner.muted()
+    }
+
+    #[inline]
+    pub(crate) fn update_track(&self, track: Option<Track>) {
+        self.inner.update_track(track);
+    }
+
+    #[inline]
+    pub(crate) fn update_info(&self, info: proto::TrackInfo) {
+        self.inner.update_info(info);
+    }
+}

livekit/src/room/room_session.rs

@@ -1,7 +1,7 @@
-use crate::participant::{ConnectionQuality, ParticipantInternalTrait};
+use crate::participant::ConnectionQuality;
 use crate::prelude::*;
 use crate::proto;
-use crate::rtc_engine::{EngineEvent, EngineEvents, EngineResult, RTCEngine};
+use crate::rtc_engine::{EngineEvent, EngineEvents, EngineResult, RtcEngine};
 use crate::signal_client::SignalOptions;
 use crate::{RoomError, RoomEvent, RoomResult, SimulateScenario};
 use livekit_utils::observer::Dispatcher;
@@ -9,8 +9,7 @@ use parking_lot::{Mutex, RwLock, RwLockReadGuard};
 use std::collections::HashMap;
 use std::sync::atomic::{AtomicU8, Ordering};
 use std::sync::Arc;
-use tokio::sync::mpsc;
+use tokio::sync::{mpsc, oneshot};
-use tokio::sync::oneshot;
 use tokio::task::JoinHandle;
 use tracing::{error, info, instrument, Level};
 
@@ -22,17 +21,6 @@ pub enum ConnectionState {
     Unknown,
 }
 
-impl From<u8> for ConnectionState {
-    fn from(value: u8) -> Self {
-        match value {
-            0 => ConnectionState::Disconnected,
-            1 => ConnectionState::Connected,
-            2 => ConnectionState::Reconnecting,
-            _ => ConnectionState::Unknown,
-        }
-    }
-}
-
 /// Internal representation of a RoomSession
 #[derive(Debug)]
 struct SessionInner {
@@ -40,12 +28,12 @@ struct SessionInner {
     sid: Mutex<RoomSid>,
     name: Mutex<String>,
     metadata: Mutex<String>,
-    participants: RwLock<HashMap<ParticipantSid, Arc<RemoteParticipant>>>,
+    participants: RwLock<HashMap<ParticipantSid, RemoteParticipant>>,
     participants_tasks: RwLock<HashMap<ParticipantSid, (JoinHandle<()>, oneshot::Sender<()>)>>,
     active_speakers: RwLock<Vec<Participant>>,
-    rtc_engine: Arc<RTCEngine>,
+    rtc_engine: Arc<RtcEngine>,
-    local_participant: Arc<LocalParticipant>,
+    local_participant: LocalParticipant,
-    dispatcher: Mutex<Dispatcher<RoomEvent>>,
+    dispatcher: Dispatcher<RoomEvent>,
 }
 
 #[derive(Debug)]
@@ -64,7 +52,7 @@ pub struct RoomSession {
 
 impl SessionHandle {
     pub async fn connect(url: &str, token: &str) -> RoomResult<Self> {
-        let (rtc_engine, engine_events) = RTCEngine::new();
+        let (rtc_engine, engine_events) = RtcEngine::new();
         let rtc_engine = Arc::new(rtc_engine);
         rtc_engine
             .connect(url, token, SignalOptions::default())
@@ -72,13 +60,13 @@ impl SessionHandle {
 
         let join_response = rtc_engine.join_response().unwrap();
         let pi = join_response.participant.unwrap().clone();
-        let local_participant = Arc::new(LocalParticipant::new(
+        let local_participant = LocalParticipant::new(
             rtc_engine.clone(),
             pi.sid.into(),
             pi.identity.into(),
             pi.name,
             pi.metadata,
-        ));
+        );
 
         let room_info = join_response.room.unwrap();
         let inner = Arc::new(SessionInner {
@@ -99,7 +87,7 @@ impl SessionHandle {
                 let pi = pi.clone();
                 inner.create_participant(pi.sid.into(), pi.identity.into(), pi.name, pi.metadata)
             };
-            participant.update_info(pi.clone(), false);
+            participant.update_info(pi.clone());
         }
 
         let (close_emitter, close_receiver) = oneshot::channel();
@@ -108,7 +96,7 @@ impl SessionHandle {
         inner.update_connection_state(ConnectionState::Connected);
 
         let session = Self {
-            session: RoomSession::from(inner),
+            session: RoomSession { inner },
             session_task,
             close_emitter,
         };
@@ -122,7 +110,7 @@ impl SessionHandle {
     }
 
     pub fn subscribe(&self) -> mpsc::UnboundedReceiver<RoomEvent> {
-        self.session.inner.dispatcher.lock().register()
+        self.session.inner.dispatcher.register()
     }
 
     pub fn session(&self) -> RoomSession {
@@ -131,10 +119,6 @@ impl SessionHandle {
 }
 
 impl RoomSession {
-    fn from(inner: Arc<SessionInner>) -> Self {
-        Self { inner }
-    }
-
     pub fn sid(&self) -> RoomSid {
         self.inner.sid.lock().clone()
     }
@@ -147,7 +131,7 @@ impl RoomSession {
         self.inner.metadata.lock().clone()
     }
 
-    pub fn local_participant(&self) -> Arc<LocalParticipant> {
+    pub fn local_participant(&self) -> LocalParticipant {
         self.inner.local_participant.clone()
     }
 
@@ -155,7 +139,7 @@ impl RoomSession {
         self.inner.state.load(Ordering::Acquire).try_into().unwrap()
     }
 
-    pub fn participants(&self) -> RwLockReadGuard<HashMap<ParticipantSid, Arc<RemoteParticipant>>> {
+    pub fn participants(&self) -> RwLockReadGuard<HashMap<ParticipantSid, RemoteParticipant>> {
         self.inner.participants.read()
     }
 
@@ -226,32 +210,26 @@ impl SessionInner {
         if let Participant::Remote(remote_participant) = participant {
             match event {
                 ParticipantEvent::TrackPublished { publication } => {
-                    self.dispatcher.lock().dispatch(&RoomEvent::TrackPublished {
+                    self.dispatcher.dispatch(&RoomEvent::TrackPublished {
                         participant: remote_participant.clone(),
                         publication,
                     });
                 }
                 ParticipantEvent::TrackUnpublished { publication } => {
-                    self.dispatcher
+                    self.dispatcher.dispatch(&RoomEvent::TrackUnpublished {
-                        .lock()
-                        .dispatch(&RoomEvent::TrackUnpublished {
                         participant: remote_participant.clone(),
                         publication,
                     });
                 }
                 ParticipantEvent::TrackSubscribed { track, publication } => {
-                    self.dispatcher
+                    self.dispatcher.dispatch(&RoomEvent::TrackSubscribed {
-                        .lock()
-                        .dispatch(&RoomEvent::TrackSubscribed {
                         participant: remote_participant.clone(),
                         track,
                         publication,
                     });
                 }
                 ParticipantEvent::TrackUnsubscribed { track, publication } => {
-                    self.dispatcher
+                    self.dispatcher.dispatch(&RoomEvent::TrackUnsubscribed {
-                        .lock()
-                        .dispatch(&RoomEvent::TrackUnsubscribed {
                         participant: remote_participant.clone(),
                         track,
                         publication,
@@ -303,12 +281,12 @@ impl SessionInner {
             }
             EngineEvent::Resuming => {
                 if self.update_connection_state(ConnectionState::Reconnecting) {
-                    self.dispatcher.lock().dispatch(&RoomEvent::Reconnecting);
+                    self.dispatcher.dispatch(&RoomEvent::Reconnecting);
                 }
             }
             EngineEvent::Resumed => {
                 self.update_connection_state(ConnectionState::Connected);
-                self.dispatcher.lock().dispatch(&RoomEvent::Reconnected);
+                self.dispatcher.dispatch(&RoomEvent::Reconnected);
 
                 // TODO(theomonnom): Update subscriptions settings
                 // TODO(theomonnom): Send sync state
@@ -323,7 +301,7 @@ impl SessionInner {
             } => {
                 let payload = Arc::new(payload);
                 if let Some(participant) = self.get_participant(&participant_sid.into()) {
-                    self.dispatcher.lock().dispatch(&RoomEvent::DataReceived {
+                    self.dispatcher.dispatch(&RoomEvent::DataReceived {
                         payload: payload.clone(),
                         kind,
                         participant: participant.clone(),
@@ -357,7 +335,6 @@ impl SessionInner {
 
         self.state.store(state as u8, Ordering::Release);
         self.dispatcher
-            .lock()
             .dispatch(&RoomEvent::ConnectionStateChanged(state));
         return true;
     }
@@ -368,11 +345,10 @@ impl SessionInner {
     #[instrument(level = Level::DEBUG)]
     fn handle_participant_update(self: &Arc<Self>, updates: Vec<proto::ParticipantInfo>) {
         for pi in updates {
-            info!("test");
             if pi.sid == self.local_participant.sid()
                 || pi.identity == self.local_participant.identity()
             {
-                self.local_participant.clone().update_info(pi, true);
+                self.local_participant.clone().update_info(pi);
                 continue;
             }
 
@@ -386,7 +362,7 @@ impl SessionInner {
                         .handle_participant_disconnect(remote_participant)
                 } else {
                     // Participant is already connected, update the it
-                    remote_participant.update_info(pi.clone(), true);
+                    remote_participant.update_info(pi.clone());
                 }
             } else {
                 // Create a new participant
@@ -398,10 +374,9 @@ impl SessionInner {
 
                 let _ = self
                     .dispatcher
-                    .lock()
                     .dispatch(&RoomEvent::ParticipantConnected(remote_participant.clone()));
 
-                remote_participant.update_info(pi.clone(), true);
+                remote_participant.update_info(pi.clone());
             }
         }
     }
@@ -438,7 +413,6 @@ impl SessionInner {
 
         let _ = self
             .dispatcher
-            .lock()
             .dispatch(&RoomEvent::ActiveSpeakersChanged { speakers });
     }
 
@@ -466,7 +440,6 @@ impl SessionInner {
 
             participant.set_connection_quality(quality);
             self.dispatcher
-                .lock()
                 .dispatch(&RoomEvent::ConnectionQualityChanged {
                     participant,
                     quality,
@@ -483,7 +456,7 @@ impl SessionInner {
         }
 
         if self.update_connection_state(ConnectionState::Reconnecting) {
-            self.dispatcher.lock().dispatch(&RoomEvent::Reconnecting);
+            self.dispatcher.dispatch(&RoomEvent::Reconnecting);
         }
     }
 
@@ -493,10 +466,10 @@ impl SessionInner {
         let join_response = self.rtc_engine.join_response().unwrap();
 
         self.update_connection_state(ConnectionState::Connected);
-        self.dispatcher.lock().dispatch(&RoomEvent::Reconnected);
+        self.dispatcher.dispatch(&RoomEvent::Reconnected);
 
         if let Some(pi) = join_response.participant {
-            self.local_participant.update_info(pi, true); // The sid may have changed
+            self.local_participant.update_info(pi); // The sid may have changed
         }
 
         self.handle_participant_update(join_response.other_participants);
@@ -511,7 +484,7 @@ impl SessionInner {
         }
 
         self.update_connection_state(ConnectionState::Disconnected);
-        self.dispatcher.lock().dispatch(&RoomEvent::Disconnected);
+        self.dispatcher.dispatch(&RoomEvent::Disconnected);
     }
 
     /// Create a new participant
@@ -523,13 +496,8 @@ impl SessionInner {
         identity: ParticipantIdentity,
         name: String,
         metadata: String,
-    ) -> Arc<RemoteParticipant> {
+    ) -> RemoteParticipant {
-        let participant = Arc::new(RemoteParticipant::new(
+        let participant = RemoteParticipant::new(sid.clone(), identity, name, metadata);
-            sid.clone(),
-            identity,
-            name,
-            metadata,
-        ));
 
         // Create the participant task
         let (close_tx, close_rx) = oneshot::channel();
@@ -549,10 +517,10 @@ impl SessionInner {
     /// A participant has disconnected
     /// Cleanup the participant and emit an event
     #[instrument(level = Level::DEBUG)]
-    fn handle_participant_disconnect(self: Arc<Self>, remote_participant: Arc<RemoteParticipant>) {
+    fn handle_participant_disconnect(self: Arc<Self>, remote_participant: RemoteParticipant) {
         tokio::spawn(async move {
             for (sid, _) in &*remote_participant.tracks() {
-                remote_participant.unpublish_track(&sid, true);
+                remote_participant.unpublish_track(&sid);
             }
 
             // Close the participant task
@@ -566,16 +534,12 @@ impl SessionInner {
             }
 
             self.participants.write().remove(&remote_participant.sid());
-
             self.dispatcher
-                .lock()
+                .dispatch(&RoomEvent::ParticipantDisconnected(remote_participant));
-                .dispatch(&RoomEvent::ParticipantDisconnected(
-                    remote_participant.clone(),
-                ));
         });
     }
 
-    fn get_participant(&self, sid: &ParticipantSid) -> Option<Arc<RemoteParticipant>> {
+    fn get_participant(&self, sid: &ParticipantSid) -> Option<RemoteParticipant> {
         self.participants.read().get(sid).cloned()
     }
 }
@@ -590,3 +554,14 @@ fn unpack_stream_id(stream_id: &str) -> Option<(&str, &str)> {
         None
     }
 }
+
+impl From<u8> for ConnectionState {
+    fn from(value: u8) -> Self {
+        match value {
+            0 => ConnectionState::Disconnected,
+            1 => ConnectionState::Connected,
+            2 => ConnectionState::Reconnecting,
+            _ => ConnectionState::Unknown,
+        }
+    }
+}

livekit/src/room/track/audio_track.rs

@@ -1,32 +0,0 @@
-use super::impl_track_trait;
-use crate::prelude::*;
-use std::sync::Arc;
-
-#[derive(Clone)]
-pub enum AudioTrackHandle {
-    Local(Arc<LocalAudioTrack>),
-    Remote(Arc<RemoteAudioTrack>),
-}
-
-impl From<AudioTrackHandle> for TrackHandle {
-    fn from(audio_track: AudioTrackHandle) -> Self {
-        match audio_track {
-            AudioTrackHandle::Local(local_audio) => Self::LocalAudio(local_audio),
-            AudioTrackHandle::Remote(remote_audio) => Self::RemoteAudio(remote_audio),
-        }
-    }
-}
-
-impl TryFrom<TrackHandle> for AudioTrackHandle {
-    type Error = &'static str;
-
-    fn try_from(track: TrackHandle) -> Result<Self, Self::Error> {
-        match track {
-            TrackHandle::LocalAudio(local_audio) => Ok(Self::Local(local_audio)),
-            TrackHandle::RemoteAudio(remote_audio) => Ok(Self::Remote(remote_audio)),
-            _ => Err("not a audio track"),
-        }
-    }
-}
-
-impl_track_trait!(AudioTrackHandle, enum_dispatch, [Local, Remote]);

livekit/src/room/track/local_audio_track.rs

@@ -1,8 +1,105 @@
-use super::{impl_track_trait, TrackShared};
+use super::TrackInner;
+use crate::prelude::*;
+use crate::proto;
+use livekit_webrtc as rtc;
+use std::sync::Arc;
+use tokio::sync::mpsc;
 
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 pub struct LocalAudioTrack {
-    shared: TrackShared,
+    pub(crate) inner: Arc<TrackInner>,
 }
 
-impl_track_trait!(LocalAudioTrack);
+impl LocalAudioTrack {
+    pub(crate) fn new(
+        sid: TrackSid,
+        name: String,
+        rtc_track: rtc::media_stream::RtcAudioTrack,
+    ) -> Self {
+        Self {
+            inner: Arc::new(TrackInner::new(
+                sid,
+                name,
+                TrackKind::Audio,
+                rtc::media_stream::MediaStreamTrack::Audio(rtc_track),
+            )),
+        }
+    }
+
+    #[inline]
+    pub fn sid(&self) -> TrackSid {
+        self.inner.sid()
+    }
+
+    #[inline]
+    pub fn name(&self) -> String {
+        self.inner.name()
+    }
+
+    #[inline]
+    pub fn kind(&self) -> TrackKind {
+        self.inner.kind()
+    }
+
+    #[inline]
+    pub fn source(&self) -> TrackSource {
+        self.inner.source()
+    }
+
+    #[inline]
+    pub fn stream_state(&self) -> StreamState {
+        self.inner.stream_state()
+    }
+
+    #[inline]
+    pub fn start(&self) {
+        self.inner.start()
+    }
+
+    #[inline]
+    pub fn stop(&self) {
+        self.inner.stop()
+    }
+
+    #[inline]
+    pub fn muted(&self) -> bool {
+        self.inner.muted()
+    }
+
+    #[inline]
+    pub fn set_muted(&self, muted: bool) {
+        self.inner.set_muted(muted)
+    }
+
+    #[inline]
+    pub fn rtc_track(&self) -> rtc::media_stream::RtcAudioTrack {
+        if let rtc::media_stream::MediaStreamTrack::Audio(audio) = self.inner.rtc_track() {
+            audio
+        } else {
+            unreachable!()
+        }
+    }
+
+    #[inline]
+    pub fn register_observer(&self) -> mpsc::UnboundedReceiver<TrackEvent> {
+        self.inner.register_observer()
+    }
+
+    #[inline]
+    pub(crate) fn transceiver(&self) -> Option<rtc::rtp_transceiver::RtpTransceiver> {
+        self.inner.transceiver()
+    }
+
+    #[inline]
+    pub(crate) fn update_transceiver(
+        &self,
+        transceiver: Option<rtc::rtp_transceiver::RtpTransceiver>,
+    ) {
+        self.inner.update_transceiver(transceiver)
+    }
+
+    #[inline]
+    pub(crate) fn update_info(&self, info: proto::TrackInfo) {
+        self.inner.update_info(info)
+    }
+}

livekit/src/room/track/local_track.rs

@@ -1,32 +0,0 @@
-use super::impl_track_trait;
-use crate::prelude::*;
-use std::sync::Arc;
-
-#[derive(Clone)]
-pub enum LocalTrackHandle {
-    Audio(Arc<LocalAudioTrack>),
-    Video(Arc<LocalVideoTrack>),
-}
-
-impl From<LocalTrackHandle> for TrackHandle {
-    fn from(local_track: LocalTrackHandle) -> Self {
-        match local_track {
-            LocalTrackHandle::Audio(local_audio) => Self::LocalAudio(local_audio),
-            LocalTrackHandle::Video(local_video) => Self::LocalVideo(local_video),
-        }
-    }
-}
-
-impl TryFrom<TrackHandle> for LocalTrackHandle {
-    type Error = &'static str;
-
-    fn try_from(track: TrackHandle) -> Result<Self, Self::Error> {
-        match track {
-            TrackHandle::LocalAudio(local_audio) => Ok(Self::Audio(local_audio)),
-            TrackHandle::LocalVideo(local_video) => Ok(Self::Video(local_video)),
-            _ => Err("not a local track"),
-        }
-    }
-}
-
-impl_track_trait!(LocalTrackHandle, enum_dispatch, [Audio, Video]);

livekit/src/room/track/local_video_track.rs

@@ -1,8 +1,138 @@
-use super::{impl_track_trait, TrackShared};
+use super::TrackInner;
+use crate::proto;
+use crate::rtc_engine::lk_runtime::LkRuntime;
+use crate::{options::VideoCaptureOptions, prelude::*};
+use livekit_webrtc as rtc;
+use livekit_webrtc::peer_connection_factory::native::PeerConnectionFactoryExt;
+use parking_lot::Mutex;
+use rtc::video_source::native::NativeVideoSource;
+use std::sync::Arc;
+use tokio::sync::mpsc;
 
 #[derive(Debug)]
-pub struct LocalVideoTrack {
+struct LocalVideoTrackInner {
-    shared: TrackShared,
+    track_inner: TrackInner,
+    capture_options: Mutex<VideoCaptureOptions>,
 }
 
-impl_track_trait!(LocalVideoTrack);
+#[derive(Clone, Debug)]
+pub struct LocalVideoTrack {
+    inner: Arc<LocalVideoTrackInner>,
+}
+
+impl LocalVideoTrack {
+    pub fn new(
+        name: String,
+        rtc_track: rtc::media_stream::RtcVideoTrack,
+        capture_options: VideoCaptureOptions,
+    ) -> Self {
+        Self {
+            inner: Arc::new(LocalVideoTrackInner {
+                track_inner: TrackInner::new(
+                    "unknown".to_string().into(), // sid
+                    name,
+                    TrackKind::Video,
+                    rtc::media_stream::MediaStreamTrack::Video(rtc_track),
+                ),
+                capture_options: Mutex::new(capture_options),
+            }),
+        }
+    }
+
+    pub fn capture_options(&self) -> VideoCaptureOptions {
+        self.inner.capture_options.lock().clone()
+    }
+
+    #[inline]
+    pub fn sid(&self) -> TrackSid {
+        self.inner.track_inner.sid()
+    }
+
+    #[inline]
+    pub fn name(&self) -> String {
+        self.inner.track_inner.name()
+    }
+
+    #[inline]
+    pub fn kind(&self) -> TrackKind {
+        self.inner.track_inner.kind()
+    }
+
+    #[inline]
+    pub fn source(&self) -> TrackSource {
+        self.inner.track_inner.source()
+    }
+
+    #[inline]
+    pub fn stream_state(&self) -> StreamState {
+        self.inner.track_inner.stream_state()
+    }
+
+    #[inline]
+    pub fn start(&self) {
+        self.inner.track_inner.start()
+    }
+
+    #[inline]
+    pub fn stop(&self) {
+        self.inner.track_inner.stop()
+    }
+
+    #[inline]
+    pub fn muted(&self) -> bool {
+        self.inner.track_inner.muted()
+    }
+
+    #[inline]
+    pub fn set_muted(&self, muted: bool) {
+        self.inner.track_inner.set_muted(muted)
+    }
+
+    #[inline]
+    pub fn rtc_track(&self) -> rtc::media_stream::RtcVideoTrack {
+        if let rtc::media_stream::MediaStreamTrack::Video(video) =
+            self.inner.track_inner.rtc_track()
+        {
+            video
+        } else {
+            unreachable!()
+        }
+    }
+
+    #[inline]
+    pub fn register_observer(&self) -> mpsc::UnboundedReceiver<TrackEvent> {
+        self.inner.track_inner.register_observer()
+    }
+
+    #[inline]
+    pub fn transceiver(&self) -> Option<rtc::rtp_transceiver::RtpTransceiver> {
+        self.inner.track_inner.transceiver()
+    }
+
+    #[inline]
+    pub(crate) fn update_transceiver(
+        &self,
+        transceiver: Option<rtc::rtp_transceiver::RtpTransceiver>,
+    ) {
+        self.inner.track_inner.update_transceiver(transceiver)
+    }
+
+    #[inline]
+    pub(crate) fn update_info(&self, info: proto::TrackInfo) {
+        self.inner.track_inner.update_info(info)
+    }
+}
+
+impl LocalVideoTrack {
+    pub fn create_video_track(
+        name: &str,
+        options: VideoCaptureOptions,
+        source: NativeVideoSource,
+    ) -> LocalVideoTrack {
+        let rtc_track = LkRuntime::instance()
+            .pc_factory
+            .create_video_track(&rtc::native::create_random_uuid(), source);
+
+        Self::new(name.to_string(), rtc_track, options)
+    }
+}

livekit/src/room/track/mod.rs

@@ -2,29 +2,22 @@ use crate::prelude::*;
 use crate::proto;
 use livekit_utils::enum_dispatch;
 use livekit_utils::observer::Dispatcher;
+use livekit_webrtc as rtc;
 use parking_lot::Mutex;
+use rtc::MediaType;
 use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
-use std::sync::Arc;
 use thiserror::Error;
 use tokio::sync::mpsc;
 
-pub mod audio_track;
 pub mod local_audio_track;
-pub mod local_track;
 pub mod local_video_track;
 pub mod remote_audio_track;
-pub mod remote_track;
 pub mod remote_video_track;
-pub mod video_track;
 
-pub use audio_track::*;
 pub use local_audio_track::*;
-pub use local_track::*;
 pub use local_video_track::*;
 pub use remote_audio_track::*;
-pub use remote_track::*;
 pub use remote_video_track::*;
-pub use video_track::*;
 
 #[derive(Error, Debug, Clone)]
 pub enum TrackError {
@@ -32,51 +25,19 @@ pub enum TrackError {
     TrackNotFound(String),
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum TrackKind {
-    Unknown,
     Audio,
     Video,
 }
 
-impl From<u8> for TrackKind {
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-    fn from(val: u8) -> Self {
-        match val {
-            1 => Self::Audio,
-            2 => Self::Video,
-            _ => Self::Unknown,
-        }
-    }
-}
-
-impl From<proto::TrackType> for TrackKind {
-    fn from(r#type: proto::TrackType) -> Self {
-        match r#type {
-            proto::TrackType::Audio => Self::Audio,
-            proto::TrackType::Video => Self::Video,
-            _ => Self::Unknown,
-        }
-    }
-}
-
-#[derive(Debug)]
 pub enum StreamState {
-    Unknown,
     Active,
     Paused,
 }
 
-impl From<u8> for StreamState {
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-    fn from(val: u8) -> Self {
-        match val {
-            1 => Self::Active,
-            2 => Self::Paused,
-            _ => Self::Unknown,
-        }
-    }
-}
-
-#[derive(Debug)]
 pub enum TrackSource {
     Unknown,
     Camera,
@@ -85,9 +46,364 @@ pub enum TrackSource {
     ScreenshareAudio,
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum TrackEvent {
+    Mute,
+    Unmute,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct TrackDimension(pub u32, pub u32);
+
+#[derive(Clone, Debug)]
+pub enum Track {
+    LocalAudio(LocalAudioTrack),
+    LocalVideo(LocalVideoTrack),
+    RemoteAudio(RemoteAudioTrack),
+    RemoteVideo(RemoteVideoTrack),
+}
+
+#[derive(Clone, Debug)]
+pub enum LocalTrack {
+    Audio(LocalAudioTrack),
+    Video(LocalVideoTrack),
+}
+
+#[derive(Clone, Debug)]
+pub enum RemoteTrack {
+    Audio(RemoteAudioTrack),
+    Video(RemoteVideoTrack),
+}
+
+#[derive(Clone, Debug)]
+pub enum VideoTrack {
+    Local(LocalVideoTrack),
+    Remote(RemoteVideoTrack),
+}
+
+#[derive(Clone, Debug)]
+pub enum AudioTrack {
+    Local(LocalAudioTrack),
+    Remote(RemoteAudioTrack),
+}
+
+macro_rules! track_dispatch {
+    ([$($variant:ident),+]) => {
+        enum_dispatch!(
+            [$($variant),+];
+            pub fn sid(self: &Self) -> TrackSid;
+            pub fn name(self: &Self) -> String;
+            pub fn kind(self: &Self) -> TrackKind;
+            pub fn source(self: &Self) -> TrackSource;
+            pub fn stream_state(self: &Self) -> StreamState;
+            pub fn start(self: &Self) -> ();
+            pub fn stop(self: &Self) -> ();
+            pub fn muted(self: &Self) -> bool;
+            pub fn set_muted(self: &Self, muted: bool) -> ();
+            pub fn register_observer(self: &Self) -> mpsc::UnboundedReceiver<TrackEvent>;
+
+            pub(crate) fn transceiver(self: &Self) -> Option<rtc::rtp_transceiver::RtpTransceiver>;
+            pub(crate) fn update_transceiver(self: &Self, transceiver: Option<rtc::rtp_transceiver::RtpTransceiver>) -> ();
+            pub(crate) fn update_info(self: &Self, info: proto::TrackInfo) -> ();
+        );
+    };
+}
+
+impl Track {
+    track_dispatch!([LocalAudio, LocalVideo, RemoteAudio, RemoteVideo]);
+
+    #[inline]
+    pub fn rtc_track(&self) -> rtc::media_stream::MediaStreamTrack {
+        match self {
+            Self::LocalAudio(track) => track.rtc_track().into(),
+            Self::LocalVideo(track) => track.rtc_track().into(),
+            Self::RemoteAudio(track) => track.rtc_track().into(),
+            Self::RemoteVideo(track) => track.rtc_track().into(),
+        }
+    }
+}
+
+impl LocalTrack {
+    track_dispatch!([Audio, Video]);
+
+    #[inline]
+    pub fn rtc_track(&self) -> rtc::media_stream::MediaStreamTrack {
+        match self {
+            Self::Audio(track) => track.rtc_track().into(),
+            Self::Video(track) => track.rtc_track().into(),
+        }
+    }
+}
+
+impl RemoteTrack {
+    track_dispatch!([Audio, Video]);
+
+    #[inline]
+    pub fn rtc_track(&self) -> rtc::media_stream::MediaStreamTrack {
+        match self {
+            Self::Audio(track) => track.rtc_track().into(),
+            Self::Video(track) => track.rtc_track().into(),
+        }
+    }
+}
+
+impl VideoTrack {
+    track_dispatch!([Local, Remote]);
+
+    #[inline]
+    pub fn rtc_track(&self) -> rtc::media_stream::RtcVideoTrack {
+        match self {
+            Self::Local(track) => track.rtc_track(),
+            Self::Remote(track) => track.rtc_track(),
+        }
+    }
+}
+
+impl AudioTrack {
+    track_dispatch!([Local, Remote]);
+
+    #[inline]
+    pub fn rtc_track(&self) -> rtc::media_stream::RtcAudioTrack {
+        match self {
+            Self::Local(track) => track.rtc_track().into(),
+            Self::Remote(track) => track.rtc_track().into(),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub(crate) struct TrackInner {
+    pub sid: Mutex<TrackSid>,
+    pub name: Mutex<String>,
+    pub kind: AtomicU8,         // TrackKind
+    pub source: AtomicU8,       // TrackSource
+    pub stream_state: AtomicU8, // StreamState
+    pub muted: AtomicBool,
+    pub rtc_track: rtc::media_stream::MediaStreamTrack,
+    pub transceiver: Mutex<Option<rtc::rtp_transceiver::RtpTransceiver>>,
+    pub dispatcher: Dispatcher<TrackEvent>,
+}
+
+impl TrackInner {
+    pub fn new(
+        sid: TrackSid,
+        name: String,
+        kind: TrackKind,
+        rtc_track: rtc::media_stream::MediaStreamTrack,
+    ) -> Self {
+        Self {
+            sid: Mutex::new(sid),
+            name: Mutex::new(name),
+            kind: AtomicU8::new(kind as u8),
+            source: AtomicU8::new(TrackSource::Unknown as u8),
+            stream_state: AtomicU8::new(StreamState::Active as u8),
+            muted: AtomicBool::new(false),
+            rtc_track,
+            transceiver: Default::default(),
+            dispatcher: Default::default(),
+        }
+    }
+
+    pub fn sid(&self) -> TrackSid {
+        self.sid.lock().clone()
+    }
+
+    pub fn name(&self) -> String {
+        self.name.lock().clone()
+    }
+
+    pub fn kind(&self) -> TrackKind {
+        self.kind.load(Ordering::SeqCst).try_into().unwrap()
+    }
+
+    pub fn source(&self) -> TrackSource {
+        self.source.load(Ordering::SeqCst).into()
+    }
+
+    pub fn stream_state(&self) -> StreamState {
+        self.stream_state.load(Ordering::SeqCst).try_into().unwrap()
+    }
+
+    pub fn muted(&self) -> bool {
+        self.muted.load(Ordering::SeqCst)
+    }
+
+    pub fn start(&self) {
+        self.rtc_track.set_enabled(true);
+    }
+
+    pub fn stop(&self) {
+        self.rtc_track.set_enabled(false);
+    }
+
+    pub fn set_muted(&self, muted: bool) {
+        if self
+            .muted
+            .compare_exchange(!muted, muted, Ordering::SeqCst, Ordering::SeqCst)
+            .is_err()
+        {
+            return;
+        }
+
+        if !muted {
+            self.start();
+        } else {
+            self.stop();
+        }
+
+        let event = if muted {
+            TrackEvent::Mute
+        } else {
+            TrackEvent::Unmute
+        };
+
+        self.dispatcher.dispatch(&event);
+    }
+
+    pub fn rtc_track(&self) -> rtc::media_stream::MediaStreamTrack {
+        self.rtc_track.clone()
+    }
+
+    pub fn register_observer(&self) -> mpsc::UnboundedReceiver<TrackEvent> {
+        self.dispatcher.register()
+    }
+
+    pub fn transceiver(&self) -> Option<rtc::rtp_transceiver::RtpTransceiver> {
+        self.transceiver.lock().clone()
+    }
+
+    pub fn update_transceiver(&self, transceiver: Option<rtc::rtp_transceiver::RtpTransceiver>) {
+        *self.transceiver.lock() = transceiver;
+    }
+
+    pub fn update_info(&self, info: proto::TrackInfo) {
+        *self.name.lock() = info.name;
+        *self.sid.lock() = info.sid.into();
+        self.kind.store(
+            TrackKind::try_from(proto::TrackType::from_i32(info.r#type).unwrap()).unwrap() as u8,
+            Ordering::SeqCst,
+        );
+        self.source.store(
+            TrackSource::from(proto::TrackSource::from_i32(info.source).unwrap()) as u8,
+            Ordering::SeqCst,
+        );
+        // Muted and StreamState are not handled separately (events)
+    }
+}
+
+impl From<RemoteTrack> for Track {
+    fn from(track: RemoteTrack) -> Self {
+        match track {
+            RemoteTrack::Audio(track) => Self::RemoteAudio(track),
+            RemoteTrack::Video(track) => Self::RemoteVideo(track),
+        }
+    }
+}
+
+impl From<LocalTrack> for Track {
+    fn from(track: LocalTrack) -> Self {
+        match track {
+            LocalTrack::Audio(track) => Self::LocalAudio(track),
+            LocalTrack::Video(track) => Self::LocalVideo(track),
+        }
+    }
+}
+
+impl From<VideoTrack> for Track {
+    fn from(track: VideoTrack) -> Self {
+        match track {
+            VideoTrack::Local(track) => Self::LocalVideo(track),
+            VideoTrack::Remote(track) => Self::RemoteVideo(track),
+        }
+    }
+}
+
+impl From<AudioTrack> for Track {
+    fn from(track: AudioTrack) -> Self {
+        match track {
+            AudioTrack::Local(track) => Self::LocalAudio(track),
+            AudioTrack::Remote(track) => Self::RemoteAudio(track),
+        }
+    }
+}
+
+impl TryFrom<Track> for RemoteTrack {
+    type Error = &'static str;
+
+    fn try_from(track: Track) -> Result<Self, Self::Error> {
+        match track {
+            Track::RemoteAudio(track) => Ok(Self::Audio(track)),
+            Track::RemoteVideo(track) => Ok(Self::Video(track)),
+            _ => Err("not a remote track"),
+        }
+    }
+}
+
+impl TryFrom<Track> for LocalTrack {
+    type Error = &'static str;
+
+    fn try_from(track: Track) -> Result<Self, Self::Error> {
+        match track {
+            Track::LocalAudio(track) => Ok(Self::Audio(track)),
+            Track::LocalVideo(track) => Ok(Self::Video(track)),
+            _ => Err("not a local track"),
+        }
+    }
+}
+
+impl TryFrom<Track> for VideoTrack {
+    type Error = &'static str;
+
+    fn try_from(track: Track) -> Result<Self, Self::Error> {
+        match track {
+            Track::LocalVideo(track) => Ok(Self::Local(track)),
+            Track::RemoteVideo(track) => Ok(Self::Remote(track)),
+            _ => Err("not a video track"),
+        }
+    }
+}
+
+impl TryFrom<Track> for AudioTrack {
+    type Error = &'static str;
+
+    fn try_from(track: Track) -> Result<Self, Self::Error> {
+        match track {
+            Track::LocalAudio(track) => Ok(Self::Local(track)),
+            Track::RemoteAudio(track) => Ok(Self::Remote(track)),
+            _ => Err("not an audio track"),
+        }
+    }
+}
+
+// Conversions from integers (Useful since we're using atomic values to represent our enums)
+
+impl TryFrom<u8> for TrackKind {
+    type Error = &'static str;
+
+    fn try_from(kind: u8) -> Result<Self, Self::Error> {
+        match kind {
+            0 => Ok(Self::Audio),
+            1 => Ok(Self::Video),
+            _ => Err("invalid track kind"),
+        }
+    }
+}
+
+impl TryFrom<u8> for StreamState {
+    type Error = &'static str;
+
+    fn try_from(state: u8) -> Result<Self, Self::Error> {
+        match state {
+            0 => Ok(Self::Active),
+            1 => Ok(Self::Paused),
+            _ => Err("invalid stream state"),
+        }
+    }
+}
+
 impl From<u8> for TrackSource {
-    fn from(val: u8) -> Self {
+    fn from(source: u8) -> Self {
-        match val {
+        match source {
             1 => Self::Camera,
             2 => Self::Microphone,
             3 => Self::Screenshare,
@@ -97,197 +413,11 @@ impl From<u8> for TrackSource {
     }
 }
 
-impl From<proto::TrackSource> for TrackSource {
+impl From<TrackKind> for MediaType {
-    fn from(source: proto::TrackSource) -> Self {
+    fn from(kind: TrackKind) -> Self {
-        match source {
+        match kind {
-            proto::TrackSource::Camera => Self::Camera,
+            TrackKind::Audio => Self::Audio,
-            proto::TrackSource::Microphone => Self::Microphone,
+            TrackKind::Video => Self::Video,
-            proto::TrackSource::ScreenShare => Self::Screenshare,
-            proto::TrackSource::ScreenShareAudio => Self::ScreenshareAudio,
-            proto::TrackSource::Unknown => Self::Unknown,
         }
     }
 }
-
-#[derive(Clone, Copy, Debug)]
-pub struct TrackDimension(pub u32, pub u32);
-
-pub trait TrackTrait {
-    fn sid(&self) -> TrackSid;
-    fn name(&self) -> String;
-    fn kind(&self) -> TrackKind;
-    fn stream_state(&self) -> StreamState;
-    fn muted(&self) -> bool;
-    fn start(&self);
-    fn stop(&self);
-    fn register_observer(&self) -> mpsc::UnboundedReceiver<TrackEvent>;
-    fn set_muted(&self, muted: bool);
-}
-
-#[derive(Debug, Clone)]
-pub enum TrackEvent {
-    Mute,
-    Unmute,
-}
-
-#[derive(Debug)]
-pub(super) struct TrackShared {
-    pub(super) sid: Mutex<TrackSid>,
-    pub(super) name: Mutex<String>,
-    pub(super) kind: AtomicU8,         // TrackKind
-    pub(super) stream_state: AtomicU8, // StreamState
-    pub(super) muted: AtomicBool,
-    pub(super) rtc_track: MediaStreamTrackHandle,
-    pub(super) dispatcher: Mutex<Dispatcher<TrackEvent>>,
-}
-
-impl TrackShared {
-    pub(crate) fn new(
-        sid: TrackSid,
-        name: String,
-        kind: TrackKind,
-        rtc_track: MediaStreamTrackHandle,
-    ) -> Self {
-        Self {
-            sid: Mutex::new(sid),
-            name: Mutex::new(name),
-            kind: AtomicU8::new(kind as u8),
-            stream_state: AtomicU8::new(StreamState::Active as u8),
-            muted: AtomicBool::new(false),
-            rtc_track,
-            dispatcher: Default::default(),
-        }
-    }
-
-    pub(crate) fn start(&self) {
-        self.rtc_track.set_enabled(true);
-    }
-
-    pub(crate) fn stop(&self) {
-        self.rtc_track.set_enabled(false);
-    }
-
-    pub(crate) fn set_muted(&self, muted: bool) {
-        if self.muted.load(Ordering::SeqCst) == muted {
-            return;
-        }
-
-        self.muted.store(muted, Ordering::SeqCst);
-        self.rtc_track.set_enabled(!muted);
-
-        self.dispatcher.lock().dispatch(if muted {
-            &TrackEvent::Mute
-        } else {
-            &TrackEvent::Unmute
-        });
-    }
-
-    pub(crate) fn register_observer(&self) -> mpsc::UnboundedReceiver<TrackEvent> {
-        self.dispatcher.lock().register()
-    }
-}
-
-#[derive(Clone, Debug)]
-pub enum TrackHandle {
-    LocalVideo(Arc<LocalVideoTrack>),
-    LocalAudio(Arc<LocalAudioTrack>),
-    RemoteVideo(Arc<RemoteVideoTrack>),
-    RemoteAudio(Arc<RemoteAudioTrack>),
-}
-
-impl TrackTrait for TrackHandle {
-    enum_dispatch!(
-        [LocalVideo, LocalAudio, RemoteVideo, RemoteAudio]
-        fnc!(sid, &Self, [], TrackSid);
-        fnc!(name, &Self, [], String);
-        fnc!(kind, &Self, [], TrackKind);
-        fnc!(stream_state, &Self, [], StreamState);
-        fnc!(muted, &Self, [], bool);
-        fnc!(start, &Self, [], ());
-        fnc!(stop, &Self, [], ());
-        fnc!(register_observer, &Self, [], mpsc::UnboundedReceiver<TrackEvent>);
-        fnc!(set_muted, &Self, [muted: bool], ());
-    );
-}
-
-impl TrackHandle {
-    pub fn rtc_track(&self) -> MediaStreamTrackHandle {
-        match self {
-            Self::RemoteVideo(remote_video) => {
-                MediaStreamTrackHandle::Video(remote_video.rtc_track())
-            }
-            Self::RemoteAudio(remote_audio) => {
-                MediaStreamTrackHandle::Audio(remote_audio.rtc_track())
-            }
-            _ => todo!(),
-        }
-    }
-}
-
-macro_rules! impl_track_trait {
-    ($x:ident) => {
-        use std::sync::atomic::Ordering;
-        use tokio::sync::mpsc;
-        use $crate::room::id::TrackSid;
-        use $crate::room::track::{StreamState, TrackEvent, TrackKind, TrackTrait};
-
-        impl TrackTrait for $x {
-            fn sid(&self) -> TrackSid {
-                self.shared.sid.lock().clone()
-            }
-
-            fn name(&self) -> String {
-                self.shared.name.lock().clone()
-            }
-
-            fn kind(&self) -> TrackKind {
-                self.shared.kind.load(Ordering::SeqCst).into()
-            }
-
-            fn stream_state(&self) -> StreamState {
-                self.shared.stream_state.load(Ordering::SeqCst).into()
-            }
-
-            fn muted(&self) -> bool {
-                self.shared.muted.load(Ordering::SeqCst)
-            }
-
-            fn start(&self) {
-                self.shared.start();
-            }
-
-            fn stop(&self) {
-                self.shared.stop();
-            }
-
-            fn register_observer(&self) -> mpsc::UnboundedReceiver<TrackEvent> {
-                self.shared.register_observer()
-            }
-
-            fn set_muted(&self, muted: bool) {
-                self.shared.set_muted(muted);
-            }
-        }
-    };
-    ($x:ident, enum_dispatch, [$($variant:ident),+]) => {
-        use livekit_utils::enum_dispatch;
-        use tokio::sync::mpsc;
-
-        impl TrackTrait for $x {
-            enum_dispatch!(
-                [$($variant),+]
-                fnc!(sid, &Self, [], TrackSid);
-                fnc!(name, &Self, [], String);
-                fnc!(kind, &Self, [], TrackKind);
-                fnc!(stream_state, &Self, [], StreamState);
-                fnc!(muted, &Self, [], bool);
-                fnc!(start, &Self, [], ());
-                fnc!(stop, &Self, [], ());
-                fnc!(register_observer, &Self, [], mpsc::UnboundedReceiver<TrackEvent>);
-                fnc!(set_muted, &Self, [muted: bool], ());
-            );
-        }
-    };
-}
-
-pub(super) use impl_track_trait;

livekit/src/room/track/remote_audio_track.rs

@@ -1,31 +1,105 @@
-use super::{impl_track_trait, TrackShared};
+use super::TrackInner;
 use crate::prelude::*;
+use crate::proto;
+use livekit_webrtc as rtc;
 use std::sync::Arc;
+use tokio::sync::mpsc;
 
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 pub struct RemoteAudioTrack {
-    shared: TrackShared,
+    pub(crate) inner: Arc<TrackInner>,
 }
 
 impl RemoteAudioTrack {
-    pub(crate) fn new(sid: TrackSid, name: String, track: Arc<AudioTrack>) -> Self {
+    pub(crate) fn new(
+        sid: TrackSid,
+        name: String,
+        rtc_track: rtc::media_stream::RtcAudioTrack,
+    ) -> Self {
         Self {
-            shared: TrackShared::new(
+            inner: Arc::new(TrackInner::new(
                 sid,
                 name,
                 TrackKind::Audio,
-                MediaStreamTrackHandle::Audio(track),
+                rtc::media_stream::MediaStreamTrack::Audio(rtc_track),
-            ),
+            )),
         }
     }
 
-    pub fn rtc_track(&self) -> Arc<AudioTrack> {
+    #[inline]
-        if let MediaStreamTrackHandle::Audio(audio) = &self.shared.rtc_track {
+    pub fn sid(&self) -> TrackSid {
-            audio.clone()
+        self.inner.sid()
+    }
+
+    #[inline]
+    pub fn name(&self) -> String {
+        self.inner.name()
+    }
+
+    #[inline]
+    pub fn kind(&self) -> TrackKind {
+        self.inner.kind()
+    }
+
+    #[inline]
+    pub fn source(&self) -> TrackSource {
+        self.inner.source()
+    }
+
+    #[inline]
+    pub fn stream_state(&self) -> StreamState {
+        self.inner.stream_state()
+    }
+
+    #[inline]
+    pub fn start(&self) {
+        self.inner.start()
+    }
+
+    #[inline]
+    pub fn stop(&self) {
+        self.inner.stop()
+    }
+
+    #[inline]
+    pub fn muted(&self) -> bool {
+        self.inner.muted()
+    }
+
+    #[inline]
+    pub fn set_muted(&self, muted: bool) {
+        self.inner.set_muted(muted)
+    }
+
+    #[inline]
+    pub fn rtc_track(&self) -> rtc::media_stream::RtcAudioTrack {
+        if let rtc::media_stream::MediaStreamTrack::Audio(audio) = self.inner.rtc_track() {
+            audio
         } else {
             unreachable!()
         }
     }
-}
 
-impl_track_trait!(RemoteAudioTrack);
+    #[inline]
+    pub fn register_observer(&self) -> mpsc::UnboundedReceiver<TrackEvent> {
+        self.inner.register_observer()
+    }
+
+    #[inline]
+    pub(crate) fn transceiver(&self) -> Option<rtc::rtp_transceiver::RtpTransceiver> {
+        self.inner.transceiver()
+    }
+
+    #[inline]
+    pub(crate) fn update_transceiver(
+        &self,
+        transceiver: Option<rtc::rtp_transceiver::RtpTransceiver>,
+    ) {
+        self.inner.update_transceiver(transceiver)
+    }
+
+    #[inline]
+    pub(crate) fn update_info(&self, info: proto::TrackInfo) {
+        self.inner.update_info(info)
+    }
+}

livekit/src/room/track/remote_track.rs

@@ -1,32 +0,0 @@
-use super::impl_track_trait;
-use crate::prelude::*;
-use std::sync::Arc;
-
-#[derive(Clone, Debug)]
-pub enum RemoteTrackHandle {
-    Audio(Arc<RemoteAudioTrack>),
-    Video(Arc<RemoteVideoTrack>),
-}
-
-impl From<RemoteTrackHandle> for TrackHandle {
-    fn from(remote_track: RemoteTrackHandle) -> Self {
-        match remote_track {
-            RemoteTrackHandle::Audio(remote_audio) => Self::RemoteAudio(remote_audio),
-            RemoteTrackHandle::Video(remote_video) => Self::RemoteVideo(remote_video),
-        }
-    }
-}
-
-impl TryFrom<TrackHandle> for RemoteTrackHandle {
-    type Error = &'static str;
-
-    fn try_from(track: TrackHandle) -> Result<Self, Self::Error> {
-        match track {
-            TrackHandle::RemoteAudio(remote_audio) => Ok(Self::Audio(remote_audio)),
-            TrackHandle::RemoteVideo(remote_video) => Ok(Self::Video(remote_video)),
-            _ => Err("not a remote track"),
-        }
-    }
-}
-
-impl_track_trait!(RemoteTrackHandle, enum_dispatch, [Audio, Video]);

livekit/src/room/track/remote_video_track.rs

@@ -1,31 +1,105 @@
-use super::{impl_track_trait, TrackShared};
+use super::TrackInner;
 use crate::prelude::*;
+use crate::proto;
+use livekit_webrtc as rtc;
 use std::sync::Arc;
+use tokio::sync::mpsc;
 
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 pub struct RemoteVideoTrack {
-    shared: TrackShared,
+    pub(crate) inner: Arc<TrackInner>,
 }
 
 impl RemoteVideoTrack {
-    pub(crate) fn new(sid: TrackSid, name: String, track: Arc<VideoTrack>) -> Self {
+    pub(crate) fn new(
+        sid: TrackSid,
+        name: String,
+        rtc_track: rtc::media_stream::RtcVideoTrack,
+    ) -> Self {
         Self {
-            shared: TrackShared::new(
+            inner: Arc::new(TrackInner::new(
                 sid,
                 name,
                 TrackKind::Video,
-                MediaStreamTrackHandle::Video(track),
+                rtc::media_stream::MediaStreamTrack::Video(rtc_track),
-            ),
+            )),
         }
     }
 
-    pub fn rtc_track(&self) -> Arc<VideoTrack> {
+    #[inline]
-        if let MediaStreamTrackHandle::Video(video) = &self.shared.rtc_track {
+    pub fn sid(&self) -> TrackSid {
-            video.clone()
+        self.inner.sid()
+    }
+
+    #[inline]
+    pub fn name(&self) -> String {
+        self.inner.name()
+    }
+
+    #[inline]
+    pub fn kind(&self) -> TrackKind {
+        self.inner.kind()
+    }
+
+    #[inline]
+    pub fn source(&self) -> TrackSource {
+        self.inner.source()
+    }
+
+    #[inline]
+    pub fn stream_state(&self) -> StreamState {
+        self.inner.stream_state()
+    }
+
+    #[inline]
+    pub fn start(&self) {
+        self.inner.start()
+    }
+
+    #[inline]
+    pub fn stop(&self) {
+        self.inner.stop()
+    }
+
+    #[inline]
+    pub fn muted(&self) -> bool {
+        self.inner.muted()
+    }
+
+    #[inline]
+    pub fn set_muted(&self, muted: bool) {
+        self.inner.set_muted(muted)
+    }
+
+    #[inline]
+    pub fn rtc_track(&self) -> rtc::media_stream::RtcVideoTrack {
+        if let rtc::media_stream::MediaStreamTrack::Video(video) = self.inner.rtc_track() {
+            video
         } else {
             unreachable!()
         }
     }
-}
 
-impl_track_trait!(RemoteVideoTrack);
+    #[inline]
+    pub fn register_observer(&self) -> mpsc::UnboundedReceiver<TrackEvent> {
+        self.inner.register_observer()
+    }
+
+    #[inline]
+    pub(crate) fn transceiver(&self) -> Option<rtc::rtp_transceiver::RtpTransceiver> {
+        self.inner.transceiver()
+    }
+
+    #[inline]
+    pub(crate) fn update_transceiver(
+        &self,
+        transceiver: Option<rtc::rtp_transceiver::RtpTransceiver>,
+    ) {
+        self.inner.update_transceiver(transceiver)
+    }
+
+    #[inline]
+    pub(crate) fn update_info(&self, info: proto::TrackInfo) {
+        self.inner.update_info(info);
+    }
+}

livekit/src/room/track/video_track.rs

@@ -1,32 +0,0 @@
-use super::impl_track_trait;
-use crate::prelude::*;
-use std::sync::Arc;
-
-#[derive(Clone)]
-pub enum VideoTrackHandle {
-    Local(Arc<LocalVideoTrack>),
-    Remote(Arc<RemoteVideoTrack>),
-}
-
-impl From<VideoTrackHandle> for TrackHandle {
-    fn from(video_track: VideoTrackHandle) -> Self {
-        match video_track {
-            VideoTrackHandle::Local(local_video) => Self::LocalVideo(local_video),
-            VideoTrackHandle::Remote(remote_video) => Self::RemoteVideo(remote_video),
-        }
-    }
-}
-
-impl TryFrom<TrackHandle> for VideoTrackHandle {
-    type Error = &'static str;
-
-    fn try_from(track: TrackHandle) -> Result<Self, Self::Error> {
-        match track {
-            TrackHandle::LocalVideo(local_video) => Ok(Self::Local(local_video)),
-            TrackHandle::RemoteVideo(remote_video) => Ok(Self::Remote(remote_video)),
-            _ => Err("not a video track"),
-        }
-    }
-}
-
-impl_track_trait!(VideoTrackHandle, enum_dispatch, [Local, Remote]);

livekit/src/rtc_engine/lk_runtime.rs

@@ -1,35 +1,48 @@
-use livekit_webrtc::peer_connection_factory::PeerConnectionFactory;
+use lazy_static::lazy_static;
-use livekit_webrtc::webrtc::RTCRuntime;
+use livekit_webrtc::prelude::*;
+use parking_lot::Mutex;
 use std::fmt::{Debug, Formatter};
+use std::sync::{Arc, Weak};
 use tracing::trace;
 
-/// SAFETY: The order of initialization and deletion is important for LKRuntime.
+lazy_static! {
-/// See the C++ constructors & destructors of these fields
+    static ref LK_RUNTIME: Mutex<Weak<LkRuntime>> = Mutex::new(Weak::new());
+}
 
-pub struct LKRuntime {
+pub struct LkRuntime {
     pub pc_factory: PeerConnectionFactory,
-    pub rtc_runtime: RTCRuntime,
 }
 
-impl Debug for LKRuntime {
+impl Debug for LkRuntime {
     fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
-        write!(f, "LKRuntime")
+        f.debug_struct("LkRuntime").finish()
     }
 }
 
-impl Default for LKRuntime {
+impl LkRuntime {
+    pub fn instance() -> Arc<LkRuntime> {
+        let mut lk_runtime_ref = LK_RUNTIME.lock();
+        if let Some(lk_runtime) = lk_runtime_ref.upgrade() {
+            lk_runtime
+        } else {
+            let new_runtime = Arc::new(LkRuntime::default());
+            *lk_runtime_ref = Arc::downgrade(&new_runtime);
+            new_runtime
+        }
+    }
+}
+
+impl Default for LkRuntime {
     fn default() -> Self {
-        trace!("LKRuntime::default()");
+        trace!("LkRuntime::default()");
-        let rtc_runtime = RTCRuntime::new();
         Self {
-            pc_factory: PeerConnectionFactory::new(rtc_runtime.clone()),
+            pc_factory: PeerConnectionFactory::default(),
-            rtc_runtime,
         }
     }
 }
 
-impl Drop for LKRuntime {
+impl Drop for LkRuntime {
     fn drop(&mut self) {
-        trace!("LKRuntime::drop()");
+        trace!("LkRuntime::drop()");
     }
 }

livekit/src/rtc_engine/mod.rs

@@ -1,14 +1,11 @@
-use crate::prelude::*;
+use crate::options::TrackPublishOptions;
+use crate::prelude::LocalTrack;
 use crate::proto;
-use crate::rtc_engine::lk_runtime::LKRuntime;
+use crate::rtc_engine::lk_runtime::LkRuntime;
-use crate::rtc_engine::rtc_session::{RTCSession, SessionEvent, SessionEvents, SessionInfo};
+use crate::rtc_engine::rtc_session::{RtcSession, SessionEvent, SessionEvents, SessionInfo};
 use crate::signal_client::{SignalError, SignalOptions};
-use futures::future::BoxFuture;
-use futures::FutureExt;
-use lazy_static::lazy_static;
-use livekit_webrtc::data_channel::DataSendError;
-use livekit_webrtc::jsep::SdpParseError;
 use livekit_webrtc::prelude::*;
+use livekit_webrtc::session_description::SdpParseError;
 use parking_lot::Mutex;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Weak};
@@ -20,8 +17,8 @@ use tokio::task::JoinHandle;
 use tokio::time::{interval, Interval};
 use tracing::{error, info, warn};
 
-mod lk_runtime;
+pub mod lk_runtime;
-mod pc_transport;
+mod peer_transport;
 mod rtc_events;
 mod rtc_session;
 
@@ -46,13 +43,13 @@ pub enum EngineError {
     #[error("signal failure: {0}")]
     Signal(#[from] SignalError),
     #[error("internal webrtc failure")]
-    Rtc(#[from] RTCError),
+    Rtc(#[from] RtcError),
     #[error("failed to parse sdp")]
     Parse(#[from] SdpParseError),
     #[error("serde error")]
     Serde(#[from] serde_json::Error),
     #[error("failed to send data to the datachannel")]
-    Data(#[from] DataSendError),
+    Data(#[from] DataChannelError),
     #[error("connection error: {0}")]
     Connection(String),
     #[error("decode error")]
@@ -67,7 +64,7 @@ pub enum EngineEvent {
         updates: Vec<proto::ParticipantInfo>,
     },
     MediaTrack {
-        track: MediaStreamTrackHandle,
+        track: MediaStreamTrack,
         stream: MediaStream,
         receiver: RtpReceiver,
     },
@@ -92,23 +89,19 @@ pub enum EngineEvent {
 pub const RECONNECT_ATTEMPTS: u32 = 10;
 pub const RECONNECT_INTERVAL: Duration = Duration::from_secs(5);
 
-lazy_static! {
-    // Share one LKRuntime across all RTCEngine instances
-    static ref LK_RUNTIME: Mutex<Weak<LKRuntime>> = Mutex::new(Weak::new());
-}
 ///
 /// Represents a running RTCSession with the ability to close the session
 /// and the engine_task
 #[derive(Debug)]
 struct EngineHandle {
-    session: RTCSession,
+    session: RtcSession,
     engine_task: JoinHandle<()>,
     close_sender: oneshot::Sender<()>,
 }
 
 #[derive(Debug)]
 struct EngineInner {
-    lk_runtime: Arc<LKRuntime>,
+    lk_runtime: Arc<LkRuntime>,
     session_info: Mutex<Option<SessionInfo>>, // Last/Current Sessioninfo
     running_handle: AsyncRwLock<Option<EngineHandle>>,
     opened: AtomicBool,
@@ -121,26 +114,15 @@ struct EngineInner {
 }
 
 #[derive(Debug)]
-pub struct RTCEngine {
+pub struct RtcEngine {
     inner: Arc<EngineInner>,
 }
 
-impl RTCEngine {
+impl RtcEngine {
     pub fn new() -> (Self, EngineEvents) {
-        let lk_runtime = {
-            let mut lk_runtime_ref = LK_RUNTIME.lock();
-            if let Some(lk_runtime) = lk_runtime_ref.upgrade() {
-                lk_runtime
-            } else {
-                let new_runtime = Arc::new(LKRuntime::default());
-                *lk_runtime_ref = Arc::downgrade(&new_runtime);
-                new_runtime
-            }
-        };
-
         let (engine_emitter, engine_events) = mpsc::channel(8);
         let inner = Arc::new(EngineInner {
-            lk_runtime,
+            lk_runtime: LkRuntime::instance(),
             session_info: Default::default(),
             running_handle: Default::default(),
             opened: Default::default(),
@@ -153,6 +135,10 @@ impl RTCEngine {
         (Self { inner }, engine_events)
     }
 
+    pub(crate) fn lk_runtime(&self) -> Arc<LkRuntime> {
+        self.inner.lk_runtime.clone()
+    }
+
     #[tracing::instrument]
     pub async fn connect(
         &self,
@@ -200,6 +186,64 @@ impl RTCEngine {
         Ok(())
     }
 
+    pub async fn add_track(&self, req: proto::AddTrackRequest) -> EngineResult<proto::TrackInfo> {
+        self.inner.wait_reconnection().await?;
+        self.inner
+            .running_handle
+            .read()
+            .await
+            .as_ref()
+            .unwrap()
+            .session
+            .add_track(req)
+            .await
+    }
+
+    pub async fn remove_track(&self, sender: RtpSender) -> EngineResult<()> {
+        self.inner.wait_reconnection().await?;
+        self.inner
+            .running_handle
+            .read()
+            .await
+            .as_ref()
+            .unwrap()
+            .session
+            .remove_track(sender)
+            .await
+    }
+
+    pub async fn create_sender(
+        &self,
+        track: LocalTrack,
+        options: TrackPublishOptions,
+        encodings: Vec<RtpEncodingParameters>,
+    ) -> EngineResult<RtpTransceiver> {
+        self.inner.wait_reconnection().await?;
+        self.inner
+            .running_handle
+            .read()
+            .await
+            .as_ref()
+            .unwrap()
+            .session
+            .create_sender(track, options, encodings)
+            .await
+    }
+
+    pub async fn negotiate_publisher(&self) -> EngineResult<()> {
+        // TODO(theomonnom): guard for reconnection
+        self.inner.wait_reconnection().await?;
+        self.inner
+            .running_handle
+            .read()
+            .await
+            .as_ref()
+            .unwrap()
+            .session
+            .negotiate_publisher()
+            .await
+    }
+
     pub fn join_response(&self) -> Option<proto::JoinResponse> {
         if let Some(info) = self.inner.session_info.lock().as_ref() {
             Some(info.join_response.clone())
@@ -307,15 +351,14 @@ impl EngineInner {
         Ok(())
     }
 
-    fn connect<'a>(
+    async fn connect(
-        self: &'a Arc<Self>,
+        self: &Arc<Self>,
-        url: &'a str,
+        url: &str,
-        token: &'a str,
+        token: &str,
         options: SignalOptions,
-    ) -> BoxFuture<'a, EngineResult<()>> {
+    ) -> EngineResult<()> {
-        async {
         let (session_emitter, session_events) = mpsc::unbounded_channel();
-            let session = RTCSession::connect(
+        let session = RtcSession::connect(
             url,
             token,
             options,
@@ -325,8 +368,7 @@ impl EngineInner {
         .await?;
 
         let (close_sender, close_receiver) = oneshot::channel();
-            let engine_task =
+        let engine_task = tokio::spawn(self.clone().engine_task(session_events, close_receiver));
-                tokio::spawn(self.clone().engine_task(session_events, close_receiver));
         *self.session_info.lock() = Some(session.info().clone());
         *self.running_handle.write().await = Some(EngineHandle {
             session,
@@ -337,8 +379,6 @@ impl EngineInner {
         self.opened.store(true, Ordering::SeqCst);
         Ok(())
     }
-        .boxed()
-    }
 
     async fn terminate_session(&self) {
         if let Some(handle) = self.running_handle.write().await.take() {
@@ -362,7 +402,7 @@ impl EngineInner {
         }
 
         while self.reconnecting.load(Ordering::Acquire) {
-            tokio::task::yield_now().await;
+            tokio::task::yield_now().await; // TODO(theomonnom): Remove yield
         }
 
         if self.running_handle.read().await.is_none() {

livekit/src/rtc_engine/peer_transport.rs

@@ -1,35 +1,31 @@
 use crate::proto;
 use livekit_webrtc::prelude::*;
 use std::fmt::{Debug, Formatter};
-use std::future::Future;
-use std::pin::Pin;
 use std::time::Duration;
-use tracing::{event, Level};
+use tracing::{debug, event, Level};
 
 const NEGOTIATION_FREQUENCY: Duration = Duration::from_millis(150);
 
-pub type OnOfferHandler = Box<
+pub type OnOfferCreated = Box<dyn FnMut(SessionDescription) + Send + Sync>;
-    dyn (FnMut(SessionDescription) -> Pin<Box<dyn Future<Output = ()> + Send + 'static>>)
-        + Send
-        + Sync,
->;
 
-pub struct PCTransport {
+pub struct PeerTransport {
     signal_target: proto::SignalTarget,
     peer_connection: PeerConnection,
     pending_candidates: Vec<IceCandidate>,
-    on_offer_handler: Option<OnOfferHandler>,
+    on_offer_handler: Option<OnOfferCreated>,
     renegotiate: bool,
     restarting_ice: bool,
 }
 
-impl Debug for PCTransport {
+impl Debug for PeerTransport {
     fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
-        f.write_str("PCTransport")
+        f.debug_struct("PeerTransport")
+            .field("target", &self.signal_target)
+            .finish()
     }
 }
 
-impl PCTransport {
+impl PeerTransport {
     pub fn new(peer_connection: PeerConnection, signal_target: proto::SignalTarget) -> Self {
         Self {
             signal_target,
@@ -42,9 +38,10 @@ impl PCTransport {
     }
 
     pub fn is_connected(&self) -> bool {
-        self.peer_connection.ice_connection_state() == IceConnectionState::IceConnectionConnected
+        matches!(
-            || self.peer_connection.ice_connection_state()
+            self.peer_connection.ice_connection_state(),
-                == IceConnectionState::IceConnectionCompleted
+            IceConnectionState::Connected | IceConnectionState::Completed
+        )
     }
 
     pub fn peer_connection(&mut self) -> &mut PeerConnection {
@@ -55,8 +52,8 @@ impl PCTransport {
         self.signal_target.clone()
     }
 
-    pub fn on_offer(&mut self, handler: OnOfferHandler) {
+    pub fn on_offer(&mut self, handler: Option<OnOfferCreated>) {
-        self.on_offer_handler = Some(handler);
+        self.on_offer_handler = handler;
     }
 
     pub fn prepare_ice_restart(&mut self) {
@@ -68,8 +65,8 @@ impl PCTransport {
     }
 
     #[tracing::instrument(level = Level::DEBUG)]
-    pub async fn add_ice_candidate(&mut self, ice_candidate: IceCandidate) -> Result<(), RTCError> {
+    pub async fn add_ice_candidate(&mut self, ice_candidate: IceCandidate) -> Result<(), RtcError> {
-        if self.peer_connection.remote_description().is_some() && !self.restarting_ice {
+        if self.peer_connection.current_remote_description().is_some() && !self.restarting_ice {
             self.peer_connection
                 .add_ice_candidate(ice_candidate)
                 .await?;
@@ -85,7 +82,7 @@ impl PCTransport {
     pub async fn set_remote_description(
         &mut self,
         remote_description: SessionDescription,
-    ) -> Result<(), RTCError> {
+    ) -> Result<(), RtcError> {
         self.peer_connection
             .set_remote_description(remote_description)
             .await?;
@@ -97,31 +94,26 @@ impl PCTransport {
 
         if self.renegotiate {
             self.renegotiate = false;
-            self.create_and_send_offer(RTCOfferAnswerOptions::default())
+            self.create_and_send_offer(OfferOptions::default()).await?;
-                .await?;
         }
 
         Ok(())
     }
 
     #[tracing::instrument(level = Level::DEBUG)]
-    pub async fn negotiate(&mut self) -> Result<(), RTCError> {
+    pub async fn negotiate(&mut self) -> Result<(), RtcError> {
         // TODO(theomonnom) Debounce here with NEGOTIATION_FREQUENCY
-        self.create_and_send_offer(RTCOfferAnswerOptions::default())
+        self.create_and_send_offer(OfferOptions::default()).await
-            .await
     }
 
     #[tracing::instrument(level = Level::DEBUG)]
     pub async fn create_anwser(
         &mut self,
         offer: SessionDescription,
-        options: RTCOfferAnswerOptions,
+        options: AnswerOptions,
-    ) -> Result<SessionDescription, RTCError> {
+    ) -> Result<SessionDescription, RtcError> {
         self.set_remote_description(offer).await?;
-        let answer = self
+        let answer = self.peer_connection().create_answer(options).await?;
-            .peer_connection()
-            .create_answer(RTCOfferAnswerOptions::default())
-            .await?;
         self.peer_connection()
             .set_local_description(answer.clone())
             .await?;
@@ -130,10 +122,7 @@ impl PCTransport {
     }
 
     #[tracing::instrument(level = Level::DEBUG)]
-    pub async fn create_and_send_offer(
+    pub async fn create_and_send_offer(&mut self, options: OfferOptions) -> Result<(), RtcError> {
-        &mut self,
-        options: RTCOfferAnswerOptions,
-    ) -> Result<(), RTCError> {
         if self.on_offer_handler.is_none() {
             return Ok(());
         }
@@ -145,7 +134,8 @@ impl PCTransport {
 
         if self.peer_connection.signaling_state() == SignalingState::HaveLocalOffer {
             if options.ice_restart {
-                if let Some(remote_description) = self.peer_connection.remote_description() {
+                if let Some(remote_description) = self.peer_connection.current_remote_description()
+                {
                     self.peer_connection
                         .set_remote_description(remote_description)
                         .await?;
@@ -165,7 +155,7 @@ impl PCTransport {
         self.peer_connection
             .set_local_description(offer.clone())
             .await?;
-        self.on_offer_handler.as_mut().unwrap()(offer).await;
+        self.on_offer_handler.as_mut().unwrap()(offer);
         Ok(())
     }
 }

livekit/src/rtc_engine/rtc_events.rs

@@ -1,20 +1,15 @@
-use super::pc_transport::PCTransport;
+use super::peer_transport::PeerTransport;
 use crate::proto;
-use crate::rtc_engine::pc_transport::OnOfferHandler;
+use crate::rtc_engine::peer_transport::OnOfferCreated;
-use livekit_webrtc::data_channel::OnMessageHandler;
+use livekit_webrtc::{self as rtc, prelude::*};
-use livekit_webrtc::peer_connection::{
-    OnAddTrackHandler, OnConnectionChangeHandler, OnDataChannelHandler, OnIceCandidateErrorHandler,
-    OnIceCandidateHandler, PeerConnectionState,
-};
-use livekit_webrtc::prelude::*;
 use tokio::sync::mpsc;
-use tracing::error;
+use tracing::{debug, error};
 
-pub type RTCEmitter = mpsc::UnboundedSender<RTCEvent>;
+pub type RtcEmitter = mpsc::UnboundedSender<RtcEvent>;
-pub type RTCEvents = mpsc::UnboundedReceiver<RTCEvent>;
+pub type RtcEvents = mpsc::UnboundedReceiver<RtcEvent>;
 
 #[derive(Debug)]
-pub enum RTCEvent {
+pub enum RtcEvent {
     IceCandidate {
         ice_candidate: IceCandidate,
         target: proto::SignalTarget,
@@ -27,14 +22,16 @@ pub enum RTCEvent {
         data_channel: DataChannel,
         target: proto::SignalTarget,
     },
-    // TODO (theomonnom): Move Offer to PCTransport
+    // TODO (theomonnom): Move Offer to PeerTransport
     Offer {
         offer: SessionDescription,
         target: proto::SignalTarget,
     },
-    AddTrack {
+    Track {
-        rtp_receiver: RtpReceiver,
+        receiver: RtpReceiver,
         streams: Vec<MediaStream>,
+        track: MediaStreamTrack,
+        transceiver: RtpTransceiver,
         target: proto::SignalTarget,
     },
     Data {
@@ -46,99 +43,108 @@ pub enum RTCEvent {
 /// Handlers used to forward events to a channel
 /// Every callback here is called on the signaling thread
 
-fn on_connection_change(
+fn on_connection_state_change(
     target: proto::SignalTarget,
-    emitter: RTCEmitter,
+    emitter: RtcEmitter,
-) -> OnConnectionChangeHandler {
+) -> rtc::peer_connection::OnConnectionChange {
     Box::new(move |state| {
-        let _ = emitter.send(RTCEvent::ConnectionChange { state, target });
+        let _ = emitter.send(RtcEvent::ConnectionChange { state, target });
     })
 }
 
-fn on_ice_candidate(target: proto::SignalTarget, emitter: RTCEmitter) -> OnIceCandidateHandler {
+fn on_ice_candidate(
+    target: proto::SignalTarget,
+    emitter: RtcEmitter,
+) -> rtc::peer_connection::OnIceCandidate {
     Box::new(move |ice_candidate| {
-        let _ = emitter.send(RTCEvent::IceCandidate {
+        let _ = emitter.send(RtcEvent::IceCandidate {
             ice_candidate,
             target,
         });
     })
 }
 
-fn on_offer(target: proto::SignalTarget, emitter: RTCEmitter) -> OnOfferHandler {
+fn on_offer(target: proto::SignalTarget, emitter: RtcEmitter) -> OnOfferCreated {
     Box::new(move |offer| {
-        let _ = emitter.send(RTCEvent::Offer { offer, target });
+        let _ = emitter.send(RtcEvent::Offer { offer, target });
-
-        Box::pin(async {})
     })
 }
 
-fn on_data_channel(target: proto::SignalTarget, emitter: RTCEmitter) -> OnDataChannelHandler {
+fn on_data_channel(
-    Box::new(move |mut data_channel| {
+    target: proto::SignalTarget,
-        data_channel.on_message(on_message(emitter.clone()));
+    emitter: RtcEmitter,
+) -> rtc::peer_connection::OnDataChannel {
+    Box::new(move |data_channel| {
+        data_channel.on_message(Some(on_message(emitter.clone())));
 
-        let _ = emitter.send(RTCEvent::DataChannel {
+        let _ = emitter.send(RtcEvent::DataChannel {
             data_channel,
             target,
         });
     })
 }
 
-fn on_add_track(target: proto::SignalTarget, emitter: RTCEmitter) -> OnAddTrackHandler {
+fn on_track(target: proto::SignalTarget, emitter: RtcEmitter) -> rtc::peer_connection::OnTrack {
-    Box::new(move |rtp_receiver, streams| {
+    Box::new(move |event| {
-        let _ = emitter.send(RTCEvent::AddTrack {
+        let _ = emitter.send(RtcEvent::Track {
-            rtp_receiver,
+            receiver: event.receiver,
-            streams,
+            streams: event.streams,
+            track: event.track,
+            transceiver: event.transceiver,
             target,
         });
     })
 }
 
 fn on_ice_candidate_error(
-    target: proto::SignalTarget,
+    _target: proto::SignalTarget,
-    _emitter: RTCEmitter,
+    _emitter: RtcEmitter,
-) -> OnIceCandidateErrorHandler {
+) -> rtc::peer_connection::OnIceCandidateError {
-    Box::new(move |address, port, url, error_code, error_text| {
+    Box::new(move |ice_error| {
-        error!(
+        error!("{:?}", ice_error);
-            "ICE candidate error ({:?}): address: {} - port: {} - url: {} - error_code: {} - error_text: {}",
-            target, address, port, url, error_code, error_text
-        );
     })
 }
 
-pub fn forward_pc_events(transport: &mut PCTransport, rtc_emitter: RTCEmitter) {
+pub fn forward_pc_events(transport: &mut PeerTransport, rtc_emitter: RtcEmitter) {
     let signal_target = transport.signal_target();
     transport
         .peer_connection()
-        .on_ice_candidate(on_ice_candidate(signal_target, rtc_emitter.clone()));
+        .on_ice_candidate(Some(on_ice_candidate(signal_target, rtc_emitter.clone())));
 
     transport
         .peer_connection()
-        .on_data_channel(on_data_channel(signal_target, rtc_emitter.clone()));
+        .on_data_channel(Some(on_data_channel(signal_target, rtc_emitter.clone())));
 
     transport
         .peer_connection()
-        .on_add_track(on_add_track(signal_target, rtc_emitter.clone()));
+        .on_track(Some(on_track(signal_target, rtc_emitter.clone())));
 
     transport
         .peer_connection()
-        .on_connection_change(on_connection_change(signal_target, rtc_emitter.clone()));
+        .on_connection_state_change(Some(on_connection_state_change(
+            signal_target,
+            rtc_emitter.clone(),
+        )));
 
     transport
         .peer_connection()
-        .on_ice_candidate_error(on_ice_candidate_error(signal_target, rtc_emitter.clone()));
+        .on_ice_candidate_error(Some(on_ice_candidate_error(
+            signal_target,
+            rtc_emitter.clone(),
+        )));
 
-    transport.on_offer(on_offer(transport.signal_target(), rtc_emitter.clone()));
+    transport.on_offer(Some(on_offer(signal_target, rtc_emitter.clone())));
 }
 
-fn on_message(emitter: RTCEmitter) -> OnMessageHandler {
+fn on_message(emitter: RtcEmitter) -> rtc::data_channel::OnMessage {
-    Box::new(move |data, binary| {
+    Box::new(move |buffer| {
-        let _ = emitter.send(RTCEvent::Data {
+        let _ = emitter.send(RtcEvent::Data {
-            data: data.to_vec(),
+            data: buffer.data.to_vec(),
-            binary,
+            binary: buffer.binary,
         });
     })
 }
 
-pub fn forward_dc_events(dc: &mut DataChannel, rtc_emitter: RTCEmitter) {
+pub fn forward_dc_events(dc: &mut DataChannel, rtc_emitter: RtcEmitter) {
-    dc.on_message(on_message(rtc_emitter.clone()));
+    dc.on_message(Some(on_message(rtc_emitter.clone())));
 }

livekit/src/rtc_engine/rtc_session.rs

@@ -1,23 +1,28 @@
 use super::{rtc_events, EngineError, EngineResult, SimulateScenario};
-use crate::rtc_engine::lk_runtime::LKRuntime;
+use crate::options::TrackPublishOptions;
-use crate::rtc_engine::pc_transport::PCTransport;
+use crate::rtc_engine::lk_runtime::LkRuntime;
-use crate::rtc_engine::rtc_events::{RTCEvent, RTCEvents};
+use crate::rtc_engine::peer_transport::PeerTransport;
+use crate::rtc_engine::rtc_events::{RtcEvent, RtcEvents};
 use crate::signal_client::{SignalClient, SignalEvent, SignalEvents, SignalOptions};
+use crate::track::LocalTrack;
 use crate::{proto, signal_client};
 use livekit_webrtc::prelude::*;
 use parking_lot::Mutex;
 use prost::Message;
 use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
 use std::convert::TryInto;
 use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
 use std::sync::Arc;
 use std::time::Duration;
-use tokio::sync::{mpsc, watch, Mutex as AsyncMutex};
+use tokio::sync::Mutex as AsyncMutex;
+use tokio::sync::{mpsc, oneshot, watch};
 use tokio::task::JoinHandle;
 use tokio::time::sleep;
 use tracing::{debug, error, trace, warn};
 
-pub const MAX_ICE_CONNECT_TIMEOUT: Duration = Duration::from_secs(15);
+pub const ICE_CONNECT_TIMEOUT: Duration = Duration::from_secs(15);
+pub const TRACK_PUBLISH_TIMEOUT: Duration = Duration::from_secs(10);
 pub const LOSSY_DC_LABEL: &str = "_lossy";
 pub const RELIABLE_DC_LABEL: &str = "_reliable";
 
@@ -35,7 +40,7 @@ pub enum SessionEvent {
         kind: proto::data_packet::Kind,
     },
     MediaTrack {
-        track: MediaStreamTrackHandle,
+        track: MediaStreamTrack,
         stream: MediaStream,
         receiver: RtpReceiver,
     },
@@ -56,8 +61,8 @@ pub enum SessionEvent {
     Connected,
 }
 
-#[repr(u8)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum PCState {
+pub enum PeerState {
     New,
     Connected,
     Disconnected,
@@ -65,27 +70,27 @@ pub enum PCState {
     Closed,
 }
 
-impl TryInto<PCState> for u8 {
+impl TryFrom<u8> for PeerState {
     type Error = &'static str;
 
-    fn try_into(self) -> Result<PCState, Self::Error> {
+    fn try_from(v: u8) -> Result<Self, Self::Error> {
-        match self {
+        match v {
-            0 => Ok(PCState::New),
+            0 => Ok(Self::New),
-            1 => Ok(PCState::Connected),
+            1 => Ok(Self::Connected),
-            2 => Ok(PCState::Disconnected),
+            2 => Ok(Self::Disconnected),
-            3 => Ok(PCState::Reconnecting),
+            3 => Ok(Self::Reconnecting),
-            4 => Ok(PCState::Closed),
+            4 => Ok(Self::Closed),
-            _ => Err("invalid PCState"),
+            _ => Err("invalid PeerState"),
         }
     }
 }
 
 #[derive(Serialize, Deserialize)]
-#[allow(non_snake_case)]
+#[serde(rename_all = "camelCase")]
-struct IceCandidateJSON {
+struct IceCandidateJson {
-    sdpMid: String,
+    pub sdp_mid: String,
-    sdpMLineIndex: i32,
+    pub sdp_m_line_index: i32,
-    candidate: String,
+    pub candidate: String,
 }
 
 #[derive(Debug, Clone, Default)]
@@ -104,8 +109,10 @@ struct SessionInner {
     pc_state: AtomicU8, // PCState
     has_published: AtomicBool,
 
-    publisher_pc: AsyncMutex<PCTransport>,
+    publisher_pc: AsyncMutex<PeerTransport>,
-    subscriber_pc: AsyncMutex<PCTransport>,
+    subscriber_pc: AsyncMutex<PeerTransport>,
+
+    pending_tracks: Mutex<HashMap<String, oneshot::Sender<proto::TrackInfo>>>,
 
     // Publisher data channels
     // used to send data to other participants ( The SFU forwards the messages )
@@ -124,20 +131,21 @@ struct SessionInner {
 ///
 /// RTCSession is also responsable for the signaling and the negotation
 #[derive(Debug)]
-pub struct RTCSession {
+pub struct RtcSession {
-    lk_runtime: Arc<LKRuntime>,
+    #[allow(dead_code)]
+    lk_runtime: Arc<LkRuntime>,
     inner: Arc<SessionInner>,
     close_tx: watch::Sender<bool>, // false = is_running
     signal_task: JoinHandle<()>,
     rtc_task: JoinHandle<()>,
 }
 
-impl RTCSession {
+impl RtcSession {
     pub async fn connect(
         url: &str,
         token: &str,
         options: SignalOptions,
-        lk_runtime: Arc<LKRuntime>,
+        lk_runtime: Arc<LkRuntime>,
         session_emitter: SessionEmitter,
     ) -> EngineResult<Self> {
         // Connect to the SignalClient
@@ -148,16 +156,16 @@ impl RTCSession {
         debug!("received JoinResponse: {:?}", join_response);
 
         let (rtc_emitter, rtc_events) = mpsc::unbounded_channel();
-        let rtc_config = RTCConfiguration::from(join_response.clone());
+        let rtc_config = RtcConfiguration::from(join_response.clone());
 
-        let mut publisher_pc = PCTransport::new(
+        let mut publisher_pc = PeerTransport::new(
             lk_runtime
                 .pc_factory
                 .create_peer_connection(rtc_config.clone())?,
             proto::SignalTarget::Publisher,
         );
 
-        let mut subscriber_pc = PCTransport::new(
+        let mut subscriber_pc = PeerTransport::new(
             lk_runtime
                 .pc_factory
                 .create_peer_connection(rtc_config.clone())?,
@@ -181,7 +189,7 @@ impl RTCSession {
             },
         )?;
 
-        // Forward events received in the Signaling Thread to our rtc channel
+        // Forward events received inside the signaling thread to our rtc channel
         rtc_events::forward_pc_events(&mut publisher_pc, rtc_emitter.clone());
         rtc_events::forward_pc_events(&mut subscriber_pc, rtc_emitter.clone());
         rtc_events::forward_dc_events(&mut lossy_dc, rtc_emitter.clone());
@@ -197,11 +205,12 @@ impl RTCSession {
         let (close_tx, close_rx) = watch::channel(false);
         let inner = Arc::new(SessionInner {
             info: session_info,
-            pc_state: AtomicU8::new(PCState::New as u8),
+            pc_state: AtomicU8::new(PeerState::New as u8),
             has_published: Default::default(),
             signal_client,
             publisher_pc: AsyncMutex::new(publisher_pc),
             subscriber_pc: AsyncMutex::new(subscriber_pc),
+            pending_tracks: Default::default(),
             lossy_dc,
             reliable_dc,
             subscriber_dc: Default::default(),
@@ -228,6 +237,31 @@ impl RTCSession {
         Ok(session)
     }
 
+    #[inline]
+    pub async fn add_track(&self, req: proto::AddTrackRequest) -> EngineResult<proto::TrackInfo> {
+        self.inner.add_track(req).await
+    }
+
+    #[inline]
+    pub async fn remove_track(&self, sender: RtpSender) -> EngineResult<()> {
+        self.inner.remove_track(sender).await
+    }
+
+    #[inline]
+    pub async fn create_sender(
+        &self,
+        track: LocalTrack,
+        options: TrackPublishOptions,
+        encodings: Vec<RtpEncodingParameters>,
+    ) -> EngineResult<RtpTransceiver> {
+        self.inner.create_sender(track, options, encodings).await
+    }
+
+    #[inline]
+    pub async fn negotiate_publisher(&self) -> EngineResult<()> {
+        self.inner.negotiate_publisher().await
+    }
+
     /// Close the PeerConnections and the SignalClient
     #[tracing::instrument]
     pub async fn close(self) {
@@ -238,6 +272,7 @@ impl RTCSession {
         let _ = self.signal_task.await;
     }
 
+    #[inline]
     pub async fn publish_data(
         &self,
         data: &proto::DataPacket,
@@ -246,25 +281,28 @@ impl RTCSession {
         self.inner.publish_data(data, kind).await
     }
 
+    #[inline]
     pub async fn restart(&self) -> EngineResult<()> {
         self.inner.restart_session().await
     }
 
+    #[inline]
     pub async fn wait_pc_connection(&self) -> EngineResult<()> {
         self.inner.wait_pc_connection().await
     }
 
+    #[inline]
     pub async fn simulate_scenario(&self, scenario: SimulateScenario) {
         self.inner.simulate_scenario(scenario).await
     }
-}
 
-impl RTCSession {
+    #[inline]
     pub fn info(&self) -> &SessionInfo {
         &self.inner.info
     }
 
-    pub fn state(&self) -> PCState {
+    #[inline]
+    pub fn state(&self) -> PeerState {
         self.inner
             .pc_state
             .load(Ordering::SeqCst)
@@ -272,18 +310,22 @@ impl RTCSession {
             .unwrap()
     }
 
-    pub fn publisher(&self) -> &AsyncMutex<PCTransport> {
+    #[inline]
+    pub fn publisher(&self) -> &AsyncMutex<PeerTransport> {
         &self.inner.publisher_pc
     }
 
-    pub fn subscriber(&self) -> &AsyncMutex<PCTransport> {
+    #[inline]
+    pub fn subscriber(&self) -> &AsyncMutex<PeerTransport> {
         &self.inner.subscriber_pc
     }
 
+    #[inline]
     pub fn signal_client(&self) -> &Arc<SignalClient> {
         &self.inner.signal_client
     }
 
+    #[inline]
     pub fn data_channel(&self, kind: proto::data_packet::Kind) -> &DataChannel {
         &self.inner.data_channel(kind)
     }
@@ -292,7 +334,7 @@ impl RTCSession {
 impl SessionInner {
     async fn rtc_task(
         self: Arc<Self>,
-        mut rtc_events: RTCEvents,
+        mut rtc_events: RtcEvents,
         mut close_receiver: watch::Receiver<bool>,
     ) {
         loop {
@@ -355,7 +397,8 @@ impl SessionInner {
         match event {
             proto::signal_response::Message::Answer(answer) => {
                 trace!("received publisher answer: {:?}", answer);
-                let answer = SessionDescription::from(answer.r#type.parse().unwrap(), &answer.sdp)?;
+                let answer =
+                    SessionDescription::parse(&answer.sdp, answer.r#type.parse().unwrap())?;
                 self.publisher_pc
                     .lock()
                     .await
@@ -364,12 +407,12 @@ impl SessionInner {
             }
             proto::signal_response::Message::Offer(offer) => {
                 trace!("received subscriber offer: {:?}", offer);
-                let offer = SessionDescription::from(offer.r#type.parse().unwrap(), &offer.sdp)?;
+                let offer = SessionDescription::parse(&offer.sdp, offer.r#type.parse().unwrap())?;
                 let answer = self
                     .subscriber_pc
                     .lock()
                     .await
-                    .create_anwser(offer, RTCOfferAnswerOptions::default())
+                    .create_anwser(offer, AnswerOptions::default())
                     .await?;
 
                 self.signal_client
@@ -384,11 +427,11 @@ impl SessionInner {
             proto::signal_response::Message::Trickle(trickle) => {
                 let target = proto::SignalTarget::from_i32(trickle.target).unwrap();
                 let ice_candidate = {
-                    let json = serde_json::from_str::<IceCandidateJSON>(&trickle.candidate_init)?;
+                    let json = serde_json::from_str::<IceCandidateJson>(&trickle.candidate_init)?;
-                    IceCandidate::from(&json.sdpMid, json.sdpMLineIndex, &json.candidate)?
+                    IceCandidate::parse(&json.sdp_mid, json.sdp_m_line_index, &json.candidate)?
                 };
 
-                trace!("received ice_candidate {:?} {:?}", target, ice_candidate);
+                debug!("received ice_candidate {:?} {:?}", target, ice_candidate);
 
                 if target == proto::SignalTarget::Publisher {
                     self.publisher_pc
@@ -428,24 +471,31 @@ impl SessionInner {
                     updates: quality.updates,
                 });
             }
+            proto::signal_response::Message::TrackPublished(publish_res) => {
+                let mut pending_tracks = self.pending_tracks.lock();
+                if let Some(tx) = pending_tracks.remove(&publish_res.cid) {
+                    let _ = tx.send(publish_res.track.unwrap());
+                }
+            }
+
             _ => {}
         }
 
         Ok(())
     }
 
-    async fn on_rtc_event(&self, event: RTCEvent) -> EngineResult<()> {
+    async fn on_rtc_event(&self, event: RtcEvent) -> EngineResult<()> {
         match event {
-            RTCEvent::IceCandidate {
+            RtcEvent::IceCandidate {
                 ice_candidate,
                 target,
             } => {
                 self.signal_client
                     .send(proto::signal_request::Message::Trickle(
                         proto::TrickleRequest {
-                            candidate_init: serde_json::to_string(&IceCandidateJSON {
+                            candidate_init: serde_json::to_string(&IceCandidateJson {
-                                sdpMid: ice_candidate.sdp_mid(),
+                                sdp_mid: ice_candidate.sdp_mid(),
-                                sdpMLineIndex: ice_candidate.sdp_mline_index(),
+                                sdp_m_line_index: ice_candidate.sdp_mline_index(),
                                 candidate: ice_candidate.candidate(),
                             })?,
                             target: target as i32,
@@ -453,21 +503,21 @@ impl SessionInner {
                     ))
                     .await;
             }
-            RTCEvent::ConnectionChange { state, target } => {
+            RtcEvent::ConnectionChange { state, target } => {
-                trace!("connection change, {:?} {:?}", state, target);
+                debug!("connection change, {:?} {:?}", state, target);
                 let is_primary = self.info.join_response.subscriber_primary
                     && target == proto::SignalTarget::Subscriber;
 
                 if is_primary && state == PeerConnectionState::Connected {
                     let old_state = self
                         .pc_state
-                        .swap(PCState::Connected as u8, Ordering::SeqCst);
+                        .swap(PeerState::Connected as u8, Ordering::SeqCst);
-                    if old_state == PCState::New as u8 {
+                    if old_state == PeerState::New as u8 {
                         let _ = self.emitter.send(SessionEvent::Connected);
                     }
                 } else if state == PeerConnectionState::Failed {
                     self.pc_state
-                        .store(PCState::Disconnected as u8, Ordering::SeqCst);
+                        .store(PeerState::Disconnected as u8, Ordering::SeqCst);
 
                     self.on_session_disconnected(
                         "pc_state failed",
@@ -478,14 +528,15 @@ impl SessionInner {
                     );
                 }
             }
-            RTCEvent::DataChannel {
+            RtcEvent::DataChannel {
                 data_channel,
                 target: _,
             } => {
                 self.subscriber_dc.lock().push(data_channel);
             }
-            RTCEvent::Offer { offer, target: _ } => {
+            RtcEvent::Offer { offer, target: _ } => {
                 // Send the publisher offer to the server
+                debug!("sending publisher offer: {:?}", offer);
                 self.signal_client
                     .send(proto::signal_request::Message::Offer(
                         proto::SessionDescription {
@@ -495,22 +546,24 @@ impl SessionInner {
                     ))
                     .await;
             }
-            RTCEvent::AddTrack {
+            RtcEvent::Track {
-                rtp_receiver,
+                receiver,
                 mut streams,
+                track,
+                transceiver: _,
                 target: _,
             } => {
                 if !streams.is_empty() {
                     let _ = self.emitter.send(SessionEvent::MediaTrack {
-                        track: rtp_receiver.track(),
                         stream: streams.remove(0),
-                        receiver: rtp_receiver,
+                        track,
+                        receiver,
                     });
                 } else {
-                    warn!("AddTrack event with no streams");
+                    warn!("Track event with no streams");
                 }
             }
-            RTCEvent::Data { data, binary } => {
+            RtcEvent::Data { data, binary } => {
                 if !binary {
                     Err(EngineError::Internal(
                         "text messages aren't supported".to_string(),
@@ -534,6 +587,106 @@ impl SessionInner {
         Ok(())
     }
 
+    async fn add_track(&self, req: proto::AddTrackRequest) -> EngineResult<proto::TrackInfo> {
+        let (tx, rx) = oneshot::channel();
+        let cid = req.cid.clone();
+        {
+            let mut pendings_tracks = self.pending_tracks.lock();
+            if pendings_tracks.contains_key(&req.cid) {
+                Err(EngineError::Internal("track already published".to_string()))?;
+            }
+
+            pendings_tracks.insert(cid.clone(), tx);
+        }
+
+        self.signal_client
+            .send(proto::signal_request::Message::AddTrack(req))
+            .await;
+
+        // Wait the result from the server (TrackInfo)
+        tokio::select! {
+            Ok(info) = rx => Ok(info),
+            _ = sleep(TRACK_PUBLISH_TIMEOUT) => {
+                self.pending_tracks.lock().remove(&cid);
+                Err(EngineError::Internal("track publication timed out, no response received from the server".to_string()))
+            },
+            else => {
+                Err(EngineError::Internal(
+                    "track publication cancelled".to_string(),
+                ))
+            }
+        }
+    }
+
+    async fn remove_track(&self, sender: RtpSender) -> EngineResult<()> {
+        if let Some(track) = sender.track() {
+            let mut pending_tracks = self.pending_tracks.lock();
+            pending_tracks.remove(&track.id());
+        }
+
+        self.publisher_pc
+            .lock()
+            .await
+            .peer_connection()
+            .remove_track(sender)?;
+
+        Ok(())
+    }
+
+    async fn create_sender(
+        &self,
+        track: LocalTrack,
+        options: TrackPublishOptions,
+        encodings: Vec<RtpEncodingParameters>,
+    ) -> EngineResult<RtpTransceiver> {
+        let init = RtpTransceiverInit {
+            direction: RtpTransceiverDirection::SendOnly,
+            stream_ids: Default::default(),
+            send_encodings: encodings,
+        };
+
+        let transceiver = self
+            .publisher_pc
+            .lock()
+            .await
+            .peer_connection()
+            .add_transceiver(track.rtc_track(), init)?;
+
+        let capabilities = LkRuntime::instance()
+            .pc_factory
+            .get_rtp_sender_capabilities(track.kind().into());
+
+        let mut matched = Vec::new();
+        let mut partial_matched = Vec::new();
+        let mut unmatched = Vec::new();
+
+        for codec in capabilities.codecs {
+            let mime_type = codec.mime_type.to_lowercase();
+            if mime_type == "audio/opus" {
+                matched.push(codec);
+            } else if mime_type == format!("video/{}", options.video_codec.as_str()) {
+                if let Some(sdp_fmtp_line) = codec.sdp_fmtp_line.as_ref() {
+                    // for h264 codecs that have sdpFmtpLine available, use only if the
+                    // profile-level-id is 42e01f for cross-browser compatibility
+                    if sdp_fmtp_line.contains("profile-level-id=42e01f") {
+                        matched.push(codec);
+                        continue;
+                    }
+                }
+                partial_matched.push(codec);
+            } else {
+                unmatched.push(codec);
+            }
+        }
+
+        matched.append(&mut partial_matched);
+        matched.append(&mut unmatched);
+
+        transceiver.set_codec_preferences(matched)?;
+
+        Ok(transceiver)
+    }
+
     /// Called when the SignalClient or one of the PeerConnection has lost the connection
     /// The RTCEngine may try a reconnect.
     fn on_session_disconnected(
@@ -663,7 +816,7 @@ impl SessionInner {
             self.publisher_pc
                 .lock()
                 .await
-                .create_and_send_offer(RTCOfferAnswerOptions {
+                .create_and_send_offer(OfferOptions {
                     ice_restart: true,
                     ..Default::default()
                 })
@@ -680,7 +833,7 @@ impl SessionInner {
     // Timeout after ['MAX_ICE_CONNECT_TIMEOUT']
     async fn wait_pc_connection(&self) -> EngineResult<()> {
         let wait_connected = async move {
-            while self.pc_state.load(Ordering::Acquire) != PCState::Connected as u8 {
+            while self.pc_state.load(Ordering::Acquire) != PeerState::Connected as u8 {
                 if self.closed.load(Ordering::Acquire) {
                     return Err(EngineError::Connection("closed".to_string()));
                 }
@@ -693,7 +846,7 @@ impl SessionInner {
 
         tokio::select! {
             res = wait_connected => res,
-            _ = sleep(MAX_ICE_CONNECT_TIMEOUT) => {
+            _ = sleep(ICE_CONNECT_TIMEOUT) => {
                 let err = EngineError::Connection("wait_pc_connection timed out".to_string());
                 Err(err)
             }
@@ -724,7 +877,7 @@ impl SessionInner {
                 .await
                 .peer_connection()
                 .ice_connection_state()
-                != IceConnectionState::IceConnectionChecking
+                != IceConnectionState::Checking
         {
             let _ = self.negotiate_publisher().await;
         }
@@ -749,7 +902,7 @@ impl SessionInner {
 
         tokio::select! {
             res = wait_connected => res,
-            _ = sleep(MAX_ICE_CONNECT_TIMEOUT) => {
+            _ = sleep(ICE_CONNECT_TIMEOUT) => {
                 let err = EngineError::Connection("could not establish publisher connection: timeout".to_string());
                 error!(error = ?err);
                 Err(err)

livekit/src/signal_client/mod.rs

@@ -114,13 +114,13 @@ impl SignalClient {
     }
 }
 
-impl From<proto::JoinResponse> for RTCConfiguration {
+impl From<proto::JoinResponse> for RtcConfiguration {
     fn from(join_response: proto::JoinResponse) -> Self {
         Self {
             ice_servers: {
                 let mut servers = vec![];
                 for ice_server in join_response.ice_servers.clone() {
-                    servers.push(ICEServer {
+                    servers.push(IceServer {
                         urls: ice_server.urls,
                         username: ice_server.username,
                         password: ice_server.credential,

livekit/src/signal_client/signal_stream.rs

@@ -34,7 +34,7 @@ enum InternalMessage {
 ///
 /// It is replaced by [SignalClient] at each reconnection.
 #[derive(Debug)]
-pub(crate) struct SignalStream {
+pub(super) struct SignalStream {
     internal_tx: mpsc::Sender<InternalMessage>,
     read_handle: JoinHandle<()>,
     write_handle: JoinHandle<()>,
@@ -46,7 +46,7 @@ impl SignalStream {
     ///
     /// SignalStream will never try to reconnect if the connection has been
     /// closed.
-    pub(super) async fn connect(
+    pub async fn connect(
         url: &str,
         token: &str,
         options: SignalOptions,

webrtc-sys/build.rs

@@ -7,7 +7,7 @@ use std::path;
 use std::process::Command;
 use tar::Archive;
 
-const WEBRTC_TAG: &str = "m104.5112.08";
+const WEBRTC_TAG: &str = "m104.5112.09";
 
 fn download_prebuilt(
     target_os: &str,

webrtc-sys/include/livekit/data_channel.h

@@ -37,7 +37,7 @@ class DataChannel {
       std::shared_ptr<RTCRuntime> rtc_runtime,
       rtc::scoped_refptr<webrtc::DataChannelInterface> data_channel);
 
-  void register_observer(NativeDataChannelObserver& observer) const;
+  void register_observer(NativeDataChannelObserver* observer) const;
   void unregister_observer() const;
   bool send(const DataBuffer& buffer) const;
   rust::String label() const;
@@ -52,14 +52,17 @@ class DataChannel {
 std::unique_ptr<NativeDataChannelInit> create_data_channel_init(
     DataChannelInit init);
 
-static std::unique_ptr<DataChannel> _unique_data_channel() {
+static std::shared_ptr<DataChannel> _shared_data_channel() {
   return nullptr;  // Ignore
 }
 
 class NativeDataChannelObserver : public webrtc::DataChannelObserver {
  public:
   explicit NativeDataChannelObserver(
-      rust::Box<DataChannelObserverWrapper> observer);
+      rust::Box<DataChannelObserverWrapper> observer,
+      DataChannel* dc);
+
+  ~NativeDataChannelObserver();
 
   void OnStateChange() override;
   void OnMessage(const webrtc::DataBuffer& buffer) override;
@@ -67,8 +70,10 @@ class NativeDataChannelObserver : public webrtc::DataChannelObserver {
 
  private:
   rust::Box<DataChannelObserverWrapper> observer_;
+  DataChannel* dc_;
 };
 
-std::unique_ptr<NativeDataChannelObserver> create_native_data_channel_observer(
+std::shared_ptr<NativeDataChannelObserver> create_native_data_channel_observer(
-    rust::Box<DataChannelObserverWrapper> observer);
+    rust::Box<DataChannelObserverWrapper> observer,
+    DataChannel* dc);
 }  // namespace livekit

webrtc-sys/include/livekit/jsep.h

@@ -65,6 +65,7 @@ class SessionDescription {
   explicit SessionDescription(
       std::unique_ptr<webrtc::SessionDescriptionInterface> session_description);
 
+  SdpType sdp_type() const;
   rust::String stringify() const;
   std::unique_ptr<SessionDescription> clone() const;
   std::unique_ptr<webrtc::SessionDescriptionInterface> release();

webrtc-sys/include/livekit/media_stream.h

@@ -26,6 +26,7 @@
 #include "rtc_base/synchronization/mutex.h"
 #include "rtc_base/timestamp_aligner.h"
 #include "rust/cxx.h"
+#include "system_wrappers/include/clock.h"
 
 namespace livekit {
 class MediaStream;
@@ -50,10 +51,8 @@ class MediaStream {
   std::shared_ptr<AudioTrack> find_audio_track(rust::String track_id) const;
   std::shared_ptr<VideoTrack> find_video_track(rust::String track_id) const;
 
-  bool add_audio_track(std::shared_ptr<AudioTrack> audio_track) const;
+  bool add_track(std::shared_ptr<MediaStreamTrack> track) const;
-  bool add_video_track(std::shared_ptr<VideoTrack> video_track) const;
+  bool remove_track(std::shared_ptr<MediaStreamTrack> track) const;
-  bool remove_audio_track(std::shared_ptr<AudioTrack> audio_track) const;
-  bool remove_video_track(std::shared_ptr<VideoTrack> video_track) const;
 
  private:
   rtc::scoped_refptr<webrtc::MediaStreamInterface> media_stream_;
@@ -121,7 +120,7 @@ class NativeVideoFrameSink
   rust::Box<VideoFrameSinkWrapper> observer_;
 };
 
-std::unique_ptr<NativeVideoFrameSink> create_native_video_frame_sink(
+std::unique_ptr<NativeVideoFrameSink> new_native_video_frame_sink(
     rust::Box<VideoFrameSinkWrapper> observer);
 
 // Native impl of the WebRTC interface
@@ -146,7 +145,7 @@ class AdaptedVideoTrackSource {
  public:
   AdaptedVideoTrackSource(rtc::scoped_refptr<NativeVideoTrackSource> source);
 
-  bool on_captured_frame(std::unique_ptr<VideoFrame> frame)
+  bool on_captured_frame(const std::unique_ptr<VideoFrame>& frame)
       const;  // frames pushed from Rust (+interior mutability)
 
   rtc::scoped_refptr<NativeVideoTrackSource> get() const;
@@ -155,14 +154,26 @@ class AdaptedVideoTrackSource {
   rtc::scoped_refptr<NativeVideoTrackSource> source_;
 };
 
-std::unique_ptr<AdaptedVideoTrackSource> create_adapted_video_track_source();
+std::shared_ptr<AdaptedVideoTrackSource> new_adapted_video_track_source();
 
-static const VideoTrack* media_to_video(const MediaStreamTrack* track) {
+static std::shared_ptr<MediaStreamTrack> video_to_media(
-  return static_cast<const VideoTrack*>(track);
+    std::shared_ptr<VideoTrack> track) {
+  return track;
 }
 
-static const AudioTrack* media_to_audio(const MediaStreamTrack* track) {
+static std::shared_ptr<MediaStreamTrack> audio_to_media(
-  return static_cast<const AudioTrack*>(track);
+    std::shared_ptr<AudioTrack> track) {
+  return track;
+}
+
+static std::shared_ptr<VideoTrack> media_to_video(
+    std::shared_ptr<MediaStreamTrack> track) {
+  return std::static_pointer_cast<VideoTrack>(track);
+}
+
+static std::shared_ptr<AudioTrack> media_to_audio(
+    std::shared_ptr<MediaStreamTrack> track) {
+  return std::static_pointer_cast<AudioTrack>(track);
 }
 
 static std::shared_ptr<MediaStreamTrack> _shared_media_stream_track() {

webrtc-sys/include/livekit/peer_connection.h

@@ -57,7 +57,7 @@ class PeerConnection {
   void set_remote_description(std::unique_ptr<SessionDescription> desc,
                               NativeSetRemoteSdpObserverHandle& observer) const;
 
-  std::unique_ptr<DataChannel> create_data_channel(
+  std::shared_ptr<DataChannel> create_data_channel(
       rust::String label,
       std::unique_ptr<NativeDataChannelInit> init) const;
 
@@ -84,24 +84,34 @@ class PeerConnection {
 
   rust::Vec<RtpTransceiverPtr> get_transceivers() const;
 
+  std::unique_ptr<SessionDescription> current_local_description() const;
+
+  std::unique_ptr<SessionDescription> current_remote_description() const;
+
+  std::unique_ptr<SessionDescription> pending_local_description() const;
+
+  std::unique_ptr<SessionDescription> pending_remote_description() const;
+
   std::unique_ptr<SessionDescription> local_description() const;
 
   std::unique_ptr<SessionDescription> remote_description() const;
 
+  PeerConnectionState connection_state() const;
+
   SignalingState signaling_state() const;
 
   IceGatheringState ice_gathering_state() const;
 
   IceConnectionState ice_connection_state() const;
 
-  void close();
+  void close() const;
 
  private:
   std::shared_ptr<RTCRuntime> rtc_runtime_;
   rtc::scoped_refptr<webrtc::PeerConnectionInterface> peer_connection_;
 };
 
-static std::unique_ptr<PeerConnection> _unique_peer_connection() {
+static std::shared_ptr<PeerConnection> _shared_peer_connection() {
   return nullptr;  // Ignore
 }
 
@@ -126,6 +136,8 @@ class NativePeerConnectionObserver : public webrtc::PeerConnectionObserver {
       std::shared_ptr<RTCRuntime> rtc_runtime,
       rust::Box<PeerConnectionObserverWrapper> observer);
 
+  ~NativePeerConnectionObserver();
+
   void OnSignalingChange(
       webrtc::PeerConnectionInterface::SignalingState new_state) override;
 
@@ -188,7 +200,7 @@ class NativePeerConnectionObserver : public webrtc::PeerConnectionObserver {
   rust::Box<PeerConnectionObserverWrapper> observer_;
 };
 
-std::unique_ptr<NativePeerConnectionObserver>
+std::shared_ptr<NativePeerConnectionObserver>
 create_native_peer_connection_observer(
     std::shared_ptr<RTCRuntime> rtc_runtime,
     rust::Box<PeerConnectionObserverWrapper> observer);

webrtc-sys/include/livekit/peer_connection_factory.h

@@ -17,7 +17,10 @@
 #pragma once
 
 #include "api/peer_connection_interface.h"
+#include "media_stream.h"
 #include "peer_connection.h"
+#include "rtp_parameters.h"
+#include "rust/cxx.h"
 #include "webrtc.h"
 
 namespace livekit {
@@ -35,16 +38,24 @@ class PeerConnectionFactory {
   explicit PeerConnectionFactory(std::shared_ptr<RTCRuntime> rtc_runtime);
   ~PeerConnectionFactory();
 
-  std::unique_ptr<PeerConnection> create_peer_connection(
+  std::shared_ptr<PeerConnection> create_peer_connection(
       std::unique_ptr<NativeRTCConfiguration> config,
-      NativePeerConnectionObserver& observer) const;
+      NativePeerConnectionObserver* observer) const;
+
+  std::shared_ptr<VideoTrack> create_video_track(
+      rust::String label,
+      std::shared_ptr<AdaptedVideoTrackSource> source) const;
+
+  RtpCapabilities get_rtp_sender_capabilities(MediaType type) const;
+
+  RtpCapabilities get_rtp_receiver_capabilities(MediaType type) const;
 
  private:
   std::shared_ptr<RTCRuntime> rtc_runtime_;
   rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface> peer_factory_;
 };
 
-std::unique_ptr<PeerConnectionFactory> create_peer_connection_factory(
+std::shared_ptr<PeerConnectionFactory> create_peer_connection_factory(
     std::shared_ptr<RTCRuntime> rtc_runtime);
 std::unique_ptr<NativeRTCConfiguration> create_rtc_configuration(
     RTCConfiguration conf);

webrtc-sys/include/livekit/video_frame.h

@@ -57,7 +57,7 @@ class VideoFrameBuilder {
   VideoFrameBuilder() = default;
 
   // TODO(theomonnom): other setters?
-  void set_video_frame_buffer(std::unique_ptr<VideoFrameBuffer> buffer);
+  void set_video_frame_buffer(const VideoFrameBuffer& buffer);
   void set_timestamp_us(int64_t timestamp_us);
   void set_rotation(VideoRotation rotation);
   void set_id(uint16_t id);
@@ -67,6 +67,6 @@ class VideoFrameBuilder {
   webrtc::VideoFrame::Builder builder_;
 };
 
-std::unique_ptr<VideoFrameBuilder> create_video_frame_builder();
+std::unique_ptr<VideoFrameBuilder> new_video_frame_builder();
 
 }  // namespace livekit

webrtc-sys/include/livekit/video_frame_buffer.h

@@ -49,8 +49,9 @@ class VideoFrameBuffer {
   int width() const;
   int height() const;
 
-  // Takes ownership
+  std::unique_ptr<I420Buffer> to_i420() const;
-  std::unique_ptr<I420Buffer> to_i420();
+
+  // Requires ownership
   std::unique_ptr<I420Buffer> get_i420();
   std::unique_ptr<I420ABuffer> get_i420a();
   std::unique_ptr<I422Buffer> get_i422();
@@ -131,7 +132,9 @@ class BiplanarYuv8Buffer : public BiplanarYuvBuffer {
   webrtc::BiplanarYuv8Buffer* buffer() const;
 };
 
-std::unique_ptr<I420Buffer> create_i420_buffer(int width, int height);
+std::unique_ptr<I420Buffer> new_i420_buffer(int width, int height);
+std::unique_ptr<I420Buffer> copy_i420_buffer(
+    const std::unique_ptr<I420Buffer>& i420);
 
 class I420Buffer : public PlanarYuv8Buffer {
  public:
@@ -141,6 +144,12 @@ class I420Buffer : public PlanarYuv8Buffer {
 class I420ABuffer : public I420Buffer {
  public:
   explicit I420ABuffer(rtc::scoped_refptr<webrtc::I420ABufferInterface> buffer);
+
+  int stride_a() const;
+  const uint8_t* data_a() const;
+
+ private:
+  webrtc::I420ABufferInterface* buffer() const;
 };
 
 class I422Buffer : public PlanarYuv8Buffer {

webrtc-sys/include/livekit/webrtc.h

@@ -18,6 +18,7 @@
 
 #include "rtc_base/physical_socket_server.h"
 #include "rtc_base/ssl_adapter.h"
+#include "rust/cxx.h"
 
 #ifdef WEBRTC_WIN
 #include "rtc_base/win32_socket_init.h"
@@ -54,6 +55,8 @@ class RTCRuntime {
 #endif
 };
 
+rust::String create_random_uuid();
+
 std::shared_ptr<RTCRuntime> create_rtc_runtime();
 
 }  // namespace livekit

webrtc-sys/include/livekit/yuv_helper.h

@@ -17,12 +17,19 @@
 #pragma once
 
 #include <memory>
+#include <stdexcept>
+#include <string>
 
 #include "api/video/yuv_helper.h"
 #include "webrtc-sys/src/yuv_helper.rs.h"
 
 namespace livekit {
 
+#define THROW_ON_ERROR(ret)                                           \
+  if (ret != 0) {                                                     \
+    throw std::runtime_error("libyuv error: " + std::to_string(ret)); \
+  }
+
 static void i420_to_argb(const uint8_t* src_y,
                          int src_stride_y,
                          const uint8_t* src_u,
@@ -33,8 +40,9 @@ static void i420_to_argb(const uint8_t* src_y,
                          int dst_stride_argb,
                          int width,
                          int height) {
-  webrtc::I420ToARGB(src_y, src_stride_y, src_u, src_stride_u, src_v,
+  THROW_ON_ERROR(webrtc::I420ToARGB(src_y, src_stride_y, src_u, src_stride_u,
-                     src_stride_v, dst_argb, dst_stride_argb, width, height);
+                                    src_v, src_stride_v, dst_argb,
+                                    dst_stride_argb, width, height));
 }
 
 static void i420_to_bgra(const uint8_t* src_y,
@@ -47,8 +55,9 @@ static void i420_to_bgra(const uint8_t* src_y,
                          int dst_stride_bgra,
                          int width,
                          int height) {
-  webrtc::I420ToBGRA(src_y, src_stride_y, src_u, src_stride_u, src_v,
+  THROW_ON_ERROR(webrtc::I420ToBGRA(src_y, src_stride_y, src_u, src_stride_u,
-                     src_stride_v, dst_bgra, dst_stride_bgra, width, height);
+                                    src_v, src_stride_v, dst_bgra,
+                                    dst_stride_bgra, width, height));
 }
 
 static void i420_to_abgr(const uint8_t* src_y,
@@ -61,8 +70,9 @@ static void i420_to_abgr(const uint8_t* src_y,
                          int dst_stride_abgr,
                          int width,
                          int height) {
-  webrtc::I420ToABGR(src_y, src_stride_y, src_u, src_stride_u, src_v,
+  THROW_ON_ERROR(webrtc::I420ToABGR(src_y, src_stride_y, src_u, src_stride_u,
-                     src_stride_v, dst_abgr, dst_stride_abgr, width, height);
+                                    src_v, src_stride_v, dst_abgr,
+                                    dst_stride_abgr, width, height));
 }
 
 static void i420_to_rgba(const uint8_t* src_y,
@@ -75,8 +85,49 @@ static void i420_to_rgba(const uint8_t* src_y,
                          int dst_stride_rgba,
                          int width,
                          int height) {
-  webrtc::I420ToRGBA(src_y, src_stride_y, src_u, src_stride_u, src_v,
+  THROW_ON_ERROR(webrtc::I420ToRGBA(src_y, src_stride_y, src_u, src_stride_u,
-                     src_stride_v, dst_rgba, dst_stride_rgba, width, height);
+                                    src_v, src_stride_v, dst_rgba,
+                                    dst_stride_rgba, width, height));
+}
+
+static void argb_to_i420(const uint8_t* src_argb,
+                         int src_stride_argb,
+                         uint8_t* dst_y,
+                         int dst_stride_y,
+                         uint8_t* dst_u,
+                         int dst_stride_u,
+                         uint8_t* dst_v,
+                         int dst_stride_v,
+                         int width,
+                         int height) {
+  THROW_ON_ERROR(webrtc::ARGBToI420(src_argb, src_stride_argb, dst_y,
+                                    dst_stride_y, dst_u, dst_stride_u, dst_v,
+                                    dst_stride_v, width, height));
+}
+
+static void abgr_to_i420(const uint8_t* src_abgr,
+                         int src_stride_abgr,
+                         uint8_t* dst_y,
+                         int dst_stride_y,
+                         uint8_t* dst_u,
+                         int dst_stride_u,
+                         uint8_t* dst_v,
+                         int dst_stride_v,
+                         int width,
+                         int height) {
+  THROW_ON_ERROR(webrtc::ABGRToI420(src_abgr, src_stride_abgr, dst_y,
+                                    dst_stride_y, dst_u, dst_stride_u, dst_v,
+                                    dst_stride_v, width, height));
+}
+
+static void argb_to_rgb24(const uint8_t* src_argb,
+                          int src_stride_argb,
+                          uint8_t* dst_rgb24,
+                          int dst_stride_rgb24,
+                          int width,
+                          int height) {
+  THROW_ON_ERROR(webrtc::ARGBToRGB24(src_argb, src_stride_argb, dst_rgb24,
+                                     dst_stride_rgb24, width, height));
 }
 
 }  // namespace livekit