talksik/client-sdk-rust
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: video publishing (#42)

Browse Source 
- Prepare webrtc abstraction ( for future wasm support )
- Added track publish support for videos
  - Added LogoTrack example to simple_room demo
- Lot of cleanup
- There are compiler warnings I'll solve on our v1 release
This commit is contained in:
Théo Monnom
2023-03-18 03:25:16 +01:00
committed by GitHub
parent 4443eae434
commit cad6d36201
123 changed files with 8534 additions and 4601 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+60 -53
View File
@@ -418,9 +418,9 @@ dependencies = [
[[package]]
name = "futures-channel"
version = "0.3.25"
version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed"
checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5"
dependencies = [
"futures-core",
"futures-sink",
@@ -428,9 +428,9 @@ dependencies = [
[[package]]
name = "futures-core"
version = "0.3.25"
version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac"
checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608"
[[package]]
name = "futures-executor"
@@ -462,9 +462,9 @@ dependencies = [
[[package]]
name = "futures-sink"
version = "0.3.25"
version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9"
checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364"
[[package]]
name = "futures-task"
@@ -779,7 +779,6 @@ checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4"
name = "livekit"
version = "0.1.1"
dependencies = [
"futures",
"futures-util",
"lazy_static",
"livekit-utils",
@@ -792,6 +791,7 @@ dependencies = [
"serde_json",
"thiserror",
"tokio",
"tokio-stream",
"tokio-tungstenite",
"tracing",
"url",
@@ -801,6 +801,7 @@ dependencies = [
name = "livekit-ffi"
version = "0.1.1"
dependencies = [
"futures-util",
"lazy_static",
"livekit",
"parking_lot",
@@ -815,6 +816,7 @@ dependencies = [
name = "livekit-utils"
version = "0.1.1"
dependencies = [
"futures-util",
"parking_lot",
"tokio",
]
@@ -825,10 +827,15 @@ version = "0.1.1"
dependencies = [
"cxx",
"env_logger 0.9.3",
"futures",
"js-sys",
"livekit-utils",
"log",
"thiserror",
"tokio",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
"webrtc-sys",
]
@@ -1354,17 +1361,6 @@ dependencies = [
"serde",
]
[[package]]
name = "sha-1"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c"
dependencies = [
"cfg-if",
"cpufeatures",
"digest",
]
[[package]]
name = "sha1"
version = "0.10.5"
@@ -1536,9 +1532,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
[[package]]
name = "tokio"
version = "1.23.0"
version = "1.26.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eab6d665857cc6ca78d6e80303a02cea7a7851e85dfbd77cbdc09bd129f1ef46"
checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64"
dependencies = [
"autocfg",
"bytes",
@@ -1551,7 +1547,7 @@ dependencies = [
"signal-hook-registry",
"socket2",
"tokio-macros",
"windows-sys 0.42.0",
"windows-sys 0.45.0",
]
[[package]]
@@ -1576,10 +1572,21 @@ dependencies = [
]
[[package]]
name = "tokio-tungstenite"
version = "0.17.2"
name = "tokio-stream"
version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f714dd15bead90401d77e04243611caec13726c2408afd5b31901dfcdcb3b181"
checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313"
dependencies = [
"futures-core",
"pin-project-lite",
"tokio",
]
[[package]]
name = "tokio-tungstenite"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54319c93411147bced34cb5609a80e0a8e44c5999c93903a81cd866630ec0bfd"
dependencies = [
"futures-util",
"log",
@@ -1649,9 +1656,9 @@ checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
[[package]]
name = "tungstenite"
version = "0.17.3"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e27992fd6a8c29ee7eef28fc78349aa244134e10ad447ce3b9f0ac0ed0fa4ce0"
checksum = "30ee6ab729cd4cf0fd55218530c4522ed30b7b6081752839b68fcec8d0960788"
dependencies = [
"base64 0.13.1",
"byteorder",
@@ -1661,7 +1668,7 @@ dependencies = [
"log",
"native-tls",
"rand",
"sha-1",
"sha1",
"thiserror",
"url",
"utf-8",
@@ -1899,12 +1906,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc 0.42.2",
"windows_i686_gnu 0.42.2",
"windows_i686_msvc 0.42.2",
"windows_x86_64_gnu 0.42.2",
"windows_aarch64_msvc 0.42.1",
"windows_i686_gnu 0.42.1",
"windows_i686_msvc 0.42.1",
"windows_x86_64_gnu 0.42.1",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc 0.42.2",
"windows_x86_64_msvc 0.42.1",
]
[[package]]
@@ -1918,24 +1925,24 @@ dependencies = [
[[package]]
name = "windows-targets"
version = "0.42.2"
version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"
checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc 0.42.2",
"windows_i686_gnu 0.42.2",
"windows_i686_msvc 0.42.2",
"windows_x86_64_gnu 0.42.2",
"windows_aarch64_msvc 0.42.1",
"windows_i686_gnu 0.42.1",
"windows_i686_msvc 0.42.1",
"windows_x86_64_gnu 0.42.1",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc 0.42.2",
"windows_x86_64_msvc 0.42.1",
]
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.42.2"
version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608"
[[package]]
name = "windows_aarch64_msvc"
@@ -1945,9 +1952,9 @@ checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"
[[package]]
name = "windows_aarch64_msvc"
version = "0.42.2"
version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7"
[[package]]
name = "windows_i686_gnu"
@@ -1957,9 +1964,9 @@ checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"
[[package]]
name = "windows_i686_gnu"
version = "0.42.2"
version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640"
[[package]]
name = "windows_i686_msvc"
@@ -1969,9 +1976,9 @@ checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"
[[package]]
name = "windows_i686_msvc"
version = "0.42.2"
version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605"
[[package]]
name = "windows_x86_64_gnu"
@@ -1981,15 +1988,15 @@ checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"
[[package]]
name = "windows_x86_64_gnu"
version = "0.42.2"
version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.42.2"
version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463"
[[package]]
name = "windows_x86_64_msvc"
@@ -1999,9 +2006,9 @@ checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
[[package]]
name = "windows_x86_64_msvc"
version = "0.42.2"
version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd"
[[package]]
name = "winreg"
examples/Cargo.lock
Generated
+566 -110
View File
File diff suppressed because it is too large Load Diff
examples/simple_room/Cargo.toml
+2
View File
@@ -16,3 +16,5 @@ parking_lot = "0.12.1"
egui = "0.20.1"
egui-wgpu = { version = "0.20.0", features = ["winit"] }
egui-winit = "0.20.1"
image = "0.24.5"
examples/simple_room/src/app.rs
+78 -49
View File
@@ -1,9 +1,16 @@
use crate::events::UiCmd;
use crate::logo_track::LogoTrack;
use crate::video_renderer::VideoRenderer;
use crate::{events::AsyncCmd, video_grid::VideoGrid};
use egui::{Rounding, Stroke};
use egui_wgpu::WgpuConfiguration;
use image::ImageFormat;
use livekit::options::{TrackPublishOptions, VideoCaptureOptions};
use livekit::prelude::*;
use livekit::webrtc::native::yuv_helper;
use livekit::webrtc::video_frame::native::I420BufferExt;
use livekit::webrtc::video_frame::{I420Buffer, VideoFrame, VideoRotation};
use livekit::webrtc::video_source::native::NativeVideoSource;
use livekit::SimulateScenario;
use parking_lot::Mutex;
use std::collections::HashMap;
@@ -11,6 +18,7 @@ use std::sync::{
atomic::{AtomicBool, Ordering},
Arc,
};
use std::time::Duration;
use tokio::sync::{mpsc, oneshot};
// Useful default constants for developing
@@ -25,16 +33,22 @@ use winit::{
window::{WindowBuilder, WindowId},
};
struct Session {
room: Room,
logo_track: LogoTrack,
close_tx: oneshot::Sender<()>,
handle: tokio::task::JoinHandle<()>,
}
struct AppState {
room: Mutex<Option<Room>>,
close_tx: Mutex<Option<oneshot::Sender<()>>>,
session: Mutex<Option<Session>>,
connecting: AtomicBool,
}
struct App {
state: Arc<AppState>,
video_renderers: HashMap<(ParticipantSid, TrackSid), VideoRenderer>,
egui_context: egui::Context,
egui_state: egui_winit::State,
egui_painter: egui_wgpu::winit::Painter,
@@ -69,8 +83,7 @@ pub fn run(rt: tokio::runtime::Runtime) {
let (ui_cmd_tx, ui_cmd_rx) = mpsc::unbounded_channel::<UiCmd>();
let state = Arc::new(AppState {
room: Default::default(),
close_tx: Default::default(),
session: Default::default(),
connecting: AtomicBool::new(false),
});
@@ -94,45 +107,53 @@ pub fn run(rt: tokio::runtime::Runtime) {
while let Some(event) = async_cmd_rx.recv().await {
match event {
AsyncCmd::RoomConnect { url, token } => {
if let Some(close_tx) = state.close_tx.lock().take() {
let _ = state.room.lock().take().unwrap().close().await;
let _ = close_tx.send(());
}
state.connecting.store(true, Ordering::SeqCst);
let res = Room::connect(&url, &token).await;
match res {
Ok((room, room_events)) => {
let (close_tx, close_rx) = oneshot::channel();
state.room.lock().replace(room);
state.close_tx.lock().replace(close_tx);
if let Ok((room, room_events)) = res {
let (close_tx, close_rx) = oneshot::channel();
let logo_track = LogoTrack::new(room.session());
let handle = tokio::spawn(room_task(
state.clone(),
room_events,
close_rx,
ui_cmd_tx.clone(),
));
tokio::spawn(room_task(
state.clone(),
room_events,
close_rx,
ui_cmd_tx.clone(),
));
*state.session.lock() = Some(Session {
room,
logo_track,
close_tx,
handle,
});
let _ = ui_cmd_tx.send(UiCmd::ConnectResult { result: Ok(()) });
}
Err(err) => {
let _ = ui_cmd_tx.send(UiCmd::ConnectResult { result: Err(err) });
}
let _ = ui_cmd_tx.send(UiCmd::ConnectResult { result: Ok(()) });
} else if let Err(err) = res {
let _ = ui_cmd_tx.send(UiCmd::ConnectResult { result: Err(err) });
}
state.connecting.store(false, Ordering::SeqCst);
}
AsyncCmd::RoomDisconnect => {
if let Some(close_tx) = state.close_tx.lock().take() {
let _ = state.room.lock().take().unwrap().close().await;
let _ = close_tx.send(());
if let Some(session) = state.session.lock().take() {
let _ = session.room.close().await;
let _ = session.close_tx.send(());
let _ = session.handle.await;
}
}
AsyncCmd::SimulateScenario { scenario } => {
if let Some(room) = state.room.lock().as_ref() {
let _ = room.session().simulate_scenario(scenario).await;
if let Some(session) = state.session.lock().as_ref() {
let _ = session.room.session().simulate_scenario(scenario).await;
}
}
AsyncCmd::ToggleLogo => {
if let Some(session) = state.session.lock().as_mut() {
let logo_track = &mut session.logo_track;
if !logo_track.is_published() {
logo_track.publish().await.unwrap();
} else {
logo_track.unpublish().await.unwrap();
}
}
}
}
@@ -140,7 +161,7 @@ pub fn run(rt: tokio::runtime::Runtime) {
});
tokio::task::block_in_place(move || loop {
// UI/Main Thread
// ui/main thread
event_loop.run(move |event, _, control_flow| {
app.update(event, control_flow);
});
@@ -183,7 +204,7 @@ impl App {
track, participant, ..
} => {
match track.clone() {
RemoteTrackHandle::Video(video_track) => {
RemoteTrack::Video(video_track) => {
// Create a new VideoRenderer
let video_renderer = VideoRenderer::new(
self.egui_painter.render_state().clone().unwrap(),
@@ -192,7 +213,7 @@ impl App {
self.video_renderers
.insert((participant.sid(), track.sid()), video_renderer);
}
RemoteTrackHandle::Audio(_) => {
RemoteTrack::Audio(_) => {
// The demo doesn't support Audio rendering at the moment.
}
};
@@ -300,6 +321,12 @@ impl App {
});
}
});
ui.menu_button("Publish", |ui| {
if ui.button("CustomTrack - LK Logo").clicked() {
let _ = self.cmd_tx.send(AsyncCmd::ToggleLogo);
}
});
});
});
@@ -321,9 +348,9 @@ impl App {
ui.horizontal(|ui| {
let connecting = self.state.connecting.load(Ordering::SeqCst);
let session = self.state.session.lock();
let room = self.state.room.lock();
ui.add_enabled_ui(!connecting && room.is_none(), |ui| {
ui.add_enabled_ui(!connecting && session.is_none(), |ui| {
if ui.button("Connect").clicked() {
self.connection_failure = None;
let _ = self.cmd_tx.send(AsyncCmd::RoomConnect {
@@ -337,7 +364,7 @@ impl App {
ui.spinner();
}
if room.is_some() {
if session.is_some() {
if ui.button("Disconnect").clicked() {
let _ = self.cmd_tx.send(AsyncCmd::RoomDisconnect);
}
@@ -352,17 +379,16 @@ impl App {
{
// Room Info
let room = self.state.room.lock();
if let Some(room) = room.as_ref() {
ui.label(format!("Name: {}", room.session().name()));
ui.label(format!("SID: {}", room.session().sid()));
if let Some(session) = self.state.session.lock().as_ref() {
ui.label(format!("Name: {}", session.room.session().name()));
ui.label(format!("SID: {}", session.room.session().sid()));
ui.label(format!(
"ConnectionState: {:?}",
room.session().connection_state()
session.room.session().connection_state()
));
ui.label(format!(
"ParticipantCount: {:?}",
room.session().participants().len() + 1
session.room.session().participants().len() + 1
));
}
}
@@ -408,12 +434,15 @@ impl App {
);
}
let name = self.state.room.lock().as_ref().and_then(|room| {
room.session()
.participants()
.get(participant_sid)
.map(|p| p.name())
});
let name =
self.state.session.lock().as_ref().and_then(|session| {
session
.room
.session()
.participants()
.get(participant_sid)
.map(|p| p.name())
});
if let Some(name) = name {
ui.painter().text(
examples/simple_room/src/events.rs
+1
View File
@@ -6,6 +6,7 @@ pub enum AsyncCmd {
RoomConnect { url: String, token: String },
RoomDisconnect,
SimulateScenario { scenario: SimulateScenario },
ToggleLogo, // Unpublish/Publish a logo track
}
#[derive(Debug)]
examples/simple_room/src/logo_track.rs
+206
View File
@@ -0,0 +1,206 @@
use image::ImageFormat;
use image::RgbaImage;
use livekit::options::{TrackPublishOptions, VideoCaptureOptions};
use livekit::prelude::*;
use livekit::webrtc::{
native::yuv_helper,
video_frame::native::I420BufferExt,
video_frame::{I420Buffer, VideoFrame, VideoRotation},
video_source::native::NativeVideoSource,
};
use parking_lot::Mutex;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::oneshot;
use tokio::task::JoinHandle;
// The logo must not be bigger than the framebuffer
const PIXEL_SIZE: usize = 4;
const FRAME_RATE: u64 = 30;
const MOVE_SPEED: i32 = 16;
const FB_WIDTH: usize = 1280;
const FB_HEIGHT: usize = 720;
#[derive(Clone)]
struct FrameData {
image: Arc<RgbaImage>,
framebuffer: Arc<Mutex<Vec<u8>>>,
video_frame: Arc<Mutex<VideoFrame<I420Buffer>>>,
pos: (u32, u32),
direction: (i32, i32),
}
struct TrackHandle {
close_tx: oneshot::Sender<()>,
track: LocalVideoTrack,
task: JoinHandle<()>,
}
pub struct LogoTrack {
rtc_source: NativeVideoSource,
session: RoomSession,
handle: Option<TrackHandle>,
}
impl LogoTrack {
pub fn new(session: RoomSession) -> Self {
Self {
rtc_source: NativeVideoSource::default(),
session,
handle: None,
}
}
pub fn is_published(&self) -> bool {
self.handle.is_some()
}
pub async fn publish(&mut self) -> Result<(), RoomError> {
self.unpublish().await;
let (close_tx, close_rx) = oneshot::channel();
let track = LocalVideoTrack::create_video_track(
"livekit_logo",
VideoCaptureOptions::default(),
self.rtc_source.clone(),
);
let task = tokio::spawn(Self::track_task(close_rx, self.rtc_source.clone()));
self.session
.local_participant()
.publish_track(
LocalTrack::Video(track.clone()),
TrackPublishOptions {
source: TrackSource::Camera,
..Default::default()
},
)
.await?;
let handle = TrackHandle {
close_tx,
task,
track,
};
self.handle = Some(handle);
Ok(())
}
pub async fn unpublish(&mut self) -> Result<(), RoomError> {
if let Some(handle) = self.handle.take() {
let _ = handle.close_tx.send(());
let _ = handle.task.await;
self.session
.local_participant()
.unpublish_track(handle.track.sid(), true)
.await?;
}
Ok(())
}
async fn track_task(mut close_rx: oneshot::Receiver<()>, rtc_source: NativeVideoSource) {
let mut interval = tokio::time::interval(Duration::from_millis(1000 / FRAME_RATE));
let image = tokio::task::spawn_blocking(|| {
image::load_from_memory_with_format(include_bytes!("moving-logo.png"), ImageFormat::Png)
.unwrap()
.to_rgba8()
})
.await
.unwrap();
let mut data = FrameData {
image: Arc::new(image),
framebuffer: Arc::new(Mutex::new(vec![0u8; (FB_WIDTH * FB_HEIGHT * 4) as usize])),
video_frame: Arc::new(Mutex::new(VideoFrame {
rotation: VideoRotation::VideoRotation0,
buffer: I420Buffer::new(FB_WIDTH as u32, FB_HEIGHT as u32),
timestamp: 0,
})),
pos: (0, 0),
direction: (1, 1),
};
loop {
tokio::select! {
_ = &mut close_rx => {
break;
}
_ = interval.tick() => {}
}
data.pos.0 = (data.pos.0 as i32 + data.direction.0 * MOVE_SPEED) as u32;
data.pos.1 = (data.pos.1 as i32 + data.direction.1 * MOVE_SPEED) as u32;
if data.pos.0 >= (FB_WIDTH - data.image.width() as usize) as u32 {
data.direction.0 = -1;
} else if data.pos.0 <= 0 {
data.direction.0 = 1;
}
if data.pos.1 >= (FB_HEIGHT - data.image.height() as usize) as u32 {
data.direction.1 = -1;
} else if data.pos.1 <= 0 {
data.direction.1 = 1;
}
tokio::task::spawn_blocking({
let data = data.clone();
let source = rtc_source.clone();
move || {
let image = data.image.as_raw();
let mut framebuffer = data.framebuffer.lock();
let mut video_frame = data.video_frame.lock();
let i420_buffer = &mut video_frame.buffer;
let stride_y = i420_buffer.stride_y();
let stride_u = i420_buffer.stride_u();
let stride_v = i420_buffer.stride_v();
let (data_y, data_u, data_v) = i420_buffer.data_mut();
framebuffer.fill(0);
for i in 0..data.image.height() as usize {
let x = data.pos.0 as usize;
let y = data.pos.1 as usize;
let frame_width = data.image.width() as usize;
let logo_stride = frame_width * PIXEL_SIZE;
let row_start = (x + ((i + y) * FB_WIDTH)) * PIXEL_SIZE;
let row_end = row_start + logo_stride;
framebuffer[row_start..row_end].copy_from_slice(
&image[i * logo_stride..i * logo_stride + logo_stride],
);
}
yuv_helper::abgr_to_i420(
&framebuffer,
(FB_WIDTH * PIXEL_SIZE) as i32,
data_y,
stride_y,
data_u,
stride_u,
data_v,
stride_v,
FB_WIDTH as i32,
FB_HEIGHT as i32,
)
.unwrap();
source.capture_frame(&*video_frame);
}
})
.await;
}
}
}
impl Drop for LogoTrack {
fn drop(&mut self) {
if let Some(handle) = self.handle.take() {
let _ = handle.close_tx.send(());
}
}
}
examples/simple_room/src/main.rs
+1
View File
@@ -1,5 +1,6 @@
mod app;
mod events;
mod logo_track;
mod video_grid;
mod video_renderer;
examples/simple_room/src/moving-logo.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 7.0 KiB

examples/simple_room/src/video_renderer.rs
+62 -58
View File
@@ -1,18 +1,17 @@
use livekit::prelude::*;
use livekit::webrtc::video_frame_buffer::*;
use livekit::webrtc::yuv_helper;
use std::convert::TryInto;
use futures::StreamExt;
use livekit::webrtc::native::yuv_helper;
use livekit::webrtc::prelude::*;
use livekit::webrtc::video_stream::native::NativeVideoStream;
use std::num::NonZeroU32;
use std::{
ops::DerefMut,
sync::{Arc, Mutex},
};
use tracing::debug_span;
use tracing::{error, warn};
pub struct VideoRenderer {
internal: Arc<Mutex<RendererInternal>>,
rtc_track: Arc<VideoTrack>,
rtc_track: RtcVideoTrack,
}
struct RendererInternal {
@@ -86,7 +85,7 @@ impl RendererInternal {
}
impl VideoRenderer {
pub fn new(render_state: egui_wgpu::RenderState, rtc_track: Arc<VideoTrack>) -> Self {
pub fn new(render_state: egui_wgpu::RenderState, rtc_track: RtcVideoTrack) -> Self {
let internal = Arc::new(Mutex::new(RendererInternal {
render_state,
width: 0,
@@ -97,62 +96,73 @@ impl VideoRenderer {
egui_texture: None,
}));
rtc_track.on_frame({
let mut video_sink = NativeVideoStream::new(rtc_track.clone());
tokio::spawn({
let internal = internal.clone();
async move {
while let Some(frame) = video_sink.next().await {
let internal = internal.clone();
// Process the frame
let _ = tokio::task::spawn_blocking(move || {
let span = debug_span!("texture_upload");
let _enter = span.enter();
Box::new(move |_frame, buffer| {
let span = debug_span!("texture_upload");
let _enter = span.enter();
let mut internal = internal.lock().unwrap();
let buffer = frame.buffer.to_i420();
let mut internal = internal.lock().unwrap();
let buffer = buffer.to_i420();
let width: u32 = buffer.width().try_into().unwrap();
let height: u32 = buffer.height().try_into().unwrap();
let width: u32 = buffer.width().try_into().unwrap();
let height: u32 = buffer.height().try_into().unwrap();
internal.ensure_texture_size(width, height);
internal.ensure_texture_size(width, height);
let rgba_ptr = internal.rgba_data.deref_mut();
let rgba_stride = buffer.width() * 4;
let rgba_ptr = internal.rgba_data.deref_mut();
let rgba_stride = buffer.width() * 4;
let (data_y, data_u, data_v) = buffer.data();
yuv_helper::i420_to_abgr(
buffer.data_y(),
buffer.stride_y(),
buffer.data_u(),
buffer.stride_u(),
buffer.data_v(),
buffer.stride_v(),
rgba_ptr,
rgba_stride,
buffer.width(),
buffer.height(),
);
yuv_helper::i420_to_abgr(
data_y,
buffer.stride_y(),
data_u,
buffer.stride_u(),
data_v,
buffer.stride_v(),
rgba_ptr,
rgba_stride,
buffer.width(),
buffer.height(),
)
.unwrap();
let copy_desc = wgpu::ImageCopyTexture {
texture: internal.texture.as_ref().unwrap(),
mip_level: 0,
origin: wgpu::Origin3d::default(),
aspect: wgpu::TextureAspect::default(),
};
let copy_desc = wgpu::ImageCopyTexture {
texture: internal.texture.as_ref().unwrap(),
mip_level: 0,
origin: wgpu::Origin3d::default(),
aspect: wgpu::TextureAspect::default(),
};
let copy_layout = wgpu::ImageDataLayout {
bytes_per_row: Some(NonZeroU32::new(width * 4).unwrap()),
..Default::default()
};
let copy_layout = wgpu::ImageDataLayout {
bytes_per_row: Some(NonZeroU32::new(width * 4).unwrap()),
..Default::default()
};
let copy_size = wgpu::Extent3d {
width,
height,
..Default::default()
};
let copy_size = wgpu::Extent3d {
width,
height,
..Default::default()
};
internal.render_state.queue.write_texture(
copy_desc,
&internal.rgba_data,
copy_layout,
copy_size,
);
})
internal.render_state.queue.write_texture(
copy_desc,
&internal.rgba_data,
copy_layout,
copy_size,
);
})
.await;
}
}
});
Self {
@@ -165,9 +175,3 @@ impl VideoRenderer {
self.internal.lock().unwrap().egui_texture.clone()
}
}
impl Drop for VideoRenderer {
fn drop(&mut self) {
self.rtc_track.on_frame(Box::new(|_, _| {}));
}
}
livekit-ffi/Cargo.toml
+1 -1
View File
@@ -14,10 +14,10 @@ prost = "0.11.0"
prost-types = "0.11.1"
lazy_static = "1.4.0"
thiserror = "1.0.38"
futures-util = "0.3.23"
[build-dependencies]
prost-build = { version = "0.11.1" }
[lib]
crate-type = ["cdylib", "staticlib"]
livekit-ffi/protocol/ffi.proto
+9 -13
View File
@@ -279,19 +279,12 @@ message TrackEvent {
message FrameReceived {
VideoFrameInfo frame = 1;
VideoFrameBufferInfo frame_buffer = 2;
VideoFrameBufferInfo buffer = 2;
}
message VideoFrameInfo {
int32 width = 1;
int32 height = 2;
uint32 size = 3;
uint32 id = 4; // uint16
int64 timestamp_us = 5;
int64 ntp_time_ms = 6;
uint32 transport_frame_id = 7;
uint32 timestamp = 8;
VideoRotation rotation = 9;
int64 timestamp = 1;
VideoRotation rotation = 2;
}
message VideoFrameBufferInfo {
@@ -312,11 +305,13 @@ message PlanarYuvBufferInfo {
int32 stride_y = 3;
int32 stride_u = 4;
int32 stride_v = 5;
int32 stride_a = 6;
// *const u8 or *const u16
uint64 data_y_ptr = 6;
uint64 data_u_ptr = 7;
uint64 data_v_ptr = 8;
uint64 data_y_ptr = 7;
uint64 data_u_ptr = 8;
uint64 data_v_ptr = 9;
uint64 data_a_ptr = 10; // nullptr = no alpha
}
message BiplanarYuvBufferInfo {
@@ -341,6 +336,7 @@ enum VideoFrameBufferType {
I444 = 4;
I010 = 5;
NV12 = 6;
WEBGL = 7;
}
/// Participant Events
livekit-ffi/src/server/conversion.rs → livekit-ffi/src/server/conversion/mod.rs
+13 -145
View File
@@ -1,12 +1,12 @@
use crate::{proto, server::FFIHandleId};
use livekit::{
prelude::*,
webrtc::video_frame_buffer::{
BiplanarYuv8Buffer, BiplanarYuvBuffer, I010Buffer, I420ABuffer, I420Buffer, I422Buffer,
I444Buffer, NV12Buffer, PlanarYuv16BBuffer, PlanarYuv8Buffer, PlanarYuvBuffer,
},
};
use std::sync::Arc;
use livekit::prelude::*;
use livekit::webrtc::prelude::*;
use std::any::Any;
pub mod participant;
pub mod publication;
pub mod room;
pub mod video_frame;
impl From<FFIHandleId> for proto::FfiHandleId {
fn from(id: FFIHandleId) -> Self {
@@ -30,8 +30,8 @@ macro_rules! impl_participant_into {
};
}
impl_participant_into!(&Arc<LocalParticipant>);
impl_participant_into!(&Arc<RemoteParticipant>);
impl_participant_into!(&LocalParticipant);
impl_participant_into!(&RemoteParticipant);
impl_participant_into!(&Participant);
impl From<TrackSource> for proto::TrackSource {
@@ -92,14 +92,13 @@ impl_track_into!(&LocalAudioTrack);
impl_track_into!(&LocalVideoTrack);
impl_track_into!(&RemoteAudioTrack);
impl_track_into!(&RemoteVideoTrack);
impl_track_into!(&TrackHandle);
impl_track_into!(&LocalTrackHandle);
impl_track_into!(&RemoteTrackHandle);
impl_track_into!(&Track);
impl_track_into!(&LocalTrack);
impl_track_into!(&RemoteTrack);
impl From<TrackKind> for proto::TrackKind {
fn from(kind: TrackKind) -> Self {
match kind {
TrackKind::Unknown => proto::TrackKind::KindUnknown,
TrackKind::Audio => proto::TrackKind::KindAudio,
TrackKind::Video => proto::TrackKind::KindVideo,
}
@@ -109,7 +108,6 @@ impl From<TrackKind> for proto::TrackKind {
impl From<StreamState> for proto::StreamState {
fn from(state: StreamState) -> Self {
match state {
StreamState::Unknown => Self::StateUnknown,
StreamState::Active => Self::StateActive,
StreamState::Paused => Self::StatePaused,
}
@@ -182,136 +180,6 @@ impl proto::RoomEvent {
}
}
impl From<VideoRotation> for proto::VideoRotation {
fn from(rotation: VideoRotation) -> proto::VideoRotation {
match rotation {
VideoRotation::VideoRotation0 => Self::VideoRotation0,
VideoRotation::VideoRotation90 => Self::VideoRotation90,
VideoRotation::VideoRotation180 => Self::VideoRotation180,
VideoRotation::VideoRotation270 => Self::VideoRotation270,
}
}
}
impl From<VideoFrame> for proto::VideoFrameInfo {
fn from(frame: VideoFrame) -> Self {
Self {
width: frame.width(),
height: frame.height(),
size: frame.size(),
id: frame.id() as u32,
timestamp_us: frame.timestamp_us(),
ntp_time_ms: frame.ntp_time_ms(),
transport_frame_id: frame.transport_frame_id(),
timestamp: frame.timestamp(),
rotation: proto::VideoRotation::from(frame.rotation()).into(),
}
}
}
impl From<VideoFrameBufferType> for proto::VideoFrameBufferType {
fn from(buffer_type: VideoFrameBufferType) -> Self {
match buffer_type {
VideoFrameBufferType::Native => Self::Native,
VideoFrameBufferType::I420 => Self::I420,
VideoFrameBufferType::I420A => Self::I420a,
VideoFrameBufferType::I422 => Self::I422,
VideoFrameBufferType::I444 => Self::I444,
VideoFrameBufferType::I010 => Self::I010,
VideoFrameBufferType::NV12 => Self::Nv12,
}
}
}
macro_rules! impl_yuv_into {
($b:ty) => {
impl From<$b> for proto::PlanarYuvBufferInfo {
fn from(buffer: $b) -> Self {
Self {
chroma_width: buffer.chroma_width(),
chroma_height: buffer.chroma_height(),
stride_y: buffer.stride_y(),
stride_u: buffer.stride_u(),
stride_v: buffer.stride_v(),
data_y_ptr: buffer.data_y().as_ptr() as u64,
data_u_ptr: buffer.data_u().as_ptr() as u64,
data_v_ptr: buffer.data_v().as_ptr() as u64,
}
}
}
};
}
impl_yuv_into!(&I420Buffer);
impl_yuv_into!(&I420ABuffer);
impl_yuv_into!(&I422Buffer);
impl_yuv_into!(&I444Buffer);
impl_yuv_into!(&I010Buffer);
macro_rules! impl_biyuv_into {
($b:ty) => {
impl From<$b> for proto::BiplanarYuvBufferInfo {
fn from(buffer: $b) -> Self {
Self {
chroma_width: buffer.chroma_width(),
chroma_height: buffer.chroma_height(),
stride_y: buffer.stride_y(),
stride_uv: buffer.stride_uv(),
data_y_ptr: buffer.data_y().as_ptr() as u64,
data_uv_ptr: buffer.data_uv().as_ptr() as u64,
}
}
}
};
}
impl_biyuv_into!(&NV12Buffer);
impl proto::VideoFrameBufferInfo {
pub fn from(handle_id: FFIHandleId, buffer: &VideoFrameBuffer) -> Self {
Self {
handle: Some(handle_id.into()),
buffer_type: proto::VideoFrameBufferType::from(buffer.buffer_type()).into(),
width: buffer.width(),
height: buffer.height(),
buffer: Some(match &buffer {
VideoFrameBuffer::Native(_) => {
proto::video_frame_buffer_info::Buffer::Native(proto::NativeBufferInfo {})
}
VideoFrameBuffer::I420(i420) => {
proto::video_frame_buffer_info::Buffer::Yuv(i420.into())
}
VideoFrameBuffer::I420A(i420a) => {
proto::video_frame_buffer_info::Buffer::Yuv(i420a.into())
}
VideoFrameBuffer::I422(i422) => {
proto::video_frame_buffer_info::Buffer::Yuv(i422.into())
}
VideoFrameBuffer::I444(i444) => {
proto::video_frame_buffer_info::Buffer::Yuv(i444.into())
}
VideoFrameBuffer::I010(i010) => {
proto::video_frame_buffer_info::Buffer::Yuv(i010.into())
}
VideoFrameBuffer::NV12(nv12) => {
proto::video_frame_buffer_info::Buffer::BiYuv(nv12.into())
}
}),
}
}
}
impl From<proto::VideoFormatType> for VideoFormatType {
fn from(format: proto::VideoFormatType) -> Self {
match format {
proto::VideoFormatType::FormatArgb => Self::ARGB,
proto::VideoFormatType::FormatBgra => Self::BGRA,
proto::VideoFormatType::FormatAbgr => Self::ABGR,
proto::VideoFormatType::FormatRgba => Self::RGBA,
}
}
}
impl From<&RoomSession> for proto::RoomInfo {
fn from(session: &RoomSession) -> Self {
Self {
livekit-ffi/src/server/conversion/participant.rs
View File
livekit-ffi/src/server/conversion/publication.rs
View File
livekit-ffi/src/server/conversion/room.rs
View File
livekit-ffi/src/server/conversion/video_frame.rs
+203
View File
@@ -0,0 +1,203 @@
use crate::{proto, server::FFIHandleId};
use livekit::webrtc::prelude::*;
use livekit::webrtc::video_frame;
use std::any::Any;
macro_rules! impl_yuv_into {
(@fields, $buffer:ident, $data_y:ident, $data_u:ident, $data_v: ident) => {
Self {
chroma_width: $buffer.chroma_width(),
chroma_height: $buffer.chroma_height(),
stride_y: $buffer.stride_y(),
stride_u: $buffer.stride_u(),
stride_v: $buffer.stride_v(),
data_y_ptr: $data_y.as_ptr() as u64,
data_u_ptr: $data_u.as_ptr() as u64,
data_v_ptr: $data_v.as_ptr() as u64,
..Default::default()
}
};
($buffer:ty, ALPHA) => {
impl From<$buffer> for proto::PlanarYuvBufferInfo {
fn from(buffer: $buffer) -> Self {
let (data_y, data_u, data_v, data_a) = buffer.data();
let mut proto = impl_yuv_into!(@fields, buffer, data_y, data_u, data_v);
proto.stride_a = buffer.stride_a();
proto.data_a_ptr = data_a.map(|data_a| data_a.as_ptr() as u64).unwrap_or(0);
proto
}
}
};
($buffer:ty) => {
impl From<$buffer> for proto::PlanarYuvBufferInfo {
fn from(buffer: $buffer) -> Self {
let (data_y, data_u, data_v) = buffer.data();
impl_yuv_into!(@fields, buffer, data_y, data_u, data_v)
}
}
};
}
macro_rules! impl_biyuv_into {
($b:ty) => {
impl From<$b> for proto::BiplanarYuvBufferInfo {
fn from(buffer: $b) -> Self {
let (data_y, data_uv) = buffer.data();
Self {
chroma_width: buffer.chroma_width(),
chroma_height: buffer.chroma_height(),
stride_y: buffer.stride_y(),
stride_uv: buffer.stride_uv(),
data_y_ptr: data_y.as_ptr() as u64,
data_uv_ptr: data_uv.as_ptr() as u64,
}
}
}
};
}
impl_yuv_into!(&I420Buffer);
impl_yuv_into!(&I420ABuffer, ALPHA);
impl_yuv_into!(&I422Buffer);
impl_yuv_into!(&I444Buffer);
impl_yuv_into!(&I010Buffer);
impl_biyuv_into!(&NV12Buffer);
impl proto::VideoFrameInfo {
pub fn from<T>(frame: &VideoFrame<T>) -> Self
where
T: VideoFrameBuffer,
{
Self {
timestamp: frame.timestamp,
rotation: proto::VideoRotation::from(frame.rotation).into(),
}
}
}
impl proto::VideoFrameBufferInfo {
pub fn from(handle: FFIHandleId, buffer: &dyn VideoFrameBuffer) -> Self {
match &buffer.buffer_type() {
#[cfg(not(target_arch = "wasm32"))]
VideoFrameBufferType::Native => Self::from_native(handle, buffer.as_native().unwrap()),
VideoFrameBufferType::I420 => Self::from_i420(handle, buffer.as_i420().unwrap()),
VideoFrameBufferType::I420A => Self::from_i420a(handle, buffer.as_i420a().unwrap()),
VideoFrameBufferType::I422 => Self::from_i422(handle, buffer.as_i422().unwrap()),
VideoFrameBufferType::I444 => Self::from_i444(handle, buffer.as_i444().unwrap()),
VideoFrameBufferType::I010 => Self::from_i010(handle, buffer.as_i010().unwrap()),
VideoFrameBufferType::NV12 => Self::from_nv12(handle, buffer.as_nv12().unwrap()),
_ => panic!("unsupported buffer type on this platform"),
}
}
#[cfg(not(target_arch = "wasm32"))]
pub fn from_native(handle_id: FFIHandleId, buffer: &video_frame::native::NativeBuffer) -> Self {
Self {
handle: Some(handle_id.into()),
buffer_type: proto::VideoFrameBufferType::Native.into(),
width: buffer.width(),
height: buffer.height(),
buffer: Some(proto::video_frame_buffer_info::Buffer::Native(
proto::NativeBufferInfo {},
)),
}
}
pub fn from_i420(handle_id: FFIHandleId, buffer: &I420Buffer) -> Self {
Self {
handle: Some(handle_id.into()),
buffer_type: proto::VideoFrameBufferType::I420.into(),
width: buffer.width(),
height: buffer.height(),
buffer: Some(proto::video_frame_buffer_info::Buffer::Yuv(buffer.into())),
}
}
pub fn from_i420a(handle_id: FFIHandleId, buffer: &I420ABuffer) -> Self {
Self {
handle: Some(handle_id.into()),
buffer_type: proto::VideoFrameBufferType::I420a.into(),
width: buffer.width(),
height: buffer.height(),
buffer: Some(proto::video_frame_buffer_info::Buffer::Yuv(buffer.into())),
}
}
pub fn from_i422(handle_id: FFIHandleId, buffer: &I422Buffer) -> Self {
Self {
handle: Some(handle_id.into()),
buffer_type: proto::VideoFrameBufferType::I422.into(),
width: buffer.width(),
height: buffer.height(),
buffer: Some(proto::video_frame_buffer_info::Buffer::Yuv(buffer.into())),
}
}
pub fn from_i444(handle_id: FFIHandleId, buffer: &I444Buffer) -> Self {
Self {
handle: Some(handle_id.into()),
buffer_type: proto::VideoFrameBufferType::I444.into(),
width: buffer.width(),
height: buffer.height(),
buffer: Some(proto::video_frame_buffer_info::Buffer::Yuv(buffer.into())),
}
}
pub fn from_i010(handle_id: FFIHandleId, buffer: &I010Buffer) -> Self {
Self {
handle: Some(handle_id.into()),
buffer_type: proto::VideoFrameBufferType::I010.into(),
width: buffer.width(),
height: buffer.height(),
buffer: Some(proto::video_frame_buffer_info::Buffer::Yuv(buffer.into())),
}
}
pub fn from_nv12(handle_id: FFIHandleId, buffer: &NV12Buffer) -> Self {
Self {
handle: Some(handle_id.into()),
buffer_type: proto::VideoFrameBufferType::Nv12.into(),
width: buffer.width(),
height: buffer.height(),
buffer: Some(proto::video_frame_buffer_info::Buffer::BiYuv(buffer.into())),
}
}
}
impl From<proto::VideoFormatType> for VideoFormatType {
fn from(format: proto::VideoFormatType) -> Self {
match format {
proto::VideoFormatType::FormatArgb => Self::ARGB,
proto::VideoFormatType::FormatBgra => Self::BGRA,
proto::VideoFormatType::FormatAbgr => Self::ABGR,
proto::VideoFormatType::FormatRgba => Self::RGBA,
}
}
}
impl From<VideoRotation> for proto::VideoRotation {
fn from(rotation: VideoRotation) -> proto::VideoRotation {
match rotation {
VideoRotation::VideoRotation0 => Self::VideoRotation0,
VideoRotation::VideoRotation90 => Self::VideoRotation90,
VideoRotation::VideoRotation180 => Self::VideoRotation180,
VideoRotation::VideoRotation270 => Self::VideoRotation270,
}
}
}
impl From<VideoFrameBufferType> for proto::VideoFrameBufferType {
fn from(buffer_type: VideoFrameBufferType) -> Self {
match buffer_type {
VideoFrameBufferType::Native => Self::Native,
VideoFrameBufferType::I420 => Self::I420,
VideoFrameBufferType::I420A => Self::I420a,
VideoFrameBufferType::I422 => Self::I422,
VideoFrameBufferType::I444 => Self::I444,
VideoFrameBufferType::I010 => Self::I010,
VideoFrameBufferType::NV12 => Self::Nv12,
VideoFrameBufferType::WebGl => Self::Webgl,
_ => panic!("unsupported buffer type on FFI server"),
}
}
}
livekit-ffi/src/server/mod.rs
+6 -5
View File
@@ -1,6 +1,7 @@
use crate::proto;
use lazy_static::lazy_static;
use livekit::prelude::*;
use livekit::webrtc::video_frame::{native::VideoFrameBufferExt, BoxVideoFrame, VideoFrameBuffer};
use parking_lot::{Mutex, RwLock};
use prost::Message;
use std::any::Any;
@@ -166,11 +167,11 @@ impl FFIServer {
let buffer = self.release_handle(to_i420.buffer.unwrap().id as FFIHandleId);
if let Some(buffer) = buffer {
if let Ok(buffer) = buffer.downcast::<VideoFrameBuffer>() {
if let Ok(buffer) = buffer.downcast::<Box<dyn VideoFrameBuffer>>() {
let handle_id = self.next_handle_id();
let i420 = VideoFrameBuffer::I420(buffer.to_i420());
buffer_info = Some(proto::VideoFrameBufferInfo::from(handle_id, &i420));
self.insert_handle(handle_id, Box::new(i420));
let buffer = buffer.to_i420();
buffer_info = Some(proto::VideoFrameBufferInfo::from(handle_id, &buffer));
self.insert_handle(handle_id, Box::new(buffer));
}
}
@@ -188,7 +189,7 @@ impl FFIServer {
let buffer = ffi_owned.get(&(to_argb.buffer.unwrap().id as FFIHandleId));
if let Some(buffer) = buffer {
if let Some(buffer) = buffer.downcast_ref::<VideoFrameBuffer>() {
if let Some(buffer) = buffer.downcast_ref::<Box<dyn VideoFrameBuffer>>() {
let dst_buf = unsafe {
slice::from_raw_parts_mut(
to_argb.dst_ptr as *mut u8,
livekit-ffi/src/server/room.rs
+19 -16
View File
@@ -1,6 +1,8 @@
use crate::proto::{self};
use crate::proto;
use crate::server::FFIServer;
use futures_util::stream::StreamExt;
use livekit::prelude::*;
use livekit::webrtc::video_stream::native::NativeVideoStream;
use tokio::sync::{mpsc, oneshot};
pub async fn create_room(
@@ -67,9 +69,9 @@ async fn room_task(
publication: _,
participant: _,
} => {
if let RemoteTrackHandle::Video(video_track) = track {
let rtc_track = video_track.rtc_track();
rtc_track.on_frame(on_video_frame(server, video_track.sid()));
if let RemoteTrack::Video(video_track) = track {
let video_stream = NativeVideoStream::new(video_track.rtc_track());
tokio::spawn(video_frame_task(server, video_track.sid(), video_stream));
}
}
_ => {}
@@ -87,19 +89,20 @@ async fn room_task(
async fn participant_task(participant: Participant) {
let mut participant_events = participant.register_observer();
while let Some(event) = participant_events.recv().await {
// TODO convert event to proto
// TODO(theomonnom): convert event to proto
}
}
fn on_video_frame(server: &'static FFIServer, track_sid: TrackSid) -> OnFrameHandler {
// TODO(theomonnom): Should I use VideoSinkInfo here? (It'll help to have a more verbose
// lifetime)
Box::new(move |frame, buffer| {
// Frame received, create a new FFIHandle from the video buffer.
async fn video_frame_task(
server: &'static FFIServer,
track_sid: TrackSid,
mut stream: NativeVideoStream,
) {
while let Some(frame) = stream.next().await {
let handle_id = server.next_handle_id();
let proto_buffer = proto::VideoFrameBufferInfo::from(handle_id, &buffer);
server.insert_handle(handle_id, Box::new(buffer));
let frame_info = proto::VideoFrameInfo::from(&frame);
let buffer_info = proto::VideoFrameBufferInfo::from(handle_id, &frame.buffer);
server.insert_handle(handle_id, Box::new(frame.buffer));
// Send the received frame to the FFI language.
let _ = server.send_event(
@@ -107,12 +110,12 @@ fn on_video_frame(server: &'static FFIServer, track_sid: TrackSid) -> OnFrameHan
track_sid: track_sid.to_string(),
message: Some(proto::track_event::Message::FrameReceived(
proto::FrameReceived {
frame: Some(frame.into()),
frame_buffer: Some(proto_buffer),
frame: Some(frame_info),
buffer: Some(buffer_info),
},
)),
}),
None,
);
})
}
}
livekit-utils/Cargo.toml
+3 -4
View File
@@ -6,8 +6,7 @@ license = "Apache-2.0"
description = "Shared utilities for livekit"
repository = "https://github.com/livekit/client-sdk-rust"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
parking_lot = "0.12.1"
tokio = { version = "1", features = ["full"] }
tokio = { version = "1.26.0", features = ["full"] }
futures-util = "0.3"
parking_lot = "0.12"
livekit-utils/src/enum_dispatch.rs
+11 -11
View File
@@ -1,11 +1,9 @@
// TODO(theomonnom): Match the complete function signature like:
// - pub(crate) fn update_info(&self, info: ParticipantInfo) -> ();
// TODO(theomonnom): Async methods
#[macro_export]
macro_rules! enum_dispatch {
// This arm is used to avoid nested loops with the arguments
// The arguments are transformed to $combined_args TokenTree
(@match $self:ident $fnc:ident $combined_args:tt [$($variant:ident),+]) => {
// The arguments are transformed to $combined_args tt
(@match [$($variant:ident),+]: $fnc:ident, $self:ident, $combined_args:tt) => {
match $self {
$(
Self::$variant(inner) => inner.$fnc$combined_args,
@@ -13,15 +11,17 @@ macro_rules! enum_dispatch {
}
};
($vis:vis$(,)? $fnc:ident, $self:ty, [$($arg:ident: $t:ty),*], $ret:ty, [$($variant:ident),+]) => {
$vis fn $fnc(self: $self, $($arg: $t),*) -> $ret {
enum_dispatch!(@match self $fnc ($($arg,)*) [$($variant),+])
// Create the function and extract self fron the $args tt (little hack)
(@fnc [$($variant:ident),+]: $vis:vis fn $fnc:ident($self:ident: $sty:ty $(, $arg:ident: $t:ty)*) -> $ret:ty) => {
#[inline]
$vis fn $fnc($self: $sty, $($arg: $t),*) -> $ret {
enum_dispatch!(@match [$($variant),+]: $fnc, $self, ($($arg,)*))
}
};
($variants:tt $(fnc!($vis:vis$(,)? $fnc:ident, $self:ty, $args:tt, $ret:ty);)+) => {
($variants:tt; $($vis:vis fn $fnc:ident$args:tt -> $ret:ty;)+) => {
$(
enum_dispatch!($vis, $fnc, $self, $args, $ret, $variants);
)*
enum_dispatch!(@fnc $variants: $vis fn $fnc$args -> $ret);
)+
};
}
livekit-utils/src/observer.rs
+39 -8
View File
@@ -1,14 +1,16 @@
// Really basic implementation of the observer pattern using mpsc channels.
// Currently unbounded channels
use futures_util::sink::Sink;
use futures_util::task::{Context, Poll};
use parking_lot::Mutex;
use std::pin::Pin;
use std::sync::Arc;
use tokio::sync::mpsc;
#[derive(Debug)]
#[derive(Clone, Debug)]
pub struct Dispatcher<T>
where
T: Clone,
{
senders: Vec<mpsc::UnboundedSender<T>>,
senders: Arc<Mutex<Vec<mpsc::UnboundedSender<T>>>>,
}
impl<T> Default for Dispatcher<T>
@@ -26,14 +28,43 @@ impl<T> Dispatcher<T>
where
T: Clone,
{
pub fn register(&mut self) -> mpsc::UnboundedReceiver<T> {
pub fn register(&self) -> mpsc::UnboundedReceiver<T> {
let (tx, rx) = mpsc::unbounded_channel();
self.senders.push(tx);
self.senders.lock().push(tx);
rx
}
pub fn dispatch(&mut self, msg: &T) {
pub fn dispatch(&self, msg: &T) {
self.senders
.lock()
.retain(|sender| sender.send(msg.clone()).is_ok());
}
pub fn clear(&self) {
self.senders.lock().clear();
}
}
impl<T> Sink<T> for Dispatcher<T>
where
T: Clone,
{
type Error = ();
fn poll_ready(self: Pin<&mut Self>, _: &mut Context) -> Poll<Result<(), Self::Error>> {
Poll::Ready(Ok(()))
}
fn start_send(self: Pin<&mut Self>, item: T) -> Result<(), Self::Error> {
self.dispatch(&item);
Ok(())
}
fn poll_flush(self: Pin<&mut Self>, _: &mut Context) -> Poll<Result<(), Self::Error>> {
Poll::Ready(Ok(()))
}
fn poll_close(self: Pin<&mut Self>, _: &mut Context) -> Poll<Result<(), Self::Error>> {
Poll::Ready(Ok(()))
}
}
livekit-webrtc/Cargo.toml
+26 -3
View File
@@ -8,12 +8,35 @@ description = "Livekit safe bindings to libwebrtc"
repository = "https://github.com/livekit/client-sdk-rust"
[dependencies]
webrtc-sys = { path = "../webrtc-sys", version = "0.1.1" }
livekit-utils = { path = "../livekit-utils", version = "0.1.1" }
tokio = { version = "1", features = ["full"] }
cxx = "1.0"
log = "0.4"
thiserror = "1.0"
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
webrtc-sys = { path = "../webrtc-sys", version = "0.1.1" }
futures = { version = "0.3" }
tokio = { version = "1.26.0", features = ["full"] }
cxx = "1.0"
[target.'cfg(target_arch = "wasm32")'.dependencies]
wasm-bindgen = "0.2.84"
js-sys = "0.3"
wasm-bindgen-futures = "0.4.34"
web-sys = { version = "0.3.22", features=[
"MessageEvent",
"RtcPeerConnection",
"RtcSignalingState",
"RtcSdpType",
"RtcSessionDescriptionInit",
"RtcPeerConnectionIceEvent",
"RtcIceCandidate",
"RtcDataChannel",
"RtcDataChannelEvent",
"RtcDataChannelState",
"EventTarget",
"WebGlRenderingContext",
"WebGlTexture",
] }
[dev-dependencies]
env_logger = "0.9"
livekit-webrtc/src/data_channel.rs
+72 -164
View File
@@ -1,152 +1,9 @@
use std::error::Error;
use std::fmt::{Debug, Display, Formatter};
use std::sync::Mutex;
use crate::{imp::data_channel as dc_imp, rtp_parameters::Priority};
use std::{fmt::Debug, str::Utf8Error};
use thiserror::Error;
use cxx::UniquePtr;
use log::trace;
pub use sys_dc::ffi::{DataState, Priority};
use webrtc_sys::data_channel as sys_dc;
pub struct DataChannel {
cxx_handle: UniquePtr<sys_dc::ffi::DataChannel>,
observer: Box<InternalDataChannelObserver>,
// Keep alive for C++
native_observer: UniquePtr<sys_dc::ffi::NativeDataChannelObserver>,
}
impl Debug for DataChannel {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
f.debug_struct("DataChannel")
.field("label", &self.label())
.finish()
}
}
#[derive(Debug)]
pub struct DataSendError;
impl Display for DataSendError {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
write!(f, "failed to send data to the DataChannel")
}
}
impl Error for DataSendError {}
impl DataChannel {
pub(crate) fn new(cxx_handle: UniquePtr<sys_dc::ffi::DataChannel>) -> Self {
let mut observer = Box::new(InternalDataChannelObserver::default());
let mut dc = unsafe {
Self {
cxx_handle,
native_observer: sys_dc::ffi::create_native_data_channel_observer(Box::new(
sys_dc::DataChannelObserverWrapper::new(&mut *observer),
)),
observer,
}
};
unsafe {
dc.cxx_handle
.pin_mut()
.register_observer(dc.native_observer.pin_mut());
}
dc
}
pub fn send(&self, data: &[u8], binary: bool) -> Result<(), DataSendError> {
let buffer = sys_dc::ffi::DataBuffer {
ptr: data.as_ptr(),
len: data.len(),
binary,
};
self.cxx_handle
.send(&buffer)
.then_some(())
.ok_or(DataSendError {})
}
pub fn label(&self) -> String {
self.cxx_handle.label()
}
pub fn state(&self) -> DataState {
self.cxx_handle.state()
}
pub fn close(&self) {
self.cxx_handle.close();
}
pub fn on_state_change(&mut self, handler: OnStateChangeHandler) {
*self.observer.on_state_change_handler.lock().unwrap() = Some(handler);
}
pub fn on_message(&mut self, handler: OnMessageHandler) {
*self.observer.on_message_handler.lock().unwrap() = Some(handler);
}
pub fn on_buffered_amount_change(&mut self, handler: OnBufferedAmountChangeHandler) {
*self
.observer
.on_buffered_amount_change_handler
.lock()
.unwrap() = Some(handler);
}
}
impl Drop for DataChannel {
fn drop(&mut self) {
self.cxx_handle.pin_mut().unregister_observer();
}
}
pub type OnStateChangeHandler = Box<dyn FnMut() + Send + Sync>;
pub type OnMessageHandler = Box<dyn FnMut(&[u8], bool) + Send + Sync>;
pub type OnBufferedAmountChangeHandler = Box<dyn FnMut(u64) + Send + Sync>;
#[derive(Default)]
struct InternalDataChannelObserver {
on_state_change_handler: Mutex<Option<OnStateChangeHandler>>,
on_message_handler: Mutex<Option<OnMessageHandler>>,
on_buffered_amount_change_handler: Mutex<Option<OnBufferedAmountChangeHandler>>,
}
impl sys_dc::DataChannelObserver for InternalDataChannelObserver {
fn on_state_change(&self) {
trace!("DataChannel: on_state_change");
let mut handler = self.on_state_change_handler.lock().unwrap();
if let Some(f) = handler.as_mut() {
f();
}
}
fn on_message(&self, data: &[u8], is_binary: bool) {
trace!("DataChannel: on_message");
let mut handler = self.on_message_handler.lock().unwrap();
if let Some(f) = handler.as_mut() {
f(data, is_binary);
}
}
fn on_buffered_amount_change(&self, sent_data_size: u64) {
trace!("DataChannel: on_buffered_amount_change");
let mut handler = self.on_buffered_amount_change_handler.lock().unwrap();
if let Some(f) = handler.as_mut() {
f(sent_data_size);
}
}
}
#[derive(Debug)]
#[derive(Clone, Debug)]
pub struct DataChannelInit {
#[deprecated]
pub reliable: bool,
pub ordered: bool,
pub max_retransmit_time: Option<i32>,
pub max_retransmits: Option<i32>,
@@ -159,11 +16,10 @@ pub struct DataChannelInit {
impl Default for DataChannelInit {
fn default() -> Self {
Self {
reliable: false,
ordered: true,
max_retransmit_time: None,
max_retransmits: None,
protocol: "".to_string(),
protocol: String::new(),
negotiated: false,
id: -1,
priority: None,
@@ -171,20 +27,72 @@ impl Default for DataChannelInit {
}
}
impl From<DataChannelInit> for sys_dc::ffi::DataChannelInit {
fn from(init: DataChannelInit) -> Self {
Self {
reliable: init.reliable,
ordered: init.ordered,
has_max_retransmit_time: init.max_retransmit_time.is_some(),
max_retransmit_time: init.max_retransmit_time.unwrap_or_default(),
has_max_retransmits: init.max_retransmits.is_some(),
max_retransmits: init.max_retransmits.unwrap_or_default(),
protocol: init.protocol,
negotiated: init.negotiated,
id: init.id,
has_priority: init.priority.is_some(),
priority: init.priority.unwrap_or(Priority::Low),
}
#[derive(Debug, Error)]
pub enum DataChannelError {
#[error("failed to send data, dc not open? send buffer is full ?")]
Send,
#[error("only utf8 strings can be sent")]
Utf8(#[from] Utf8Error),
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum DataState {
Connecting,
Open,
Closing,
Closed,
}
#[derive(Debug)]
pub struct DataBuffer<'a> {
pub data: &'a [u8],
pub binary: bool,
}
pub type OnStateChange = Box<dyn FnMut(DataState) + Send + Sync>;
pub type OnMessage = Box<dyn FnMut(DataBuffer) + Send + Sync>;
pub type OnBufferedAmountChange = Box<dyn FnMut(u64) + Send + Sync>;
#[derive(Clone)]
pub struct DataChannel {
pub(crate) handle: dc_imp::DataChannel,
}
impl DataChannel {
pub fn send(&self, data: &[u8], binary: bool) -> Result<(), DataChannelError> {
self.handle.send(data, binary)
}
pub fn label(&self) -> String {
self.handle.label()
}
pub fn state(&self) -> DataState {
self.handle.state()
}
pub fn close(&self) {
self.handle.close()
}
pub fn on_state_change(&self, callback: Option<OnStateChange>) {
self.handle.on_state_change(callback)
}
pub fn on_message(&self, callback: Option<OnMessage>) {
self.handle.on_message(callback)
}
pub fn on_buffered_amount_change(&self, callback: Option<OnBufferedAmountChange>) {
self.handle.on_buffered_amount_change(callback)
}
}
impl Debug for DataChannel {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("DataChannel")
.field("label", &self.label())
.field("state", &self.state())
.finish()
}
}
livekit-webrtc/src/ice_candidate.rs
+40
View File
@@ -0,0 +1,40 @@
use crate::{imp::ice_candidate as imp_ic, session_description::SdpParseError};
use std::fmt::Debug;
pub struct IceCandidate {
pub(crate) handle: imp_ic::IceCandidate,
}
impl IceCandidate {
pub fn parse(
sdp_mid: &str,
sdp_mline_index: i32,
sdp: &str,
) -> Result<IceCandidate, SdpParseError> {
imp_ic::IceCandidate::parse(sdp_mid, sdp_mline_index, sdp)
}
pub fn sdp_mid(&self) -> String {
self.handle.sdp_mid()
}
pub fn sdp_mline_index(&self) -> i32 {
self.handle.sdp_mline_index()
}
pub fn candidate(&self) -> String {
self.handle.candidate()
}
}
impl ToString for IceCandidate {
fn to_string(&self) -> String {
self.handle.to_string()
}
}
impl Debug for IceCandidate {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("IceCandidate").finish()
}
}
livekit-webrtc/src/jsep.rs
-103
View File
@@ -1,103 +0,0 @@
use std::fmt::{Debug, Formatter};
use cxx::{SharedPtr, UniquePtr};
pub use sys_jsep::ffi::{SdpParseError, SdpType};
use webrtc_sys::jsep as sys_jsep;
#[derive(Clone)]
pub struct IceCandidate {
cxx_handle: SharedPtr<sys_jsep::ffi::IceCandidate>,
}
impl Debug for IceCandidate {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
write!(f, "IceCandidate[{}]", self.to_string())
}
}
impl IceCandidate {
pub fn from(
sdp_mid: &str,
sdp_mline_index: i32,
sdp: &str,
) -> Result<IceCandidate, SdpParseError> {
let res = sys_jsep::ffi::create_ice_candidate(
sdp_mid.to_string(),
sdp_mline_index,
sdp.to_string(),
);
match res {
Ok(cxx_handle) => Ok(IceCandidate::new(cxx_handle)),
Err(e) => Err(unsafe { SdpParseError::from(e.what()) }),
}
}
pub(crate) fn new(cxx_handle: SharedPtr<sys_jsep::ffi::IceCandidate>) -> Self {
Self { cxx_handle }
}
pub(crate) fn release(self) -> SharedPtr<sys_jsep::ffi::IceCandidate> {
self.cxx_handle
}
pub fn sdp_mid(&self) -> String {
self.cxx_handle.sdp_mid()
}
pub fn sdp_mline_index(&self) -> i32 {
self.cxx_handle.sdp_mline_index()
}
pub fn candidate(&self) -> String {
self.cxx_handle.candidate()
}
}
impl ToString for IceCandidate {
fn to_string(&self) -> String {
self.cxx_handle.stringify()
}
}
pub struct SessionDescription {
cxx_handle: UniquePtr<sys_jsep::ffi::SessionDescription>,
}
impl Debug for SessionDescription {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
write!(f, "SessionDescription[{}]", self.to_string())
}
}
impl SessionDescription {
pub fn from(sdp_type: SdpType, description: &str) -> Result<SessionDescription, SdpParseError> {
let res = sys_jsep::ffi::create_session_description(sdp_type, description.to_string());
match res {
Ok(cxx_handle) => Ok(SessionDescription::new(cxx_handle)),
Err(e) => Err(unsafe { SdpParseError::from(e.what()) }),
}
}
pub(crate) fn new(cxx_handle: UniquePtr<sys_jsep::ffi::SessionDescription>) -> Self {
Self { cxx_handle }
}
pub(crate) fn release(self) -> UniquePtr<sys_jsep::ffi::SessionDescription> {
self.cxx_handle
}
}
impl ToString for SessionDescription {
fn to_string(&self) -> String {
self.cxx_handle.stringify()
}
}
impl Clone for SessionDescription {
fn clone(&self) -> Self {
SessionDescription::new(self.cxx_handle.clone())
}
}
livekit-webrtc/src/lib.rs
+38 -5
View File
@@ -1,15 +1,48 @@
use thiserror::Error;
#[cfg_attr(target_arch = "wasm32", path = "web/mod.rs")]
#[cfg_attr(not(target_arch = "wasm32"), path = "native/mod.rs")]
mod imp;
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum MediaType {
Audio,
Video,
Data,
Unsupported,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum RtcErrorType {
Internal,
InvalidSdp,
InvalidState,
}
#[derive(Error, Debug)]
#[error("an RtcError occured: {error_type:?} - {message}")]
pub struct RtcError {
pub error_type: RtcErrorType,
pub message: String,
}
pub mod data_channel;
pub mod jsep;
pub mod ice_candidate;
pub mod media_stream;
pub mod peer_connection;
pub mod peer_connection_factory;
pub mod prelude;
pub mod rtc_error;
pub mod rtp_parameters;
pub mod rtp_receiver;
pub mod rtp_sender;
pub mod rtp_transceiver;
pub mod session_description;
pub mod video_frame;
pub mod video_frame_buffer;
pub mod webrtc;
pub mod yuv_helper;
pub mod video_source;
pub mod video_stream;
#[cfg(not(target_arch = "wasm32"))]
pub mod native {
pub use crate::imp::yuv_helper;
pub use webrtc_sys::webrtc::ffi::create_random_uuid;
}
livekit-webrtc/src/media_stream.rs
+111 -226
View File
@@ -1,254 +1,139 @@
use cxx::{SharedPtr, UniquePtr};
use crate::imp::media_stream as imp_ms;
use livekit_utils::enum_dispatch;
use std::fmt::{Debug, Formatter};
use std::sync::{Arc, Mutex};
use webrtc_sys::media_stream as sys_ms;
use webrtc_sys::MEDIA_TYPE_VIDEO;
use std::fmt::Debug;
pub use sys_ms::ffi::ContentHint;
pub use sys_ms::ffi::TrackState;
use crate::video_frame::VideoFrame;
use crate::video_frame_buffer::VideoFrameBuffer;
pub trait MediaStreamTrackTrait {
fn kind(&self) -> String;
fn id(&self) -> String;
fn enabled(&self) -> bool;
fn set_enabled(&self, enabled: bool) -> bool;
fn state(&self) -> TrackState;
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum RtcTrackState {
Live,
Ended,
}
#[derive(Clone)]
pub enum MediaStreamTrackHandle {
Audio(Arc<AudioTrack>),
Video(Arc<VideoTrack>),
pub struct MediaStream {
pub(crate) handle: imp_ms::MediaStream,
}
impl MediaStreamTrackHandle {
pub(crate) fn new(cxx_handle: SharedPtr<sys_ms::ffi::MediaStreamTrack>) -> Self {
if cxx_handle.kind() == MEDIA_TYPE_VIDEO {
Self::Video(VideoTrack::new(cxx_handle))
} else {
Self::Audio(AudioTrack::new(cxx_handle))
}
impl MediaStream {
pub fn id(&self) -> String {
self.handle.id()
}
// TODO(theomonnom): enum_dispatch with visibility support?
pub(crate) fn cxx_handle(&self) -> SharedPtr<sys_ms::ffi::MediaStreamTrack> {
match self {
Self::Video(video) => video.cxx_handle(),
Self::Audio(audio) => audio.cxx_handle(),
}
pub fn audio_tracks(&self) -> Vec<RtcAudioTrack> {
self.handle.audio_tracks()
}
pub fn video_tracks(&self) -> Vec<RtcVideoTrack> {
self.handle.video_tracks()
}
}
impl Debug for MediaStreamTrackHandle {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
f.debug_struct("MediaStreamTrack")
impl Debug for MediaStream {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("MediaStream")
.field("id", &self.id())
.field("audio_tracks", &self.audio_tracks())
.field("video_tracks", &self.video_tracks())
.finish()
}
}
#[derive(Clone)]
pub struct RtcVideoTrack {
pub(crate) handle: imp_ms::RtcVideoTrack,
}
#[derive(Clone)]
pub struct RtcAudioTrack {
pub(crate) handle: imp_ms::RtcAudioTrack,
}
#[derive(Debug, Clone)]
pub enum MediaStreamTrack {
Video(RtcVideoTrack),
Audio(RtcAudioTrack),
}
#[cfg(not(target_arch = "wasm32"))]
impl MediaStreamTrack {
enum_dispatch!(
[Video, Audio];
pub(crate) fn sys_handle(self: &Self) -> cxx::SharedPtr<webrtc_sys::media_stream::ffi::MediaStreamTrack>;
);
}
impl MediaStreamTrack {
enum_dispatch!(
[Video, Audio];
pub fn id(self: &Self) -> String;
pub fn enabled(self: &Self) -> bool;
pub fn set_enabled(self: &Self, enabled: bool) -> bool;
pub fn state(self: &Self) -> RtcTrackState;
);
}
macro_rules! media_stream_track {
() => {
pub fn id(&self) -> String {
self.handle.id()
}
pub fn enabled(&self) -> bool {
self.handle.enabled()
}
pub fn set_enabled(&self, enabled: bool) -> bool {
self.handle.set_enabled(enabled)
}
pub fn state(&self) -> RtcTrackState {
self.handle.state().into()
}
#[cfg(not(target_arch = "wasm32"))]
pub(crate) fn sys_handle(
&self,
) -> cxx::SharedPtr<webrtc_sys::media_stream::ffi::MediaStreamTrack> {
self.handle.sys_handle()
}
};
}
impl RtcVideoTrack {
media_stream_track!();
}
impl RtcAudioTrack {
media_stream_track!();
}
impl Debug for RtcAudioTrack {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RtcAudioTrack")
.field("id", &self.id())
.field("kind", &self.kind())
.field("enabled", &self.enabled())
.field("state", &self.state())
.finish()
}
}
impl MediaStreamTrackTrait for MediaStreamTrackHandle {
enum_dispatch!(
[Audio, Video]
fnc!(kind, &Self, [], String);
fnc!(id, &Self, [], String);
fnc!(enabled, &Self, [], bool);
fnc!(state, &Self, [], TrackState);
fnc!(set_enabled, &Self, [enabled: bool], bool);
);
}
pub struct AudioTrack {
cxx_handle: SharedPtr<sys_ms::ffi::MediaStreamTrack>,
}
impl AudioTrack {
fn new(cxx_handle: SharedPtr<sys_ms::ffi::MediaStreamTrack>) -> Arc<Self> {
Arc::new(Self { cxx_handle })
}
pub(crate) fn cxx_handle(&self) -> SharedPtr<sys_ms::ffi::MediaStreamTrack> {
self.cxx_handle.clone()
}
}
pub struct VideoTrack {
cxx_handle: SharedPtr<sys_ms::ffi::MediaStreamTrack>,
observer: Box<InternalVideoTrackSink>,
// Keep alive for c++
native_observer: UniquePtr<sys_ms::ffi::NativeVideoFrameSink>,
}
impl VideoTrack {
fn new(cxx_handle: SharedPtr<sys_ms::ffi::MediaStreamTrack>) -> Arc<Self> {
let mut observer = Box::new(InternalVideoTrackSink::default());
let mut track = unsafe {
Self {
cxx_handle,
native_observer: {
sys_ms::ffi::create_native_video_frame_sink(Box::new(
sys_ms::VideoFrameSinkWrapper::new(&mut *observer),
))
},
observer,
}
};
unsafe {
(*track.video_handle()).add_sink(track.native_observer.pin_mut());
}
Arc::new(track)
}
pub(crate) fn cxx_handle(&self) -> SharedPtr<sys_ms::ffi::MediaStreamTrack> {
self.cxx_handle.clone()
}
fn video_handle(&self) -> *const sys_ms::ffi::VideoTrack {
unsafe { sys_ms::ffi::media_to_video(&*self.cxx_handle) }
}
pub fn set_should_receive(&self, should_receive: bool) {
unsafe { (*self.video_handle()).set_should_receive(should_receive) }
}
pub fn set_content_hint(&self, hint: ContentHint) {
unsafe { (*self.video_handle()).set_content_hint(hint) }
}
pub fn should_receive(&self) -> bool {
unsafe { (*self.video_handle()).should_receive() }
}
pub fn content_hint(&self) -> ContentHint {
unsafe { (*self.video_handle()).content_hint() }
}
pub fn on_frame(&self, handler: OnFrameHandler) {
*self.observer.on_frame_handler.lock().unwrap() = Some(handler);
}
pub fn on_discarded_frame(&self, handler: OnDiscardedFrameHandler) {
*self.observer.on_discarded_frame_handler.lock().unwrap() = Some(handler);
}
pub fn on_constraints_changed(&self, handler: OnConstraintsChangedHandler) {
*self.observer.on_constraints_changed_handler.lock().unwrap() = Some(handler);
}
}
impl Drop for VideoTrack {
fn drop(&mut self) {
unsafe {
(*self.video_handle()).remove_sink(self.native_observer.pin_mut());
}
}
}
macro_rules! impl_media_stream_track_trait {
($x:ty) => {
impl MediaStreamTrackTrait for $x {
fn kind(&self) -> String {
self.cxx_handle.kind()
}
fn id(&self) -> String {
self.cxx_handle.id()
}
fn enabled(&self) -> bool {
self.cxx_handle.enabled()
}
fn set_enabled(&self, enabled: bool) -> bool {
self.cxx_handle.set_enabled(enabled)
}
fn state(&self) -> TrackState {
self.cxx_handle.state()
}
}
};
}
impl_media_stream_track_trait!(VideoTrack);
impl_media_stream_track_trait!(AudioTrack);
pub type OnFrameHandler = Box<dyn FnMut(VideoFrame, VideoFrameBuffer) + Send + Sync>;
pub type OnDiscardedFrameHandler = Box<dyn FnMut() + Send + Sync>;
pub type OnConstraintsChangedHandler = Box<dyn FnMut(VideoTrackSourceConstraints) + Send + Sync>;
#[derive(Default)]
struct InternalVideoTrackSink {
on_frame_handler: Mutex<Option<OnFrameHandler>>,
on_discarded_frame_handler: Mutex<Option<OnDiscardedFrameHandler>>,
on_constraints_changed_handler: Mutex<Option<OnConstraintsChangedHandler>>,
}
pub struct VideoTrackSourceConstraints {
pub min_fps: Option<f64>,
pub max_fps: Option<f64>,
}
impl From<sys_ms::ffi::VideoTrackSourceConstraints> for VideoTrackSourceConstraints {
fn from(cst: sys_ms::ffi::VideoTrackSourceConstraints) -> Self {
Self {
min_fps: (cst.min_fps != 1.0).then_some(cst.min_fps),
max_fps: (cst.max_fps != 1.0).then_some(cst.max_fps),
}
}
}
impl sys_ms::VideoFrameSink for InternalVideoTrackSink {
fn on_frame(&self, frame: UniquePtr<webrtc_sys::video_frame::ffi::VideoFrame>) {
if let Some(cb) = self.on_frame_handler.lock().unwrap().as_mut() {
let frame = VideoFrame::new(frame);
let video_frame_buffer = unsafe { frame.video_frame_buffer() };
cb(frame, video_frame_buffer);
}
}
fn on_discarded_frame(&self) {
if let Some(cb) = self.on_discarded_frame_handler.lock().unwrap().as_mut() {
cb();
}
}
fn on_constraints_changed(&self, constraints: sys_ms::ffi::VideoTrackSourceConstraints) {
if let Some(cb) = self.on_constraints_changed_handler.lock().unwrap().as_mut() {
cb(constraints.into());
}
}
}
pub struct MediaStream {
cxx_handle: SharedPtr<sys_ms::ffi::MediaStream>,
}
impl Debug for MediaStream {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
f.debug_struct("MediaStream")
impl Debug for RtcVideoTrack {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RtcVideoTrack")
.field("id", &self.id())
.field("enabled", &self.enabled())
.field("state", &self.state())
.finish()
}
}
impl MediaStream {
pub(crate) fn new(cxx_handle: SharedPtr<sys_ms::ffi::MediaStream>) -> Self {
Self { cxx_handle }
impl From<RtcAudioTrack> for MediaStreamTrack {
fn from(track: RtcAudioTrack) -> Self {
Self::Audio(track)
}
}
pub fn id(&self) -> String {
self.cxx_handle.id()
impl From<RtcVideoTrack> for MediaStreamTrack {
fn from(track: RtcVideoTrack) -> Self {
Self::Video(track)
}
}
livekit-webrtc/src/native/data_channel.rs
+137
View File
@@ -0,0 +1,137 @@
use crate::data_channel::{
DataBuffer, DataChannelError, DataChannelInit, DataState, OnBufferedAmountChange, OnMessage,
OnStateChange,
};
use cxx::SharedPtr;
use std::str;
use std::sync::{Arc, Mutex};
use webrtc_sys::data_channel as sys_dc;
impl From<sys_dc::ffi::DataState> for DataState {
fn from(value: sys_dc::ffi::DataState) -> Self {
match value {
sys_dc::ffi::DataState::Connecting => Self::Connecting,
sys_dc::ffi::DataState::Open => Self::Open,
sys_dc::ffi::DataState::Closing => Self::Closing,
sys_dc::ffi::DataState::Closed => Self::Closed,
_ => panic!("unknown data channel state"),
}
}
}
impl From<DataChannelInit> for sys_dc::ffi::DataChannelInit {
fn from(value: DataChannelInit) -> Self {
Self {
ordered: value.ordered,
has_max_retransmit_time: value.max_retransmit_time.is_some(),
max_retransmit_time: value.max_retransmit_time.unwrap_or_default(),
has_max_retransmits: value.max_retransmits.is_some(),
max_retransmits: value.max_retransmits.unwrap_or_default(),
protocol: value.protocol,
id: value.id,
has_priority: false,
priority: sys_dc::ffi::Priority::Medium,
negotiated: value.negotiated,
}
}
}
#[derive(Clone)]
pub struct DataChannel {
#[allow(dead_code)]
native_observer: SharedPtr<sys_dc::ffi::NativeDataChannelObserver>,
observer: Arc<DataChannelObserver>,
pub(crate) sys_handle: SharedPtr<sys_dc::ffi::DataChannel>,
}
impl DataChannel {
pub fn configure(sys_handle: SharedPtr<sys_dc::ffi::DataChannel>) -> Self {
unsafe {
let observer = Arc::new(DataChannelObserver::default());
let dc = Self {
sys_handle: sys_handle.clone(),
native_observer: sys_dc::ffi::create_native_data_channel_observer(
Box::new(sys_dc::DataChannelObserverWrapper::new(observer.clone())),
&*sys_handle as *const _ as *mut _,
),
observer,
};
dc.sys_handle
.register_observer(&*dc.native_observer as *const _ as *mut _);
dc
}
}
pub fn send(&self, data: &[u8], binary: bool) -> Result<(), DataChannelError> {
if !binary {
str::from_utf8(data)?;
}
let buffer = sys_dc::ffi::DataBuffer {
ptr: data.as_ptr(),
len: data.len(),
binary,
};
self.sys_handle
.send(&buffer)
.then_some(())
.ok_or(DataChannelError::Send)
}
pub fn label(&self) -> String {
self.sys_handle.label()
}
pub fn state(&self) -> DataState {
self.sys_handle.state().into()
}
pub fn close(&self) {
self.sys_handle.close();
}
pub fn on_state_change(&self, handler: Option<OnStateChange>) {
*self.observer.state_change_handler.lock().unwrap() = handler;
}
pub fn on_message(&self, handler: Option<OnMessage>) {
*self.observer.message_handler.lock().unwrap() = handler;
}
pub fn on_buffered_amount_change(&self, handler: Option<OnBufferedAmountChange>) {
*self.observer.buffered_amount_change_handler.lock().unwrap() = handler;
}
}
#[derive(Default)]
struct DataChannelObserver {
state_change_handler: Mutex<Option<OnStateChange>>,
message_handler: Mutex<Option<OnMessage>>,
buffered_amount_change_handler: Mutex<Option<OnBufferedAmountChange>>,
}
impl sys_dc::DataChannelObserver for DataChannelObserver {
fn on_state_change(&self, state: sys_dc::ffi::DataState) {
let mut handler = self.state_change_handler.lock().unwrap();
if let Some(f) = handler.as_mut() {
f(state.into());
}
}
fn on_message(&self, data: &[u8], binary: bool) {
let mut handler = self.message_handler.lock().unwrap();
if let Some(f) = handler.as_mut() {
f(DataBuffer { data, binary });
}
}
fn on_buffered_amount_change(&self, sent_data_size: u64) {
let mut handler = self.buffered_amount_change_handler.lock().unwrap();
if let Some(f) = handler.as_mut() {
f(sent_data_size);
}
}
}
livekit-webrtc/src/native/ice_candidate.rs
+48
View File
@@ -0,0 +1,48 @@
use crate::ice_candidate as ic;
use crate::session_description::SdpParseError;
use cxx::SharedPtr;
use webrtc_sys::jsep as sys_jsep;
#[derive(Clone)]
pub struct IceCandidate {
pub(crate) sys_handle: SharedPtr<sys_jsep::ffi::IceCandidate>,
}
impl IceCandidate {
pub fn parse(
sdp_mid: &str,
sdp_mline_index: i32,
sdp: &str,
) -> Result<ic::IceCandidate, SdpParseError> {
let res = sys_jsep::ffi::create_ice_candidate(
sdp_mid.to_string(),
sdp_mline_index,
sdp.to_string(),
);
match res {
Ok(sys_handle) => Ok(ic::IceCandidate {
handle: IceCandidate { sys_handle },
}),
Err(e) => Err(unsafe { sys_jsep::ffi::SdpParseError::from(e.what()).into() }),
}
}
pub fn sdp_mid(&self) -> String {
self.sys_handle.sdp_mid()
}
pub fn sdp_mline_index(&self) -> i32 {
self.sys_handle.sdp_mline_index()
}
pub fn candidate(&self) -> String {
self.sys_handle.candidate()
}
}
impl ToString for IceCandidate {
fn to_string(&self) -> String {
self.sys_handle.stringify()
}
}
livekit-webrtc/src/native/media_stream.rs
+118
View File
@@ -0,0 +1,118 @@
use crate::media_stream::{self, MediaStreamTrack, RtcTrackState};
use cxx::SharedPtr;
use webrtc_sys::media_stream as sys_ms;
use webrtc_sys::media_stream::ffi::{
audio_to_media, media_to_audio, media_to_video, video_to_media,
};
use webrtc_sys::{MEDIA_TYPE_AUDIO, MEDIA_TYPE_VIDEO};
impl From<sys_ms::ffi::TrackState> for RtcTrackState {
fn from(state: sys_ms::ffi::TrackState) -> Self {
match state {
sys_ms::ffi::TrackState::Live => RtcTrackState::Live,
sys_ms::ffi::TrackState::Ended => RtcTrackState::Ended,
_ => panic!("unknown TrackState"),
}
}
}
#[derive(Clone)]
pub struct MediaStream {
pub(crate) sys_handle: SharedPtr<sys_ms::ffi::MediaStream>,
}
impl MediaStream {
pub fn id(&self) -> String {
self.sys_handle.id()
}
pub fn audio_tracks(&self) -> Vec<media_stream::RtcAudioTrack> {
self.sys_handle
.get_audio_tracks()
.into_iter()
.map(|t| media_stream::RtcAudioTrack {
handle: RtcAudioTrack { sys_handle: t.ptr },
})
.collect()
}
pub fn video_tracks(&self) -> Vec<media_stream::RtcVideoTrack> {
self.sys_handle
.get_video_tracks()
.into_iter()
.map(|t| media_stream::RtcVideoTrack {
handle: RtcVideoTrack { sys_handle: t.ptr },
})
.collect()
}
}
pub fn new_media_stream_track(
sys_handle: SharedPtr<sys_ms::ffi::MediaStreamTrack>,
) -> MediaStreamTrack {
if sys_handle.kind() == MEDIA_TYPE_AUDIO {
MediaStreamTrack::Audio(media_stream::RtcAudioTrack {
handle: RtcAudioTrack {
sys_handle: media_to_audio(sys_handle),
},
})
} else if sys_handle.kind() == MEDIA_TYPE_VIDEO {
MediaStreamTrack::Video(media_stream::RtcVideoTrack {
handle: RtcVideoTrack {
sys_handle: media_to_video(sys_handle),
},
})
} else {
panic!("unknown track kind")
}
}
macro_rules! impl_media_stream_track {
($cast:ident) => {
pub fn id(&self) -> String {
let ptr = sys_ms::ffi::$cast(self.sys_handle.clone());
ptr.id()
}
pub fn enabled(&self) -> bool {
let ptr = sys_ms::ffi::$cast(self.sys_handle.clone());
ptr.enabled()
}
pub fn set_enabled(&self, enabled: bool) -> bool {
let ptr = sys_ms::ffi::$cast(self.sys_handle.clone());
ptr.set_enabled(enabled)
}
pub fn state(&self) -> RtcTrackState {
let ptr = sys_ms::ffi::$cast(self.sys_handle.clone());
ptr.state().into()
}
};
}
#[derive(Clone)]
pub struct RtcVideoTrack {
pub(crate) sys_handle: SharedPtr<sys_ms::ffi::VideoTrack>,
}
impl RtcVideoTrack {
impl_media_stream_track!(video_to_media);
pub fn sys_handle(&self) -> SharedPtr<sys_ms::ffi::MediaStreamTrack> {
video_to_media(self.sys_handle.clone())
}
}
#[derive(Clone)]
pub struct RtcAudioTrack {
pub(crate) sys_handle: SharedPtr<sys_ms::ffi::AudioTrack>,
}
impl RtcAudioTrack {
impl_media_stream_track!(audio_to_media);
pub fn sys_handle(&self) -> SharedPtr<sys_ms::ffi::MediaStreamTrack> {
audio_to_media(self.sys_handle.clone())
}
}
livekit-webrtc/src/native/mod.rs
+48
View File
@@ -0,0 +1,48 @@
pub mod data_channel;
pub mod ice_candidate;
pub mod media_stream;
pub mod peer_connection;
pub mod peer_connection_factory;
pub mod rtp_parameters;
pub mod rtp_receiver;
pub mod rtp_sender;
pub mod rtp_transceiver;
pub mod session_description;
pub mod video_frame;
pub mod video_source;
pub mod video_stream;
pub mod yuv_helper;
use crate::MediaType;
use crate::{RtcError, RtcErrorType};
use webrtc_sys::rtc_error as sys_err;
use webrtc_sys::webrtc as sys_rtc;
impl From<sys_err::ffi::RTCErrorType> for RtcErrorType {
fn from(value: sys_err::ffi::RTCErrorType) -> Self {
match value {
sys_err::ffi::RTCErrorType::InvalidState => Self::InvalidState,
_ => Self::Internal,
}
}
}
impl From<sys_err::ffi::RTCError> for RtcError {
fn from(value: sys_err::ffi::RTCError) -> Self {
Self {
error_type: value.error_type.into(),
message: value.message,
}
}
}
impl From<MediaType> for sys_rtc::ffi::MediaType {
fn from(value: MediaType) -> Self {
match value {
MediaType::Audio => Self::Audio,
MediaType::Video => Self::Video,
MediaType::Data => Self::Data,
MediaType::Unsupported => Self::Unsupported,
}
}
}
livekit-webrtc/src/native/peer_connection.rs
+579
View File
@@ -0,0 +1,579 @@
use crate::data_channel::DataChannel;
use crate::data_channel::DataChannelInit;
use crate::ice_candidate::IceCandidate;
use crate::imp::data_channel as imp_dc;
use crate::imp::ice_candidate as imp_ic;
use crate::imp::media_stream as imp_ms;
use crate::imp::rtp_receiver as imp_rr;
use crate::imp::rtp_sender as imp_rs;
use crate::imp::rtp_transceiver as imp_rt;
use crate::imp::session_description as imp_sdp;
use crate::media_stream::{MediaStream, MediaStreamTrack};
use crate::peer_connection::{
AnswerOptions, IceCandidateError, IceConnectionState, IceGatheringState, OfferOptions,
OnConnectionChange, OnDataChannel, OnIceCandidate, OnIceCandidateError, OnIceConnectionChange,
OnIceGatheringChange, OnNegotiationNeeded, OnSignalingChange, OnTrack, PeerConnectionState,
SignalingState, TrackEvent,
};
use crate::rtp_receiver::RtpReceiver;
use crate::rtp_sender::RtpSender;
use crate::rtp_transceiver::RtpTransceiver;
use crate::rtp_transceiver::RtpTransceiverInit;
use crate::MediaType;
use crate::{session_description::SessionDescription, RtcError};
use cxx::{SharedPtr, UniquePtr};
use futures::channel::oneshot;
use std::mem::ManuallyDrop;
use std::sync::{Arc, Mutex};
use webrtc_sys::data_channel as sys_dc;
use webrtc_sys::jsep as sys_jsep;
use webrtc_sys::peer_connection as sys_pc;
use webrtc_sys::rtc_error as sys_err;
impl From<OfferOptions> for sys_pc::ffi::RTCOfferAnswerOptions {
fn from(options: OfferOptions) -> Self {
Self {
ice_restart: options.ice_restart,
offer_to_receive_audio: options.offer_to_receive_audio as i32,
offer_to_receive_video: options.offer_to_receive_video as i32,
..Default::default()
}
}
}
impl From<AnswerOptions> for sys_pc::ffi::RTCOfferAnswerOptions {
fn from(_options: AnswerOptions) -> Self {
Self::default()
}
}
impl From<sys_pc::ffi::PeerConnectionState> for PeerConnectionState {
fn from(state: sys_pc::ffi::PeerConnectionState) -> Self {
match state {
sys_pc::ffi::PeerConnectionState::New => PeerConnectionState::New,
sys_pc::ffi::PeerConnectionState::Connecting => PeerConnectionState::Connecting,
sys_pc::ffi::PeerConnectionState::Connected => PeerConnectionState::Connected,
sys_pc::ffi::PeerConnectionState::Disconnected => PeerConnectionState::Disconnected,
sys_pc::ffi::PeerConnectionState::Failed => PeerConnectionState::Failed,
sys_pc::ffi::PeerConnectionState::Closed => PeerConnectionState::Closed,
_ => panic!("unknown PeerConnectionState"),
}
}
}
impl From<sys_pc::ffi::IceConnectionState> for IceConnectionState {
fn from(state: sys_pc::ffi::IceConnectionState) -> Self {
match state {
sys_pc::ffi::IceConnectionState::IceConnectionNew => IceConnectionState::New,
sys_pc::ffi::IceConnectionState::IceConnectionChecking => IceConnectionState::Checking,
sys_pc::ffi::IceConnectionState::IceConnectionConnected => {
IceConnectionState::Connected
}
sys_pc::ffi::IceConnectionState::IceConnectionCompleted => {
IceConnectionState::Completed
}
sys_pc::ffi::IceConnectionState::IceConnectionFailed => IceConnectionState::Failed,
sys_pc::ffi::IceConnectionState::IceConnectionDisconnected => {
IceConnectionState::Disconnected
}
sys_pc::ffi::IceConnectionState::IceConnectionClosed => IceConnectionState::Closed,
sys_pc::ffi::IceConnectionState::IceConnectionMax => IceConnectionState::Max,
_ => panic!("unknown IceConnectionState"),
}
}
}
impl From<sys_pc::ffi::IceGatheringState> for IceGatheringState {
fn from(state: sys_pc::ffi::IceGatheringState) -> Self {
match state {
sys_pc::ffi::IceGatheringState::IceGatheringNew => IceGatheringState::New,
sys_pc::ffi::IceGatheringState::IceGatheringGathering => IceGatheringState::Gathering,
sys_pc::ffi::IceGatheringState::IceGatheringComplete => IceGatheringState::Complete,
_ => panic!("unknown IceGatheringState"),
}
}
}
impl From<sys_pc::ffi::SignalingState> for SignalingState {
fn from(state: sys_pc::ffi::SignalingState) -> Self {
match state {
sys_pc::ffi::SignalingState::Stable => SignalingState::Stable,
sys_pc::ffi::SignalingState::HaveLocalOffer => SignalingState::HaveLocalOffer,
sys_pc::ffi::SignalingState::HaveRemoteOffer => SignalingState::HaveRemoteOffer,
sys_pc::ffi::SignalingState::HaveLocalPrAnswer => SignalingState::HaveLocalPrAnswer,
sys_pc::ffi::SignalingState::HaveRemotePrAnswer => SignalingState::HaveRemotePrAnswer,
sys_pc::ffi::SignalingState::Closed => SignalingState::Closed,
_ => panic!("unknown SignalingState"),
}
}
}
#[derive(Clone)]
pub struct PeerConnection {
native_observer: SharedPtr<sys_pc::ffi::NativePeerConnectionObserver>,
observer: Arc<PeerObserver>,
pub(crate) sys_handle: SharedPtr<sys_pc::ffi::PeerConnection>,
}
impl PeerConnection {
pub fn configure(
sys_handle: SharedPtr<sys_pc::ffi::PeerConnection>,
observer: Arc<PeerObserver>,
native_observer: SharedPtr<sys_pc::ffi::NativePeerConnectionObserver>,
) -> Self {
Self {
sys_handle,
observer,
native_observer,
}
}
pub async fn create_offer(
&self,
options: OfferOptions,
) -> Result<SessionDescription, RtcError> {
let (mut native_wrapper, mut sdp_rx, mut err_rx) = create_sdp_observer();
unsafe {
self.sys_handle
.create_offer(native_wrapper.pin_mut(), options.into());
}
futures::select! {
sdp = sdp_rx => Ok(sdp.unwrap()),
err = err_rx => Err(err.unwrap()),
}
}
pub async fn create_answer(
&self,
options: AnswerOptions,
) -> Result<SessionDescription, RtcError> {
let (mut native_wrapper, mut sdp_rx, mut err_rx) = create_sdp_observer();
unsafe {
self.sys_handle
.create_answer(native_wrapper.pin_mut(), options.into());
}
futures::select! {
sdp = sdp_rx => Ok(sdp.unwrap()),
err = err_rx => Err(err.unwrap()),
}
}
pub async fn set_local_description(&self, desc: SessionDescription) -> Result<(), RtcError> {
let (tx, rx) = oneshot::channel();
let wrapper =
sys_jsep::SetLocalSdpObserverWrapper(ManuallyDrop::new(Box::new(move |error| {
let _ = tx.send(if error.ok() { Ok(()) } else { Err(error) });
})));
let mut native_wrapper =
sys_jsep::ffi::create_native_set_local_sdp_observer(Box::new(wrapper));
unsafe {
self.sys_handle
.set_local_description(desc.handle.sys_handle, native_wrapper.pin_mut());
}
rx.await.unwrap().map_err(Into::into)
}
pub async fn set_remote_description(&self, desc: SessionDescription) -> Result<(), RtcError> {
let (tx, rx) = oneshot::channel();
let wrapper =
sys_jsep::SetRemoteSdpObserverWrapper(ManuallyDrop::new(Box::new(move |error| {
let _ = tx.send(if error.ok() { Ok(()) } else { Err(error) });
})));
let mut native_wrapper =
sys_jsep::ffi::create_native_set_remote_sdp_observer(Box::new(wrapper));
unsafe {
self.sys_handle
.set_remote_description(desc.handle.sys_handle, native_wrapper.pin_mut());
}
rx.await.unwrap().map_err(Into::into)
}
pub async fn add_ice_candidate(&self, candidate: IceCandidate) -> Result<(), RtcError> {
let (tx, rx) = oneshot::channel();
let observer =
sys_pc::AddIceCandidateObserverWrapper(ManuallyDrop::new(Box::new(|error| {
let _ = tx.send(if error.ok() { Ok(()) } else { Err(error) });
})));
let mut native_observer =
sys_pc::ffi::create_native_add_ice_candidate_observer(Box::new(observer));
self.sys_handle
.add_ice_candidate(candidate.handle.sys_handle, native_observer.pin_mut());
rx.await.unwrap().map_err(Into::into)
}
pub fn create_data_channel(
&self,
label: &str,
init: DataChannelInit,
) -> Result<DataChannel, RtcError> {
let native_init = sys_dc::ffi::create_data_channel_init(init.into());
let res = self
.sys_handle
.create_data_channel(label.to_string(), native_init);
match res {
Ok(sys_handle) => Ok(DataChannel {
handle: imp_dc::DataChannel::configure(sys_handle),
}),
Err(e) => Err(unsafe { sys_err::ffi::RTCError::from(e.what()).into() }),
}
}
pub fn add_track<T: AsRef<str>>(
&self,
track: MediaStreamTrack,
stream_ids: &[T],
) -> Result<RtpSender, RtcError> {
let stream_ids = stream_ids.iter().map(|s| s.as_ref().to_owned()).collect();
let res = self.sys_handle.add_track(track.sys_handle(), &stream_ids);
match res {
Ok(sys_handle) => Ok(RtpSender {
handle: imp_rs::RtpSender { sys_handle },
}),
Err(e) => unsafe { Err(sys_err::ffi::RTCError::from(e.what()).into()) },
}
}
pub fn add_transceiver(
&self,
track: MediaStreamTrack,
init: RtpTransceiverInit,
) -> Result<RtpTransceiver, RtcError> {
let res = self
.sys_handle
.add_transceiver(track.sys_handle(), init.into());
match res {
Ok(sys_handle) => Ok(RtpTransceiver {
handle: imp_rt::RtpTransceiver {
sys_handle: sys_handle,
},
}),
Err(e) => unsafe { Err(sys_err::ffi::RTCError::from(e.what()).into()) },
}
}
pub fn add_transceiver_for_media(
&self,
media_type: MediaType,
init: RtpTransceiverInit,
) -> Result<RtpTransceiver, RtcError> {
let res = self
.sys_handle
.add_transceiver_for_media(media_type.into(), init.into());
match res {
Ok(cxx_handle) => Ok(RtpTransceiver {
handle: imp_rt::RtpTransceiver {
sys_handle: cxx_handle,
},
}),
Err(e) => unsafe { Err(sys_err::ffi::RTCError::from(e.what()).into()) },
}
}
pub fn close(&self) {
self.sys_handle.close();
}
pub fn connection_state(&self) -> PeerConnectionState {
self.sys_handle.connection_state().into()
}
pub fn ice_connection_state(&self) -> IceConnectionState {
self.sys_handle.ice_connection_state().into()
}
pub fn ice_gathering_state(&self) -> IceGatheringState {
self.sys_handle.ice_gathering_state().into()
}
pub fn signaling_state(&self) -> SignalingState {
self.sys_handle.signaling_state().into()
}
pub fn current_local_description(&self) -> Option<SessionDescription> {
let sdp = self.sys_handle.current_local_description();
if sdp.is_null() {
return None;
}
Some(SessionDescription {
handle: imp_sdp::SessionDescription { sys_handle: sdp },
})
}
pub fn current_remote_description(&self) -> Option<SessionDescription> {
let sdp = self.sys_handle.current_remote_description();
if sdp.is_null() {
return None;
}
Some(SessionDescription {
handle: imp_sdp::SessionDescription { sys_handle: sdp },
})
}
pub fn remove_track(&self, sender: RtpSender) -> Result<(), RtcError> {
self.sys_handle
.remove_track(sender.handle.sys_handle)
.map_err(|e| unsafe { sys_err::ffi::RTCError::from(e.what()).into() })
}
pub fn senders(&self) -> Vec<RtpSender> {
self.sys_handle
.get_senders()
.into_iter()
.map(|sender| RtpSender {
handle: imp_rs::RtpSender {
sys_handle: sender.ptr,
},
})
.collect()
}
pub fn receivers(&self) -> Vec<RtpReceiver> {
self.sys_handle
.get_receivers()
.into_iter()
.map(|receiver| RtpReceiver {
handle: imp_rr::RtpReceiver {
sys_handle: receiver.ptr,
},
})
.collect()
}
pub fn transceivers(&self) -> Vec<RtpTransceiver> {
self.sys_handle
.get_transceivers()
.into_iter()
.map(|transceiver| RtpTransceiver {
handle: imp_rt::RtpTransceiver {
sys_handle: transceiver.ptr,
},
})
.collect()
}
pub fn on_connection_state_change(&self, f: Option<OnConnectionChange>) {
*self.observer.connection_change_handler.lock().unwrap() = f;
}
pub fn on_data_channel(&self, f: Option<OnDataChannel>) {
*self.observer.data_channel_handler.lock().unwrap() = f;
}
pub fn on_ice_candidate(&self, f: Option<OnIceCandidate>) {
*self.observer.ice_candidate_handler.lock().unwrap() = f;
}
pub fn on_ice_candidate_error(&self, f: Option<OnIceCandidateError>) {
*self.observer.ice_candidate_error_handler.lock().unwrap() = f;
}
pub fn on_ice_connection_state_change(&self, f: Option<OnIceConnectionChange>) {
*self.observer.ice_connection_change_handler.lock().unwrap() = f;
}
pub fn on_ice_gathering_state_change(&self, f: Option<OnIceGatheringChange>) {
*self.observer.ice_gathering_change_handler.lock().unwrap() = f;
}
pub fn on_negotiation_needed(&self, f: Option<OnNegotiationNeeded>) {
*self.observer.negotiation_needed_handler.lock().unwrap() = f;
}
pub fn on_signaling_state_change(&self, f: Option<OnSignalingChange>) {
*self.observer.signaling_change_handler.lock().unwrap() = f;
}
pub fn on_track(&self, f: Option<OnTrack>) {
*self.observer.track_handler.lock().unwrap() = f;
}
}
fn create_sdp_observer() -> (
UniquePtr<sys_pc::ffi::NativeCreateSdpObserverHandle>,
oneshot::Receiver<SessionDescription>,
oneshot::Receiver<RtcError>,
) {
let (sdp_tx, sdp_rx) = oneshot::channel();
let (err_tx, err_rx) = oneshot::channel();
let wrapper = sys_jsep::CreateSdpObserverWrapper {
on_success: ManuallyDrop::new(Box::new(move |session_description| {
let _ = sdp_tx.send(SessionDescription {
handle: imp_sdp::SessionDescription {
sys_handle: session_description,
},
});
})),
on_failure: ManuallyDrop::new(Box::new(move |error| {
let _ = err_tx.send(error.into());
})),
};
(
sys_jsep::ffi::create_native_create_sdp_observer(Box::new(wrapper)),
sdp_rx,
err_rx,
)
}
#[derive(Default)]
pub struct PeerObserver {
pub connection_change_handler: Mutex<Option<OnConnectionChange>>,
pub data_channel_handler: Mutex<Option<OnDataChannel>>,
pub ice_candidate_handler: Mutex<Option<OnIceCandidate>>,
pub ice_candidate_error_handler: Mutex<Option<OnIceCandidateError>>,
pub ice_connection_change_handler: Mutex<Option<OnIceConnectionChange>>,
pub ice_gathering_change_handler: Mutex<Option<OnIceGatheringChange>>,
pub negotiation_needed_handler: Mutex<Option<OnNegotiationNeeded>>,
pub signaling_change_handler: Mutex<Option<OnSignalingChange>>,
pub track_handler: Mutex<Option<OnTrack>>,
}
impl sys_pc::PeerConnectionObserver for PeerObserver {
fn on_signaling_change(&self, new_state: sys_pc::ffi::SignalingState) {
if let Some(f) = self.signaling_change_handler.lock().unwrap().as_mut() {
f(new_state.into());
}
}
fn on_add_stream(&self, _stream: SharedPtr<webrtc_sys::media_stream::ffi::MediaStream>) {}
fn on_remove_stream(&self, _stream: SharedPtr<webrtc_sys::media_stream::ffi::MediaStream>) {}
fn on_data_channel(&self, data_channel: SharedPtr<sys_dc::ffi::DataChannel>) {
if let Some(f) = self.data_channel_handler.lock().unwrap().as_mut() {
f(DataChannel {
handle: imp_dc::DataChannel::configure(data_channel),
});
}
}
fn on_renegotiation_needed(&self) {}
fn on_negotiation_needed_event(&self, event: u32) {
if let Some(f) = self.negotiation_needed_handler.lock().unwrap().as_mut() {
f(event);
}
}
fn on_ice_connection_change(&self, _new_state: sys_pc::ffi::IceConnectionState) {}
fn on_standardized_ice_connection_change(&self, new_state: sys_pc::ffi::IceConnectionState) {
if let Some(f) = self.ice_connection_change_handler.lock().unwrap().as_mut() {
f(new_state.into());
}
}
fn on_connection_change(&self, new_state: sys_pc::ffi::PeerConnectionState) {
if let Some(f) = self.connection_change_handler.lock().unwrap().as_mut() {
f(new_state.into());
}
}
fn on_ice_gathering_change(&self, new_state: sys_pc::ffi::IceGatheringState) {
if let Some(f) = self.ice_gathering_change_handler.lock().unwrap().as_mut() {
f(new_state.into());
}
}
fn on_ice_candidate(&self, candidate: SharedPtr<sys_jsep::ffi::IceCandidate>) {
if let Some(f) = self.ice_candidate_handler.lock().unwrap().as_mut() {
f(IceCandidate {
handle: imp_ic::IceCandidate {
sys_handle: candidate,
},
});
}
}
fn on_ice_candidate_error(
&self,
address: String,
port: i32,
url: String,
error_code: i32,
error_text: String,
) {
if let Some(f) = self.ice_candidate_error_handler.lock().unwrap().as_mut() {
f(IceCandidateError {
address,
port,
url,
error_code,
error_text,
});
}
}
fn on_ice_candidates_removed(
&self,
_removed: Vec<SharedPtr<webrtc_sys::candidate::ffi::Candidate>>,
) {
}
fn on_ice_connection_receiving_change(&self, _receiving: bool) {}
fn on_ice_selected_candidate_pair_changed(
&self,
_event: sys_pc::ffi::CandidatePairChangeEvent,
) {
}
fn on_add_track(
&self,
_receiver: SharedPtr<webrtc_sys::rtp_receiver::ffi::RtpReceiver>,
_streams: Vec<SharedPtr<webrtc_sys::media_stream::ffi::MediaStream>>,
) {
}
fn on_track(&self, transceiver: SharedPtr<webrtc_sys::rtp_transceiver::ffi::RtpTransceiver>) {
if let Some(f) = self.track_handler.lock().unwrap().as_mut() {
let receiver = transceiver.receiver();
let streams = receiver.streams();
let track = receiver.track();
f(TrackEvent {
receiver: RtpReceiver {
handle: imp_rr::RtpReceiver {
sys_handle: receiver,
},
},
streams: streams
.into_iter()
.map(|s| MediaStream {
handle: imp_ms::MediaStream { sys_handle: s.ptr },
})
.collect(),
track: imp_ms::new_media_stream_track(track),
transceiver: RtpTransceiver {
handle: imp_rt::RtpTransceiver {
sys_handle: transceiver,
},
},
});
}
}
fn on_remove_track(&self, _receiver: SharedPtr<webrtc_sys::rtp_receiver::ffi::RtpReceiver>) {}
fn on_interesting_usage(&self, _usage_pattern: i32) {}
}
livekit-webrtc/src/native/peer_connection_factory.rs
+146
View File
@@ -0,0 +1,146 @@
use crate::imp::media_stream as imp_ms;
use crate::imp::peer_connection as imp_pc;
use crate::media_stream::RtcVideoTrack;
use crate::peer_connection::PeerConnection;
use crate::peer_connection_factory::{
ContinualGatheringPolicy, IceServer, IceTransportsType, RtcConfiguration,
};
use crate::rtp_parameters::RtpCapabilities;
use crate::video_source::native::NativeVideoSource;
use crate::MediaType;
use crate::RtcError;
use cxx::SharedPtr;
use std::sync::Arc;
use webrtc_sys::peer_connection as sys_pc;
use webrtc_sys::peer_connection_factory as sys_pcf;
use webrtc_sys::rtc_error as sys_err;
use webrtc_sys::webrtc as sys_webrtc;
impl From<IceServer> for sys_pcf::ffi::ICEServer {
fn from(value: IceServer) -> Self {
sys_pcf::ffi::ICEServer {
urls: value.urls,
username: value.username,
password: value.password,
}
}
}
impl From<ContinualGatheringPolicy> for sys_pcf::ffi::ContinualGatheringPolicy {
fn from(value: ContinualGatheringPolicy) -> Self {
match value {
ContinualGatheringPolicy::GatherOnce => {
sys_pcf::ffi::ContinualGatheringPolicy::GatherOnce
}
ContinualGatheringPolicy::GatherContinually => {
sys_pcf::ffi::ContinualGatheringPolicy::GatherContinually
}
}
}
}
impl From<IceTransportsType> for sys_pcf::ffi::IceTransportsType {
fn from(value: IceTransportsType) -> Self {
match value {
IceTransportsType::None => sys_pcf::ffi::IceTransportsType::None,
IceTransportsType::Relay => sys_pcf::ffi::IceTransportsType::Relay,
IceTransportsType::NoHost => sys_pcf::ffi::IceTransportsType::NoHost,
IceTransportsType::All => sys_pcf::ffi::IceTransportsType::All,
}
}
}
impl From<RtcConfiguration> for sys_pcf::ffi::RTCConfiguration {
fn from(value: RtcConfiguration) -> Self {
Self {
ice_servers: value.ice_servers.into_iter().map(Into::into).collect(),
continual_gathering_policy: value.continual_gathering_policy.into(),
ice_transport_type: value.ice_transport_type.into(),
}
}
}
#[derive(Clone)]
pub struct RTCRuntime {
pub(crate) sys_handle: SharedPtr<sys_webrtc::ffi::RTCRuntime>,
}
impl Default for RTCRuntime {
fn default() -> Self {
Self {
sys_handle: sys_webrtc::ffi::create_rtc_runtime(),
}
}
}
#[derive(Clone)]
pub struct PeerConnectionFactory {
sys_handle: SharedPtr<sys_pcf::ffi::PeerConnectionFactory>,
#[allow(unused)]
runtime: RTCRuntime,
}
impl Default for PeerConnectionFactory {
fn default() -> Self {
let runtime = RTCRuntime::default();
Self {
sys_handle: sys_pcf::ffi::create_peer_connection_factory(runtime.sys_handle.clone()),
runtime,
}
}
}
impl PeerConnectionFactory {
pub fn create_peer_connection(
&self,
config: RtcConfiguration,
) -> Result<PeerConnection, RtcError> {
let native_config = sys_pcf::ffi::create_rtc_configuration(config.into());
unsafe {
let observer = Arc::new(imp_pc::PeerObserver::default());
let native_observer = sys_pc::ffi::create_native_peer_connection_observer(
self.runtime.clone().sys_handle,
Box::new(sys_pc::PeerConnectionObserverWrapper::new(observer.clone())),
);
let res = self
.sys_handle
.create_peer_connection(native_config, &*native_observer as *const _ as *mut _);
match res {
Ok(sys_handle) => Ok(PeerConnection {
handle: imp_pc::PeerConnection::configure(
sys_handle,
observer,
native_observer,
),
}),
Err(e) => Err(sys_err::ffi::RTCError::from(e.what()).into()),
}
}
}
pub fn create_video_track(&self, label: &str, source: NativeVideoSource) -> RtcVideoTrack {
RtcVideoTrack {
handle: imp_ms::RtcVideoTrack {
sys_handle: self
.sys_handle
.create_video_track(label.to_string(), source.handle.sys_handle()),
},
}
}
pub fn get_rtp_sender_capabilities(&self, media_type: MediaType) -> RtpCapabilities {
self.sys_handle
.get_rtp_sender_capabilities(media_type.into())
.into()
}
pub fn get_rtp_receiver_capabilities(&self, media_type: MediaType) -> RtpCapabilities {
self.sys_handle
.get_rtp_receiver_capabilities(media_type.into())
.into()
}
}
livekit-webrtc/src/native/rtp_parameters.rs
+285
View File
@@ -0,0 +1,285 @@
use crate::rtp_parameters::*;
use crate::MediaType;
use webrtc_sys::rtp_parameters as sys_rp;
use webrtc_sys::webrtc as sys_webrtc;
impl From<sys_webrtc::ffi::Priority> for Priority {
fn from(value: sys_webrtc::ffi::Priority) -> Self {
match value {
sys_webrtc::ffi::Priority::VeryLow => Self::VeryLow,
sys_webrtc::ffi::Priority::Low => Self::Low,
sys_webrtc::ffi::Priority::Medium => Self::Medium,
sys_webrtc::ffi::Priority::High => Self::High,
_ => panic!("unknown Priority"),
}
}
}
impl From<sys_rp::ffi::RtpExtension> for RtpHeaderExtensionParameters {
fn from(value: sys_rp::ffi::RtpExtension) -> Self {
Self {
uri: value.uri,
id: value.id,
encrypted: value.encrypt,
}
}
}
impl From<sys_rp::ffi::RtpParameters> for RtpParameters {
fn from(value: sys_rp::ffi::RtpParameters) -> Self {
Self {
codecs: value.codecs.into_iter().map(Into::into).collect(),
header_extensions: value
.header_extensions
.into_iter()
.map(Into::into)
.collect(),
rtcp: value.rtcp.into(),
}
}
}
impl From<sys_rp::ffi::RtpCodecParameters> for RtpCodecParameters {
fn from(value: sys_rp::ffi::RtpCodecParameters) -> Self {
Self {
mime_type: value.mime_type,
payload_type: value.payload_type as u8,
clock_rate: value.has_clock_rate.then_some(value.clock_rate as u64),
channels: value.has_num_channels.then_some(value.num_channels as u16),
}
}
}
impl From<sys_rp::ffi::RtcpParameters> for RtcpParameters {
fn from(value: sys_rp::ffi::RtcpParameters) -> Self {
Self {
cname: value.cname,
reduced_size: value.reduced_size,
}
}
}
impl From<sys_rp::ffi::RtpEncodingParameters> for RtpEncodingParameters {
fn from(value: sys_rp::ffi::RtpEncodingParameters) -> Self {
Self {
active: value.active,
max_bitrate: value
.has_max_bitrate_bps
.then_some(value.max_bitrate_bps as u64),
max_framerate: value.has_max_framerate.then_some(value.max_framerate),
priority: value.network_priority.into(),
rid: value.rid,
scale_resolution_down_by: value
.has_scale_resolution_down_by
.then_some(value.scale_resolution_down_by),
}
}
}
impl From<sys_rp::ffi::RtpCodecCapability> for RtpCodecCapability {
fn from(value: sys_rp::ffi::RtpCodecCapability) -> Self {
Self {
channels: value.has_num_channels.then_some(value.num_channels as u16),
mime_type: value.mime_type,
clock_rate: value.has_clock_rate.then_some(value.clock_rate as u64),
sdp_fmtp_line: {
let parameters: Vec<String> = value
.parameters
.into_iter()
.map(|key_value| {
if !key_value.key.is_empty() {
format!("{}={}", key_value.key, key_value.value)
} else {
key_value.value
}
})
.collect();
if !parameters.is_empty() {
Some(parameters.join(";"))
} else {
None
}
},
}
}
}
impl From<sys_rp::ffi::RtpHeaderExtensionCapability> for RtpHeaderExtensionCapability {
fn from(value: sys_rp::ffi::RtpHeaderExtensionCapability) -> Self {
Self {
direction: value.direction.into(),
uri: value.uri,
}
}
}
impl From<sys_rp::ffi::RtpCapabilities> for RtpCapabilities {
fn from(value: sys_rp::ffi::RtpCapabilities) -> Self {
Self {
codecs: value.codecs.into_iter().map(Into::into).collect(),
header_extensions: value
.header_extensions
.into_iter()
.map(Into::into)
.collect(),
}
}
}
impl From<Priority> for sys_webrtc::ffi::Priority {
fn from(value: Priority) -> Self {
match value {
Priority::VeryLow => Self::VeryLow,
Priority::Low => Self::Low,
Priority::Medium => Self::Medium,
Priority::High => Self::High,
}
}
}
impl From<RtpHeaderExtensionParameters> for sys_rp::ffi::RtpExtension {
fn from(value: RtpHeaderExtensionParameters) -> Self {
Self {
uri: value.uri,
id: value.id,
encrypt: value.encrypted,
}
}
}
impl From<RtpParameters> for sys_rp::ffi::RtpParameters {
fn from(value: RtpParameters) -> Self {
Self {
codecs: value.codecs.into_iter().map(Into::into).collect(),
header_extensions: value
.header_extensions
.into_iter()
.map(Into::into)
.collect(),
encodings: Vec::new(),
rtcp: value.rtcp.into(),
transaction_id: "".to_string(),
mid: "".to_string(),
has_degradation_preference: false,
degradation_preference: sys_rp::ffi::DegradationPreference::Balanced,
}
}
}
impl From<RtpCodecParameters> for sys_rp::ffi::RtpCodecParameters {
fn from(value: RtpCodecParameters) -> Self {
Self {
payload_type: value.payload_type as i32,
mime_type: value.mime_type,
has_clock_rate: value.clock_rate.is_some(),
clock_rate: value.clock_rate.unwrap_or_default() as i32,
has_num_channels: value.channels.is_some(),
num_channels: value.channels.unwrap_or_default() as i32,
name: "".to_string(),
kind: sys_rp::ffi::MediaType::Audio,
has_max_ptime: false,
max_ptime: 0,
has_ptime: false,
ptime: 0,
rtcp_feedback: Vec::new(),
parameters: Vec::new(),
}
}
}
impl From<RtcpParameters> for sys_rp::ffi::RtcpParameters {
fn from(value: RtcpParameters) -> Self {
Self {
cname: value.cname,
reduced_size: value.reduced_size,
has_ssrc: false,
ssrc: 0,
mux: false,
}
}
}
impl From<RtpEncodingParameters> for sys_rp::ffi::RtpEncodingParameters {
fn from(value: RtpEncodingParameters) -> Self {
Self {
active: value.active,
has_max_bitrate_bps: value.max_bitrate.is_some(),
max_bitrate_bps: value.max_bitrate.unwrap_or_default() as i32,
has_max_framerate: value.max_framerate.is_some(),
max_framerate: value.max_framerate.unwrap_or_default(),
network_priority: value.priority.into(),
rid: value.rid,
has_scale_resolution_down_by: value.scale_resolution_down_by.is_some(),
scale_resolution_down_by: value.scale_resolution_down_by.unwrap_or_default(),
adaptive_ptime: false,
bitrate_priority: sys_rp::DEFAULT_BITRATE_PRIORITY,
has_min_bitrate_bps: false,
min_bitrate_bps: 0,
has_num_temporal_layers: false,
num_temporal_layers: 0,
has_scalability_mode: false,
scalability_mode: "".to_string(),
has_ssrc: false,
ssrc: 0,
}
}
}
impl From<RtpCodecCapability> for sys_rp::ffi::RtpCodecCapability {
fn from(value: RtpCodecCapability) -> Self {
let mime_type: Vec<&str> = value.mime_type.split('/').collect();
let kind = match mime_type[0] {
"audio" => sys_webrtc::ffi::MediaType::Audio,
"video" => sys_webrtc::ffi::MediaType::Video,
_ => panic!("invalid media type"),
};
let name = mime_type[1].to_string();
Self {
name,
kind,
has_clock_rate: value.clock_rate.is_some(),
clock_rate: value.clock_rate.unwrap_or_default() as i32,
has_num_channels: value.channels.is_some(),
num_channels: value.channels.unwrap_or_default() as i32,
parameters: {
value
.sdp_fmtp_line
.map(|sdp_fmtp_line| {
sdp_fmtp_line
.split(';')
.map(|v| {
let key_value: Vec<&str> = v.split('=').collect();
if key_value.len() == 2 {
sys_rp::ffi::StringKeyValue {
key: key_value[0].to_string(),
value: key_value[1].to_string(),
}
} else {
sys_rp::ffi::StringKeyValue {
key: "".to_string(),
value: key_value[0].to_string(),
}
}
})
.collect()
})
.unwrap_or_default()
},
// Ignore
mime_type: String::default(), // !!
has_preferred_payload_type: false,
preferred_payload_type: 0,
has_max_ptime: false,
max_ptime: 0,
has_ptime: false,
ptime: 0,
rtcp_feedback: Vec::default(),
options: Vec::default(),
max_temporal_layer_extensions: 0,
max_spatial_layer_extensions: 0,
svc_multi_stream_support: false,
}
}
}
livekit-webrtc/src/native/rtp_receiver.rs
+24
View File
@@ -0,0 +1,24 @@
use super::media_stream::new_media_stream_track;
use crate::{media_stream::MediaStreamTrack, rtp_parameters::RtpParameters};
use cxx::SharedPtr;
use webrtc_sys::rtp_receiver as sys_rr;
#[derive(Clone)]
pub struct RtpReceiver {
pub(crate) sys_handle: SharedPtr<sys_rr::ffi::RtpReceiver>,
}
impl RtpReceiver {
pub fn track(&self) -> Option<MediaStreamTrack> {
let track_handle = self.sys_handle.track();
if track_handle.is_null() {
return None;
}
Some(new_media_stream_track(track_handle))
}
pub fn parameters(&self) -> RtpParameters {
self.sys_handle.get_parameters().into()
}
}
livekit-webrtc/src/native/rtp_sender.rs
+46
View File
@@ -0,0 +1,46 @@
use super::media_stream::new_media_stream_track;
use crate::{
media_stream::MediaStreamTrack, rtp_parameters::RtpParameters, RtcError, RtcErrorType,
};
use cxx::SharedPtr;
use webrtc_sys::{rtc_error::ffi::RTCError, rtp_sender as sys_rs};
#[derive(Clone)]
pub struct RtpSender {
pub(crate) sys_handle: SharedPtr<sys_rs::ffi::RtpSender>,
}
impl RtpSender {
pub fn track(&self) -> Option<MediaStreamTrack> {
let track_handle = self.sys_handle.track();
if track_handle.is_null() {
return None;
}
Some(new_media_stream_track(track_handle))
}
pub fn set_track(&self, track: Option<MediaStreamTrack>) -> Result<(), RtcError> {
if !self
.sys_handle
.set_track(track.map_or(SharedPtr::null(), |t| t.sys_handle()))
{
return Err(RtcError {
error_type: RtcErrorType::InvalidState,
message: "Failed to set track".to_string(),
});
}
Ok(())
}
pub fn parameters(&self) -> RtpParameters {
self.sys_handle.get_parameters().into()
}
pub fn set_parameters(&self, parameters: RtpParameters) -> Result<(), RtcError> {
self.sys_handle
.set_parameters(parameters.into())
.map_err(|e| unsafe { RTCError::from(e.what()).into() })
}
}
livekit-webrtc/src/native/rtp_transceiver.rs
+94
View File
@@ -0,0 +1,94 @@
use crate::imp::rtp_receiver::RtpReceiver;
use crate::imp::rtp_sender::RtpSender;
use crate::rtp_parameters::RtpCodecCapability;
use crate::rtp_receiver;
use crate::rtp_sender;
use crate::rtp_transceiver::RtpTransceiverDirection;
use crate::rtp_transceiver::RtpTransceiverInit;
use crate::MediaType;
use crate::RtcError;
use cxx::SharedPtr;
use webrtc_sys::rtc_error as sys_err;
use webrtc_sys::rtp_transceiver as sys_rt;
use webrtc_sys::webrtc as sys_webrtc;
impl From<sys_webrtc::ffi::RtpTransceiverDirection> for RtpTransceiverDirection {
fn from(value: sys_webrtc::ffi::RtpTransceiverDirection) -> Self {
match value {
sys_webrtc::ffi::RtpTransceiverDirection::SendRecv => Self::SendRecv,
sys_webrtc::ffi::RtpTransceiverDirection::SendOnly => Self::SendOnly,
sys_webrtc::ffi::RtpTransceiverDirection::RecvOnly => Self::RecvOnly,
sys_webrtc::ffi::RtpTransceiverDirection::Inactive => Self::Inactive,
_ => panic!("unknown RtpTransceiverDirection"),
}
}
}
impl From<RtpTransceiverDirection> for sys_webrtc::ffi::RtpTransceiverDirection {
fn from(value: RtpTransceiverDirection) -> Self {
match value {
RtpTransceiverDirection::SendRecv => Self::SendRecv,
RtpTransceiverDirection::SendOnly => Self::SendOnly,
RtpTransceiverDirection::RecvOnly => Self::RecvOnly,
RtpTransceiverDirection::Inactive => Self::Inactive,
_ => panic!("unknown RtpTransceiverDirection"),
}
}
}
impl From<RtpTransceiverInit> for sys_rt::ffi::RtpTransceiverInit {
fn from(value: RtpTransceiverInit) -> Self {
Self {
direction: value.direction.into(),
stream_ids: value.stream_ids,
send_encodings: value.send_encodings.into_iter().map(Into::into).collect(),
}
}
}
#[derive(Clone)]
pub struct RtpTransceiver {
pub(crate) sys_handle: SharedPtr<sys_rt::ffi::RtpTransceiver>,
}
impl RtpTransceiver {
pub fn mid(&self) -> Option<String> {
self.sys_handle.mid().ok()
}
pub fn current_direction(&self) -> Option<RtpTransceiverDirection> {
self.sys_handle.current_direction().ok().map(Into::into)
}
pub fn direction(&self) -> RtpTransceiverDirection {
self.sys_handle.direction().into()
}
pub fn sender(&self) -> rtp_sender::RtpSender {
rtp_sender::RtpSender {
handle: RtpSender {
sys_handle: self.sys_handle.sender(),
},
}
}
pub fn receiver(&self) -> rtp_receiver::RtpReceiver {
rtp_receiver::RtpReceiver {
handle: RtpReceiver {
sys_handle: self.sys_handle.receiver(),
},
}
}
pub fn set_codec_preferences(&self, codecs: Vec<RtpCodecCapability>) -> Result<(), RtcError> {
self.sys_handle
.set_codec_preferences(codecs.into_iter().map(Into::into).collect())
.map_err(|e| unsafe { sys_err::ffi::RTCError::from(e.what()).into() })
}
pub fn stop(&self) -> Result<(), RtcError> {
self.sys_handle
.stop_standard()
.map_err(|e| unsafe { sys_err::ffi::RTCError::from(e.what()).into() })
}
}
livekit-webrtc/src/native/session_description.rs
+72
View File
@@ -0,0 +1,72 @@
use crate::session_description::{self, SdpParseError, SdpType};
use cxx::UniquePtr;
use webrtc_sys::jsep as sys_jsep;
impl From<sys_jsep::ffi::SdpType> for SdpType {
fn from(sdp_type: sys_jsep::ffi::SdpType) -> Self {
match sdp_type {
sys_jsep::ffi::SdpType::Offer => SdpType::Offer,
sys_jsep::ffi::SdpType::PrAnswer => SdpType::PrAnswer,
sys_jsep::ffi::SdpType::Answer => SdpType::Answer,
sys_jsep::ffi::SdpType::Rollback => SdpType::Rollback,
_ => panic!("unknown SdpType"),
}
}
}
impl From<SdpType> for sys_jsep::ffi::SdpType {
fn from(sdp_type: SdpType) -> Self {
match sdp_type {
SdpType::Offer => sys_jsep::ffi::SdpType::Offer,
SdpType::PrAnswer => sys_jsep::ffi::SdpType::PrAnswer,
SdpType::Answer => sys_jsep::ffi::SdpType::Answer,
SdpType::Rollback => sys_jsep::ffi::SdpType::Rollback,
}
}
}
impl From<sys_jsep::ffi::SdpParseError> for SdpParseError {
fn from(e: sys_jsep::ffi::SdpParseError) -> Self {
Self {
line: e.line,
description: e.description,
}
}
}
pub struct SessionDescription {
pub(crate) sys_handle: UniquePtr<sys_jsep::ffi::SessionDescription>,
}
impl SessionDescription {
pub fn parse(
sdp: &str,
sdp_type: SdpType,
) -> Result<session_description::SessionDescription, SdpParseError> {
let res = sys_jsep::ffi::create_session_description(sdp_type.into(), sdp.to_owned());
match res {
Ok(sys_handle) => Ok(session_description::SessionDescription {
handle: SessionDescription { sys_handle },
}),
Err(e) => Err(unsafe { sys_jsep::ffi::SdpParseError::from(e.what()).into() }),
}
}
pub fn sdp_type(&self) -> SdpType {
self.sys_handle.sdp_type().into()
}
}
impl ToString for SessionDescription {
fn to_string(&self) -> String {
self.sys_handle.stringify()
}
}
impl Clone for SessionDescription {
fn clone(&self) -> Self {
SessionDescription {
sys_handle: self.sys_handle.clone(),
}
}
}
livekit-webrtc/src/native/video_frame.rs
+764
View File
@@ -0,0 +1,764 @@
use super::yuv_helper::{self, ConvertError};
use crate::video_frame::VideoRotation;
use crate::video_frame::{self as vf, VideoFormatType};
use cxx::UniquePtr;
use std::slice;
use webrtc_sys::video_frame as vf_sys;
use webrtc_sys::video_frame_buffer as vfb_sys;
/// We don't use vf::VideoFrameBuffer trait for the types inside this module to avoid confusion
/// because irectly using platform specific types is not valid (e.g user callback)
/// All the types inside this module are only used internally. For public types, see the top level video_frame.rs
pub fn new_video_frame_buffer(
mut sys_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>,
) -> Box<dyn vf::VideoFrameBuffer + Send + Sync> {
unsafe {
match sys_handle.buffer_type().into() {
vfb_sys::ffi::VideoFrameBufferType::Native => Box::new(vf::native::NativeBuffer {
handle: NativeBuffer { sys_handle },
}),
vfb_sys::ffi::VideoFrameBufferType::I420 => Box::new(vf::I420Buffer {
handle: I420Buffer {
sys_handle: sys_handle.pin_mut().get_i420(),
},
}),
vfb_sys::ffi::VideoFrameBufferType::I420A => Box::new(vf::I420ABuffer {
handle: I420ABuffer {
sys_handle: sys_handle.pin_mut().get_i420a(),
},
}),
vfb_sys::ffi::VideoFrameBufferType::I422 => Box::new(vf::I422Buffer {
handle: I422Buffer {
sys_handle: sys_handle.pin_mut().get_i422(),
},
}),
vfb_sys::ffi::VideoFrameBufferType::I444 => Box::new(vf::I444Buffer {
handle: I444Buffer {
sys_handle: sys_handle.pin_mut().get_i444(),
},
}),
vfb_sys::ffi::VideoFrameBufferType::I010 => Box::new(vf::I010Buffer {
handle: I010Buffer {
sys_handle: sys_handle.pin_mut().get_i010(),
},
}),
vfb_sys::ffi::VideoFrameBufferType::NV12 => Box::new(vf::NV12Buffer {
handle: NV12Buffer {
sys_handle: sys_handle.pin_mut().get_nv12(),
},
}),
_ => unreachable!(),
}
}
}
impl From<vf_sys::ffi::VideoRotation> for VideoRotation {
fn from(rotation: vf_sys::ffi::VideoRotation) -> Self {
match rotation {
vf_sys::ffi::VideoRotation::VideoRotation0 => Self::VideoRotation0,
vf_sys::ffi::VideoRotation::VideoRotation90 => Self::VideoRotation90,
vf_sys::ffi::VideoRotation::VideoRotation180 => Self::VideoRotation180,
vf_sys::ffi::VideoRotation::VideoRotation270 => Self::VideoRotation270,
_ => panic!("invalid VideoRotation"),
}
}
}
impl From<VideoRotation> for vf_sys::ffi::VideoRotation {
fn from(rotation: VideoRotation) -> Self {
match rotation {
VideoRotation::VideoRotation0 => Self::VideoRotation0,
VideoRotation::VideoRotation90 => Self::VideoRotation90,
VideoRotation::VideoRotation180 => Self::VideoRotation180,
VideoRotation::VideoRotation270 => Self::VideoRotation270,
}
}
}
macro_rules! recursive_cast {
($ptr:expr $(, $fnc:ident)*) => {
{
let ptr = $ptr;
$(
let ptr = vfb_sys::ffi::$fnc(ptr);
)*
ptr
}
};
}
pub struct NativeBuffer {
sys_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>,
}
pub struct I420Buffer {
sys_handle: UniquePtr<vfb_sys::ffi::I420Buffer>,
}
pub struct I420ABuffer {
sys_handle: UniquePtr<vfb_sys::ffi::I420ABuffer>,
}
pub struct I422Buffer {
sys_handle: UniquePtr<vfb_sys::ffi::I422Buffer>,
}
pub struct I444Buffer {
sys_handle: UniquePtr<vfb_sys::ffi::I444Buffer>,
}
pub struct I010Buffer {
sys_handle: UniquePtr<vfb_sys::ffi::I010Buffer>,
}
pub struct NV12Buffer {
sys_handle: UniquePtr<vfb_sys::ffi::NV12Buffer>,
}
macro_rules! impl_to_argb {
(I420Buffer [$($variant:ident: $fnc:ident),+], $format:ident, $self:ident, $dst:ident, $dst_stride:ident, $dst_width:ident, $dst_height:ident) => {
match $format {
$(
VideoFormatType::$variant => {
let (data_y, data_u, data_v) = $self.data();
yuv_helper::$fnc(
data_y,
$self.stride_y(),
data_u,
$self.stride_u(),
data_v,
$self.stride_v(),
$dst,
$dst_stride,
$dst_width,
$dst_height,
)
}
)+
}
};
(I420ABuffer) => {
todo!();
}
}
#[allow(unused_unsafe)]
impl NativeBuffer {
pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
&*self.sys_handle
}
pub fn width(&self) -> i32 {
self.sys_handle.width()
}
pub fn height(&self) -> i32 {
self.sys_handle.height()
}
pub fn to_i420(&self) -> I420Buffer {
I420Buffer {
sys_handle: unsafe { self.sys_handle.to_i420() },
}
}
pub fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: i32,
dst_width: i32,
dst_height: i32,
) -> Result<(), ConvertError> {
self.to_i420()
.to_argb(format, dst, dst_stride, dst_width, dst_height)
}
}
impl I420Buffer {
pub fn new(width: u32, height: u32) -> vf::I420Buffer {
vf::I420Buffer {
handle: I420Buffer {
sys_handle: vfb_sys::ffi::new_i420_buffer(
width.try_into().unwrap(),
height.try_into().unwrap(),
),
},
}
}
pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
unsafe { &*recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb) }
}
pub fn width(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).width()
}
}
pub fn height(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).height()
}
}
pub fn chroma_width(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
(*ptr).chroma_width()
}
}
pub fn chroma_height(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
(*ptr).chroma_height()
}
}
pub fn stride_y(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
(*ptr).stride_y()
}
}
pub fn stride_u(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
(*ptr).stride_u()
}
}
pub fn stride_v(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
(*ptr).stride_v()
}
}
pub fn to_i420(&self) -> I420Buffer {
I420Buffer {
sys_handle: unsafe {
// We make a copy of the buffer because internally, when calling ToI420()
// if the buffer is of type I420, libwebrtc will reuse the same underlying pointer
// for the new created type
let copy = vfb_sys::ffi::copy_i420_buffer(&self.sys_handle);
let ptr = recursive_cast!(&*copy, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).to_i420()
},
}
}
pub fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: i32,
dst_width: i32,
dst_height: i32,
) -> Result<(), ConvertError> {
impl_to_argb!(
I420Buffer
[
ARGB: i420_to_argb,
BGRA: i420_to_bgra,
ABGR: i420_to_abgr,
RGBA: i420_to_rgba
],
format, self, dst, dst_stride, dst_width, dst_height
)
}
pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8);
let chroma_height = (self.height() + 1) / 2;
(
slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * chroma_height) as usize),
slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * chroma_height) as usize),
)
}
}
}
impl I420ABuffer {
pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
unsafe { &*recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb) }
}
pub fn width(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).width()
}
}
pub fn height(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).height()
}
}
pub fn chroma_width(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
(*ptr).chroma_width()
}
}
pub fn chroma_height(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
(*ptr).chroma_height()
}
}
pub fn stride_y(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
(*ptr).stride_y()
}
}
pub fn stride_u(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
(*ptr).stride_u()
}
}
pub fn stride_v(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
(*ptr).stride_v()
}
}
pub fn stride_a(&self) -> i32 {
self.sys_handle.stride_a()
}
pub fn to_i420(&self) -> I420Buffer {
I420Buffer {
sys_handle: unsafe {
let ptr =
recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).to_i420()
},
}
}
pub fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: i32,
dst_width: i32,
dst_height: i32,
) -> Result<(), ConvertError> {
self.to_i420()
.to_argb(format, dst, dst_stride, dst_width, dst_height)
}
pub fn data(&self) -> (&[u8], &[u8], &[u8], Option<&[u8]>) {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8);
let chroma_height = (self.height() + 1) / 2;
let data_a = self.sys_handle.data_a();
let has_data_a = !data_a.is_null();
(
slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * chroma_height) as usize),
slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * chroma_height) as usize),
has_data_a.then_some(slice::from_raw_parts(
data_a,
(self.stride_a() * self.height()) as usize,
)),
)
}
}
}
impl I422Buffer {
pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
unsafe { &*recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb) }
}
pub fn width(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).width()
}
}
pub fn height(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).height()
}
}
pub fn chroma_width(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
(*ptr).chroma_width()
}
}
pub fn chroma_height(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
(*ptr).chroma_height()
}
}
pub fn stride_y(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
(*ptr).stride_y()
}
}
pub fn stride_u(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
(*ptr).stride_u()
}
}
pub fn stride_v(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
(*ptr).stride_v()
}
}
pub fn to_i420(&self) -> I420Buffer {
I420Buffer {
sys_handle: unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).to_i420()
},
}
}
pub fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: i32,
dst_width: i32,
dst_height: i32,
) -> Result<(), ConvertError> {
self.to_i420()
.to_argb(format, dst, dst_stride, dst_width, dst_height)
}
pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8);
(
slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * self.height()) as usize),
slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * self.height()) as usize),
)
}
}
}
impl I444Buffer {
pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
unsafe { &*recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb) }
}
pub fn width(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).width()
}
}
pub fn height(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).height()
}
}
pub fn chroma_width(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
(*ptr).chroma_width()
}
}
pub fn chroma_height(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
(*ptr).chroma_height()
}
}
pub fn stride_y(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
(*ptr).stride_y()
}
}
pub fn stride_u(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
(*ptr).stride_u()
}
}
pub fn stride_v(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
(*ptr).stride_v()
}
}
pub fn to_i420(&self) -> I420Buffer {
I420Buffer {
sys_handle: unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).to_i420()
},
}
}
pub fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: i32,
dst_width: i32,
dst_height: i32,
) -> Result<(), ConvertError> {
self.to_i420()
.to_argb(format, dst, dst_stride, dst_width, dst_height)
}
pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8);
(
slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * self.height()) as usize),
slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * self.height()) as usize),
)
}
}
}
impl I010Buffer {
pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
unsafe { &*recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb) }
}
pub fn width(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb);
(*ptr).width()
}
}
pub fn height(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb);
(*ptr).height()
}
}
pub fn chroma_width(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
(*ptr).chroma_width()
}
}
pub fn chroma_height(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
(*ptr).chroma_height()
}
}
pub fn stride_y(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
(*ptr).stride_y()
}
}
pub fn stride_u(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
(*ptr).stride_u()
}
}
pub fn stride_v(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
(*ptr).stride_v()
}
}
pub fn to_i420(&self) -> I420Buffer {
I420Buffer {
sys_handle: unsafe {
let ptr =
recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb);
(*ptr).to_i420()
},
}
}
pub fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: i32,
dst_width: i32,
dst_height: i32,
) -> Result<(), ConvertError> {
self.to_i420()
.to_argb(format, dst, dst_stride, dst_width, dst_height)
}
pub fn data(&self) -> (&[u16], &[u16], &[u16]) {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b);
let chroma_height = (self.height() + 1) / 2;
(
slice::from_raw_parts(
(*ptr).data_y(),
(self.stride_y() * self.height()) as usize / 2,
),
slice::from_raw_parts(
(*ptr).data_u(),
(self.stride_u() * chroma_height) as usize / 2,
),
slice::from_raw_parts(
(*ptr).data_v(),
(self.stride_v() * chroma_height) as usize / 2,
),
)
}
}
}
impl NV12Buffer {
pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
unsafe {
&*recursive_cast!(
&*self.sys_handle,
nv12_to_biyuv8,
biyuv8_to_biyuv,
biyuv_to_vfb
)
}
}
pub fn width(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(
&*self.sys_handle,
nv12_to_biyuv8,
biyuv8_to_biyuv,
biyuv_to_vfb
);
(*ptr).width()
}
}
pub fn height(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(
&*self.sys_handle,
nv12_to_biyuv8,
biyuv8_to_biyuv,
biyuv_to_vfb
);
(*ptr).height()
}
}
pub fn chroma_width(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv);
(*ptr).chroma_width()
}
}
pub fn chroma_height(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv);
(*ptr).chroma_height()
}
}
pub fn stride_y(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv);
(*ptr).stride_y()
}
}
pub fn stride_uv(&self) -> i32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv);
(*ptr).stride_uv()
}
}
pub fn to_i420(&self) -> I420Buffer {
I420Buffer {
sys_handle: unsafe {
let ptr = recursive_cast!(
&*self.sys_handle,
nv12_to_biyuv8,
biyuv8_to_biyuv,
biyuv_to_vfb
);
(*ptr).to_i420()
},
}
}
pub fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: i32,
dst_width: i32,
dst_height: i32,
) -> Result<(), ConvertError> {
self.to_i420()
.to_argb(format, dst, dst_stride, dst_width, dst_height)
}
pub fn data(&self) -> (&[u8], &[u8]) {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8);
let chroma_height = (self.height() + 1) / 2;
(
slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
slice::from_raw_parts(
(*ptr).data_uv(),
(self.stride_uv() * chroma_height) as usize,
),
)
}
}
}
livekit-webrtc/src/native/video_source.rs
+34
View File
@@ -0,0 +1,34 @@
use crate::video_frame::{VideoFrame, VideoFrameBuffer};
use cxx::SharedPtr;
use webrtc_sys::media_stream as ms_sys;
use webrtc_sys::video_frame as vf_sys;
#[derive(Clone)]
pub struct NativeVideoSource {
sys_handle: SharedPtr<ms_sys::ffi::AdaptedVideoTrackSource>,
}
impl Default for NativeVideoSource {
fn default() -> Self {
Self {
sys_handle: ms_sys::ffi::new_adapted_video_track_source(),
}
}
}
impl NativeVideoSource {
pub fn sys_handle(&self) -> SharedPtr<ms_sys::ffi::AdaptedVideoTrackSource> {
self.sys_handle.clone()
}
pub fn capture_frame<T: VideoFrameBuffer>(&self, frame: &VideoFrame<T>) {
let mut builder = vf_sys::ffi::new_video_frame_builder();
builder.pin_mut().set_rotation(frame.rotation.into());
builder
.pin_mut()
.set_video_frame_buffer(frame.buffer.sys_handle());
let frame = builder.pin_mut().build();
self.sys_handle.on_captured_frame(&frame);
}
}
livekit-webrtc/src/native/video_stream.rs
+84
View File
@@ -0,0 +1,84 @@
use super::video_frame::new_video_frame_buffer;
use crate::media_stream::RtcVideoTrack;
use crate::video_frame::{BoxVideoFrame, VideoFrame};
use cxx::UniquePtr;
use futures::stream::Stream;
use std::pin::Pin;
use std::task::{Context, Poll};
use tokio::sync::mpsc;
use webrtc_sys::media_stream as sys_ms;
pub struct NativeVideoStream {
native_observer: UniquePtr<sys_ms::ffi::NativeVideoFrameSink>,
_observer: Box<VideoTrackObserver>,
video_track: RtcVideoTrack,
frame_rx: mpsc::UnboundedReceiver<BoxVideoFrame>,
}
impl NativeVideoStream {
pub fn new(video_track: RtcVideoTrack) -> Self {
let (frame_tx, frame_rx) = mpsc::unbounded_channel();
let mut observer = Box::new(VideoTrackObserver { frame_tx });
let mut native_observer = unsafe {
sys_ms::ffi::new_native_video_frame_sink(Box::new(sys_ms::VideoFrameSinkWrapper::new(
&mut *observer,
)))
};
unsafe {
sys_ms::ffi::media_to_video(video_track.sys_handle())
.add_sink(native_observer.pin_mut());
}
Self {
native_observer,
_observer: observer,
video_track,
frame_rx,
}
}
pub fn track(&self) -> RtcVideoTrack {
self.video_track.clone()
}
pub fn close(&mut self) {
self.frame_rx.close();
unsafe {
sys_ms::ffi::media_to_video(self.video_track.sys_handle())
.remove_sink(self.native_observer.pin_mut());
}
}
}
impl Drop for NativeVideoStream {
fn drop(&mut self) {
self.close();
}
}
impl Stream for NativeVideoStream {
type Item = BoxVideoFrame;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
self.frame_rx.poll_recv(cx)
}
}
struct VideoTrackObserver {
frame_tx: mpsc::UnboundedSender<BoxVideoFrame>,
}
impl sys_ms::VideoFrameSink for VideoTrackObserver {
fn on_frame(&self, frame: UniquePtr<webrtc_sys::video_frame::ffi::VideoFrame>) {
let _ = self.frame_tx.send(VideoFrame {
rotation: frame.rotation().into(),
timestamp: frame.timestamp_us(),
buffer: new_video_frame_buffer(unsafe { frame.video_frame_buffer() }),
});
}
fn on_discarded_frame(&self) {}
fn on_constraints_changed(&self, _constraints: sys_ms::ffi::VideoTrackSourceConstraints) {}
}
livekit-webrtc/src/native/yuv_helper.rs
+183
View File
@@ -0,0 +1,183 @@
use thiserror::Error;
use webrtc_sys::yuv_helper as yuv_sys;
#[derive(Error, Debug)]
pub enum ConvertError {
#[error("conversion failed: {0}")]
Convert(&'static str),
}
#[inline]
fn argb_assert_safety(
src: &[u8],
src_stride: i32,
_width: i32,
height: i32,
) -> Result<(), ConvertError> {
let min = (src_stride * height) as usize;
if src.len() < min {
return Err(ConvertError::Convert("dst isn't large enough"));
}
Ok(())
}
#[inline]
fn i420_assert_safety(
src_y: &[u8],
src_stride_y: i32,
src_u: &[u8],
src_stride_u: i32,
src_v: &[u8],
src_stride_v: i32,
_width: i32,
height: i32,
) -> Result<(), ConvertError> {
let chroma_height = (height + 1) / 2;
let min_y = (src_stride_y * height) as usize;
let min_u = (src_stride_u * chroma_height) as usize;
let min_v = (src_stride_v * chroma_height) as usize;
if src_y.len() < min_y {
return Err(ConvertError::Convert("src_y isn't large enough"));
}
if src_u.len() < min_u {
return Err(ConvertError::Convert("src_u isn't large enough"));
}
if src_v.len() < min_v {
return Err(ConvertError::Convert("src_v isn't large enough"));
}
Ok(())
}
macro_rules! i420_to_x {
($x:ident) => {
pub fn $x(
src_y: &[u8],
src_stride_y: i32,
src_u: &[u8],
src_stride_u: i32,
src_v: &[u8],
src_stride_v: i32,
dst: &mut [u8],
dst_stride: i32,
width: i32,
height: i32,
) -> Result<(), ConvertError> {
argb_assert_safety(dst, dst_stride, width, height)?;
i420_assert_safety(
src_y,
src_stride_y,
src_u,
src_stride_u,
src_v,
src_stride_v,
width,
height,
)?;
unsafe {
yuv_sys::ffi::$x(
src_y.as_ptr(),
src_stride_y,
src_u.as_ptr(),
src_stride_u,
src_v.as_ptr(),
src_stride_v,
dst.as_mut_ptr(),
dst_stride,
width,
height,
)
.unwrap();
}
Ok(())
}
};
}
macro_rules! x_to_i420 {
($x:ident) => {
pub fn $x(
src_argb: &[u8],
src_stride_argb: i32,
dst_y: &mut [u8],
dst_stride_y: i32,
dst_u: &mut [u8],
dst_stride_u: i32,
dst_v: &mut [u8],
dst_stride_v: i32,
width: i32,
height: i32,
) -> Result<(), ConvertError> {
argb_assert_safety(src_argb, src_stride_argb, width, height)?;
i420_assert_safety(
dst_y,
dst_stride_y,
dst_u,
dst_stride_u,
dst_v,
dst_stride_v,
width,
height,
)?;
unsafe {
yuv_sys::ffi::$x(
src_argb.as_ptr(),
src_stride_argb,
dst_y.as_mut_ptr(),
dst_stride_y,
dst_u.as_mut_ptr(),
dst_stride_u,
dst_v.as_mut_ptr(),
dst_stride_v,
width,
height,
)
.unwrap();
}
Ok(())
}
};
}
pub fn argb_to_rgb24(
src_argb: &[u8],
src_stride_argb: i32,
dst_rgb24: &mut [u8],
dst_stride_rgb24: i32,
width: i32,
height: i32,
) -> Result<(), ConvertError> {
argb_assert_safety(src_argb, src_stride_argb, width, height)?;
argb_assert_safety(dst_rgb24, dst_stride_rgb24, width, height)?;
unsafe {
yuv_sys::ffi::argb_to_rgb24(
src_argb.as_ptr(),
src_stride_argb,
dst_rgb24.as_mut_ptr(),
dst_stride_rgb24,
width,
height,
)
.unwrap();
}
Ok(())
}
x_to_i420!(argb_to_i420);
x_to_i420!(abgr_to_i420);
i420_to_x!(i420_to_argb);
i420_to_x!(i420_to_bgra);
i420_to_x!(i420_to_abgr);
i420_to_x!(i420_to_rgba);
livekit-webrtc/src/peer_connection.rs
+165 -662
View File
@@ -1,737 +1,240 @@
use crate::prelude::*;
use cxx::{SharedPtr, UniquePtr};
use log::trace;
use std::fmt::{Debug, Formatter};
use std::mem::ManuallyDrop;
use std::sync::{Arc, Mutex};
use tokio::sync::{mpsc, oneshot};
use std::fmt::Debug;
use webrtc_sys::candidate as sys_ca;
use webrtc_sys::data_channel as sys_dc;
use webrtc_sys::jsep as sys_jsep;
use webrtc_sys::media_stream as sys_ms;
use webrtc_sys::peer_connection as sys_pc;
use webrtc_sys::rtp_receiver as sys_rr;
use webrtc_sys::rtp_sender as sys_rs;
use webrtc_sys::rtp_transceiver as sys_rt;
use crate::data_channel::{DataChannel, DataChannelInit};
use crate::ice_candidate::IceCandidate;
use crate::imp::peer_connection as imp_pc;
use crate::media_stream::{MediaStream, MediaStreamTrack};
use crate::rtp_receiver::RtpReceiver;
use crate::rtp_sender::RtpSender;
use crate::rtp_transceiver::{RtpTransceiver, RtpTransceiverInit};
use crate::session_description::SessionDescription;
use crate::{MediaType, RtcError};
pub use webrtc_sys::peer_connection::ffi::IceConnectionState;
pub use webrtc_sys::peer_connection::ffi::IceGatheringState;
pub use webrtc_sys::peer_connection::ffi::PeerConnectionState;
pub use webrtc_sys::peer_connection::ffi::RTCOfferAnswerOptions;
pub use webrtc_sys::peer_connection::ffi::SignalingState;
pub struct PeerConnection {
cxx_handle: UniquePtr<sys_pc::ffi::PeerConnection>,
observer: Box<InternalObserver>,
// Keep alive for C++
#[allow(unused)]
native_observer: UniquePtr<sys_pc::ffi::NativePeerConnectionObserver>,
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum PeerConnectionState {
New,
Connecting,
Connected,
Disconnected,
Failed,
Closed,
}
impl Debug for PeerConnection {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
f.debug_struct("PeerConnection")
.field("signaling_state", &self.signaling_state())
.field("ice_connection_state", &self.ice_connection_state())
.field("ice_gathering_state", &self.ice_gathering_state())
.finish()
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum IceConnectionState {
New,
Checking,
Connected,
Completed,
Failed,
Disconnected,
Closed,
Max,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum IceGatheringState {
New,
Gathering,
Complete,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum SignalingState {
Stable,
HaveLocalOffer,
HaveLocalPrAnswer,
HaveRemoteOffer,
HaveRemotePrAnswer,
Closed,
}
#[derive(Debug, Clone, Default)]
pub struct OfferOptions {
pub ice_restart: bool,
pub offer_to_receive_audio: bool,
pub offer_to_receive_video: bool,
}
#[derive(Debug, Clone, Default)]
pub struct AnswerOptions {}
#[derive(Debug, Clone)]
pub struct IceCandidateError {
pub address: String,
pub port: i32,
pub url: String,
pub error_code: i32,
pub error_text: String,
}
#[derive(Debug, Clone)]
pub struct TrackEvent {
pub receiver: RtpReceiver,
pub streams: Vec<MediaStream>,
pub track: MediaStreamTrack,
pub transceiver: RtpTransceiver,
}
pub type OnConnectionChange = Box<dyn FnMut(PeerConnectionState) + Send + Sync>;
pub type OnDataChannel = Box<dyn FnMut(DataChannel) + Send + Sync>;
pub type OnIceCandidate = Box<dyn FnMut(IceCandidate) + Send + Sync>;
pub type OnIceCandidateError = Box<dyn FnMut(IceCandidateError) + Send + Sync>;
pub type OnIceConnectionChange = Box<dyn FnMut(IceConnectionState) + Send + Sync>;
pub type OnIceGatheringChange = Box<dyn FnMut(IceGatheringState) + Send + Sync>;
pub type OnNegotiationNeeded = Box<dyn FnMut(u32) + Send + Sync>;
pub type OnSignalingChange = Box<dyn FnMut(SignalingState) + Send + Sync>;
pub type OnTrack = Box<dyn FnMut(TrackEvent) + Send + Sync>;
#[derive(Clone)]
pub struct PeerConnection {
pub(crate) handle: imp_pc::PeerConnection,
}
impl PeerConnection {
pub(crate) fn new(
cxx_handle: UniquePtr<sys_pc::ffi::PeerConnection>,
observer: Box<InternalObserver>,
native_observer: UniquePtr<sys_pc::ffi::NativePeerConnectionObserver>,
) -> Self {
Self {
cxx_handle,
observer,
native_observer,
}
}
fn create_sdp_observer() -> (
UniquePtr<sys_pc::ffi::NativeCreateSdpObserverHandle>,
mpsc::Receiver<Result<SessionDescription, RTCError>>,
) {
let (tx, rx) = mpsc::channel(1);
let wrapper = sys_jsep::CreateSdpObserverWrapper {
on_success: ManuallyDrop::new(Box::new({
let tx = tx.clone();
move |session_description| {
let _ = tx.blocking_send(Ok(SessionDescription::new(session_description)));
}
})),
on_failure: ManuallyDrop::new(Box::new(move |error| {
let _ = tx.blocking_send(Err(error));
})),
};
(
sys_jsep::ffi::create_native_create_sdp_observer(Box::new(wrapper)),
rx,
)
}
pub async fn create_offer(
&self,
options: RTCOfferAnswerOptions,
) -> Result<SessionDescription, RTCError> {
let (mut native_wrapper, mut rx) = Self::create_sdp_observer();
unsafe {
self.cxx_handle
.create_offer(native_wrapper.pin_mut(), options);
}
rx.recv().await.unwrap()
options: OfferOptions,
) -> Result<SessionDescription, RtcError> {
self.handle.create_offer(options).await
}
pub async fn create_answer(
&self,
options: RTCOfferAnswerOptions,
) -> Result<SessionDescription, RTCError> {
let (mut native_wrapper, mut rx) = Self::create_sdp_observer();
unsafe {
self.cxx_handle
.create_answer(native_wrapper.pin_mut(), options);
}
rx.recv().await.unwrap()
options: AnswerOptions,
) -> Result<SessionDescription, RtcError> {
self.handle.create_answer(options).await
}
pub async fn set_local_description(&self, desc: SessionDescription) -> Result<(), RTCError> {
let (tx, rx) = oneshot::channel();
let wrapper =
sys_jsep::SetLocalSdpObserverWrapper(ManuallyDrop::new(Box::new(move |error| {
let _ = tx.send(if error.ok() { Ok(()) } else { Err(error) });
})));
let mut native_wrapper =
sys_jsep::ffi::create_native_set_local_sdp_observer(Box::new(wrapper));
unsafe {
self.cxx_handle
.set_local_description(desc.release(), native_wrapper.pin_mut());
}
rx.await.unwrap()
pub async fn set_local_description(&self, desc: SessionDescription) -> Result<(), RtcError> {
self.handle.set_local_description(desc).await
}
pub async fn set_remote_description(&self, desc: SessionDescription) -> Result<(), RTCError> {
let (tx, rx) = oneshot::channel();
let wrapper =
sys_jsep::SetRemoteSdpObserverWrapper(ManuallyDrop::new(Box::new(move |error| {
let _ = tx.send(if error.ok() { Ok(()) } else { Err(error) });
})));
let mut native_wrapper =
sys_jsep::ffi::create_native_set_remote_sdp_observer(Box::new(wrapper));
unsafe {
self.cxx_handle
.set_remote_description(desc.release(), native_wrapper.pin_mut());
}
rx.await.unwrap()
pub async fn set_remote_description(&self, desc: SessionDescription) -> Result<(), RtcError> {
self.handle.set_remote_description(desc).await
}
pub fn add_track(
&self,
track: MediaStreamTrackHandle,
stream_ids: &Vec<String>,
) -> Result<RtpSender, RTCError> {
let res = self.cxx_handle.add_track(track.cxx_handle(), stream_ids);
match res {
Ok(cxx_handle) => Ok(RtpSender::new(cxx_handle)),
Err(e) => unsafe { Err(RTCError::from(e.what())) },
}
}
pub fn remove_track(&self, sender: RtpSender) -> Result<(), RTCError> {
self.cxx_handle
.remove_track(sender.cxx_handle())
.map_err(|e| unsafe { RTCError::from(e.what()) })
}
pub fn add_transceiver(
&self,
track: MediaStreamTrackHandle,
init: RtpTransceiverInit,
) -> Result<RtpTransceiver, RTCError> {
let res = self
.cxx_handle
.add_transceiver(track.cxx_handle(), init.into());
match res {
Ok(cxx_handle) => Ok(RtpTransceiver::new(cxx_handle)),
Err(e) => unsafe { Err(RTCError::from(e.what())) },
}
}
pub fn add_transceiver_for_media(
&self,
media_type: MediaType,
init: RtpTransceiverInit,
) -> Result<RtpTransceiver, RTCError> {
let res = self
.cxx_handle
.add_transceiver_for_media(media_type, init.into());
match res {
Ok(cxx_handle) => Ok(RtpTransceiver::new(cxx_handle)),
Err(e) => unsafe { Err(RTCError::from(e.what())) },
}
}
pub fn senders(&self) -> Vec<RtpSender> {
self.cxx_handle
.get_senders()
.into_iter()
.map(|sender| RtpSender::new(sender.ptr))
.collect()
}
pub fn receivers(&self) -> Vec<RtpReceiver> {
self.cxx_handle
.get_receivers()
.into_iter()
.map(|receiver| RtpReceiver::new(receiver.ptr))
.collect()
}
pub fn transceivers(&self) -> Vec<RtpTransceiver> {
self.cxx_handle
.get_transceivers()
.into_iter()
.map(|transceiver| RtpTransceiver::new(transceiver.ptr))
.collect()
pub async fn add_ice_candidate(&self, candidate: IceCandidate) -> Result<(), RtcError> {
self.handle.add_ice_candidate(candidate).await
}
pub fn create_data_channel(
&self,
label: &str,
init: DataChannelInit,
) -> Result<DataChannel, RTCError> {
let native_init = sys_dc::ffi::create_data_channel_init(init.into());
let res = self
.cxx_handle
.create_data_channel(label.to_string(), native_init);
match res {
Ok(cxx_handle) => Ok(DataChannel::new(cxx_handle)),
Err(e) => Err(unsafe { RTCError::from(e.what()) }),
}
) -> Result<DataChannel, RtcError> {
self.handle.create_data_channel(label, init)
}
// TODO(theomonnom) Use IceCandidateInit instead of IceCandidate
pub async fn add_ice_candidate(&self, candidate: IceCandidate) -> Result<(), RTCError> {
let (tx, rx) = oneshot::channel();
let observer =
sys_pc::AddIceCandidateObserverWrapper(ManuallyDrop::new(Box::new(|error| {
let _ = tx.send(if error.ok() { Ok(()) } else { Err(error) });
})));
let mut native_observer =
sys_pc::ffi::create_native_add_ice_candidate_observer(Box::new(observer));
self.cxx_handle
.add_ice_candidate(candidate.release(), native_observer.pin_mut());
rx.await.unwrap()
pub fn add_track<T: AsRef<str>>(
&self,
track: MediaStreamTrack,
streams_ids: &[T],
) -> Result<RtpSender, RtcError> {
self.handle.add_track(track, streams_ids)
}
pub fn local_description(&self) -> Option<SessionDescription> {
let local_description = self.cxx_handle.local_description();
if local_description.is_null() {
None
} else {
Some(SessionDescription::new(local_description))
}
pub fn remove_track(&self, sender: RtpSender) -> Result<(), RtcError> {
self.handle.remove_track(sender)
}
pub fn remote_description(&self) -> Option<SessionDescription> {
let remote_description = self.cxx_handle.remote_description();
if remote_description.is_null() {
None
} else {
Some(SessionDescription::new(remote_description))
}
pub fn add_transceiver(
&self,
track: MediaStreamTrack,
init: RtpTransceiverInit,
) -> Result<RtpTransceiver, RtcError> {
self.handle.add_transceiver(track, init)
}
pub fn signaling_state(&self) -> SignalingState {
self.cxx_handle.signaling_state()
pub fn add_transceiver_for_media(
&self,
media_type: MediaType,
init: RtpTransceiverInit,
) -> Result<RtpTransceiver, RtcError> {
self.handle.add_transceiver_for_media(media_type, init)
}
pub fn close(&self) {
self.handle.close()
}
pub fn ice_gathering_state(&self) -> IceGatheringState {
self.cxx_handle.ice_gathering_state()
pub fn connection_state(&self) -> PeerConnectionState {
self.handle.connection_state()
}
pub fn ice_connection_state(&self) -> IceConnectionState {
self.cxx_handle.ice_connection_state()
self.handle.ice_connection_state()
}
pub fn close(&mut self) {
self.cxx_handle.pin_mut().close();
pub fn ice_gathering_state(&self) -> IceGatheringState {
self.handle.ice_gathering_state()
}
pub fn on_signaling_change(&mut self, handler: OnSignalingChangeHandler) {
*self.observer.on_signaling_change_handler.lock().unwrap() = Some(handler);
pub fn signaling_state(&self) -> SignalingState {
self.handle.signaling_state()
}
pub fn on_add_stream(&mut self, handler: OnAddStreamHandler) {
*self.observer.on_add_stream_handler.lock().unwrap() = Some(handler);
pub fn current_local_description(&self) -> Option<SessionDescription> {
self.handle.current_local_description()
}
pub fn on_remove_stream(&mut self, handler: OnRemoveStreamHandler) {
*self.observer.on_remove_stream_handler.lock().unwrap() = Some(handler);
pub fn current_remote_description(&self) -> Option<SessionDescription> {
self.handle.current_remote_description()
}
pub fn on_data_channel(&mut self, handler: OnDataChannelHandler) {
*self.observer.on_data_channel_handler.lock().unwrap() = Some(handler);
pub fn senders(&self) -> Vec<RtpSender> {
self.handle.senders()
}
pub fn on_renegotiation_needed(&mut self, handler: OnRenegotiationNeededHandler) {
*self
.observer
.on_renegotiation_needed_handler
.lock()
.unwrap() = Some(handler);
pub fn receivers(&self) -> Vec<RtpReceiver> {
self.handle.receivers()
}
pub fn on_ice_connection_change(&mut self, handler: OnIceConnectionChangeHandler) {
*self
.observer
.on_ice_connection_change_handler
.lock()
.unwrap() = Some(handler);
pub fn transceivers(&self) -> Vec<RtpTransceiver> {
self.handle.transceivers()
}
pub fn on_standardized_ice_connection_change(
&mut self,
handler: OnStandardizedIceConnectionChangeHandler,
) {
*self
.observer
.on_standardized_ice_connection_change_handler
.lock()
.unwrap() = Some(handler);
pub fn on_connection_state_change(&self, f: Option<OnConnectionChange>) {
self.handle.on_connection_state_change(f)
}
pub fn on_connection_change(&mut self, handler: OnConnectionChangeHandler) {
*self.observer.on_connection_change_handler.lock().unwrap() = Some(handler);
pub fn on_data_channel(&self, f: Option<OnDataChannel>) {
self.handle.on_data_channel(f)
}
pub fn on_ice_gathering_change(&mut self, handler: OnIceGatheringChangeHandler) {
*self
.observer
.on_ice_gathering_change_handler
.lock()
.unwrap() = Some(handler);
pub fn on_ice_candidate(&self, f: Option<OnIceCandidate>) {
self.handle.on_ice_candidate(f)
}
pub fn on_ice_candidate(&mut self, handler: OnIceCandidateHandler) {
*self.observer.on_ice_candidate_handler.lock().unwrap() = Some(handler);
pub fn on_ice_candidate_error(&self, f: Option<OnIceCandidateError>) {
self.handle.on_ice_candidate_error(f)
}
pub fn on_ice_candidate_error(&mut self, handler: OnIceCandidateErrorHandler) {
*self.observer.on_ice_candidate_error_handler.lock().unwrap() = Some(handler);
pub fn on_ice_connection_state_change(&self, f: Option<OnIceConnectionChange>) {
self.handle.on_ice_connection_state_change(f)
}
pub fn on_ice_candidates_removed(&mut self, handler: OnIceCandidatesRemovedHandler) {
*self
.observer
.on_ice_candidates_removed_handler
.lock()
.unwrap() = Some(handler);
pub fn on_ice_gathering_state_change(&self, f: Option<OnIceGatheringChange>) {
self.handle.on_ice_gathering_state_change(f)
}
pub fn on_ice_connection_receiving_change(
&mut self,
handler: OnIceConnectionReceivingChangeHandler,
) {
*self
.observer
.on_ice_connection_receiving_change_handler
.lock()
.unwrap() = Some(handler);
pub fn on_negotiation_needed(&self, f: Option<OnNegotiationNeeded>) {
self.handle.on_negotiation_needed(f)
}
pub fn on_ice_selected_candidate_pair_changed(
&mut self,
handler: OnIceSelectedCandidatePairChangedHandler,
) {
*self
.observer
.on_ice_selected_candidate_pair_changed_handler
.lock()
.unwrap() = Some(handler);
pub fn on_signaling_state_change(&self, f: Option<OnSignalingChange>) {
self.handle.on_signaling_state_change(f)
}
pub fn on_add_track(&mut self, handler: OnAddTrackHandler) {
*self.observer.on_add_track_handler.lock().unwrap() = Some(handler);
}
pub fn on_track(&mut self, handler: OnTrackHandler) {
*self.observer.on_track_handler.lock().unwrap() = Some(handler);
}
pub fn on_remove_track(&mut self, handler: OnRemoveTrackHandler) {
*self.observer.on_remove_track_handler.lock().unwrap() = Some(handler);
}
pub fn on_interesting_usage(&mut self, handler: OnInterestingUsageHandler) {
*self.observer.on_interesting_usage_handler.lock().unwrap() = Some(handler);
pub fn on_track(&self, f: Option<OnTrack>) {
self.handle.on_track(f)
}
}
// TODO(theomonnom) Should we return futures?
pub type OnSignalingChangeHandler = Box<dyn FnMut(SignalingState) + Send + Sync>;
pub type OnAddStreamHandler = Box<dyn FnMut(MediaStream) + Send + Sync>;
pub type OnRemoveStreamHandler = Box<dyn FnMut(MediaStream) + Send + Sync>;
pub type OnDataChannelHandler = Box<dyn FnMut(DataChannel) + Send + Sync>;
pub type OnRenegotiationNeededHandler = Box<dyn FnMut() + Send + Sync>;
pub type OnNegotiationNeededEventHandler = Box<dyn FnMut(u32) + Send + Sync>;
pub type OnIceConnectionChangeHandler = Box<dyn FnMut(IceConnectionState) + Send + Sync>;
pub type OnStandardizedIceConnectionChangeHandler =
Box<dyn FnMut(IceConnectionState) + Send + Sync>;
pub type OnConnectionChangeHandler = Box<dyn FnMut(PeerConnectionState) + Send + Sync>;
pub type OnIceGatheringChangeHandler = Box<dyn FnMut(IceGatheringState) + Send + Sync>;
pub type OnIceCandidateHandler = Box<dyn FnMut(IceCandidate) + Send + Sync>;
pub type OnIceCandidateErrorHandler =
Box<dyn FnMut(String, i32, String, i32, String) + Send + Sync>;
pub type OnIceCandidatesRemovedHandler = Box<dyn FnMut(Vec<IceCandidate>) + Send + Sync>;
pub type OnIceConnectionReceivingChangeHandler = Box<dyn FnMut(bool) + Send + Sync>;
pub type OnIceSelectedCandidatePairChangedHandler =
Box<dyn FnMut(webrtc_sys::peer_connection::ffi::CandidatePairChangeEvent) + Send + Sync>;
pub type OnAddTrackHandler = Box<dyn FnMut(RtpReceiver, Vec<MediaStream>) + Send + Sync>;
pub type OnTrackHandler = Box<dyn FnMut(RtpTransceiver) + Send + Sync>;
pub type OnRemoveTrackHandler = Box<dyn FnMut(RtpReceiver) + Send + Sync>;
pub type OnInterestingUsageHandler = Box<dyn FnMut(i32) + Send + Sync>;
pub(crate) struct InternalObserver {
on_signaling_change_handler: Arc<Mutex<Option<OnSignalingChangeHandler>>>,
on_add_stream_handler: Arc<Mutex<Option<OnAddStreamHandler>>>,
on_remove_stream_handler: Arc<Mutex<Option<OnRemoveStreamHandler>>>,
on_data_channel_handler: Arc<Mutex<Option<OnDataChannelHandler>>>,
on_renegotiation_needed_handler: Arc<Mutex<Option<OnRenegotiationNeededHandler>>>,
on_negotiation_needed_event_handler: Arc<Mutex<Option<OnNegotiationNeededEventHandler>>>,
on_ice_connection_change_handler: Arc<Mutex<Option<OnIceConnectionChangeHandler>>>,
on_standardized_ice_connection_change_handler:
Arc<Mutex<Option<OnStandardizedIceConnectionChangeHandler>>>,
on_connection_change_handler: Arc<Mutex<Option<OnConnectionChangeHandler>>>,
on_ice_gathering_change_handler: Arc<Mutex<Option<OnIceGatheringChangeHandler>>>,
on_ice_candidate_handler: Arc<Mutex<Option<OnIceCandidateHandler>>>,
on_ice_candidate_error_handler: Arc<Mutex<Option<OnIceCandidateErrorHandler>>>,
on_ice_candidates_removed_handler: Arc<Mutex<Option<OnIceCandidatesRemovedHandler>>>,
on_ice_connection_receiving_change_handler:
Arc<Mutex<Option<OnIceConnectionReceivingChangeHandler>>>,
on_ice_selected_candidate_pair_changed_handler:
Arc<Mutex<Option<OnIceSelectedCandidatePairChangedHandler>>>,
on_add_track_handler: Arc<Mutex<Option<OnAddTrackHandler>>>,
on_track_handler: Arc<Mutex<Option<OnTrackHandler>>>,
on_remove_track_handler: Arc<Mutex<Option<OnRemoveTrackHandler>>>,
on_interesting_usage_handler: Arc<Mutex<Option<OnInterestingUsageHandler>>>,
}
impl Default for InternalObserver {
fn default() -> Self {
Self {
on_signaling_change_handler: Arc::new(Default::default()),
on_add_stream_handler: Arc::new(Default::default()),
on_remove_stream_handler: Arc::new(Default::default()),
on_data_channel_handler: Arc::new(Default::default()),
on_renegotiation_needed_handler: Arc::new(Default::default()),
on_negotiation_needed_event_handler: Arc::new(Default::default()),
on_ice_connection_change_handler: Arc::new(Default::default()),
on_standardized_ice_connection_change_handler: Arc::new(Default::default()),
on_connection_change_handler: Arc::new(Default::default()),
on_ice_gathering_change_handler: Arc::new(Default::default()),
on_ice_candidate_handler: Arc::new(Default::default()),
on_ice_candidate_error_handler: Arc::new(Default::default()),
on_ice_candidates_removed_handler: Arc::new(Default::default()),
on_ice_connection_receiving_change_handler: Arc::new(Default::default()),
on_ice_selected_candidate_pair_changed_handler: Arc::new(Default::default()),
on_add_track_handler: Arc::new(Default::default()),
on_track_handler: Arc::new(Default::default()),
on_remove_track_handler: Arc::new(Default::default()),
on_interesting_usage_handler: Arc::new(Default::default()),
}
}
}
// Observers are being called on the Signaling Thread
impl sys_pc::PeerConnectionObserver for InternalObserver {
fn on_signaling_change(&self, new_state: SignalingState) {
trace!("on_signaling_change, {:?}", new_state);
let mut handler = self.on_signaling_change_handler.lock().unwrap();
if let Some(f) = handler.as_mut() {
f(new_state);
}
}
fn on_add_stream(&self, stream: SharedPtr<sys_ms::ffi::MediaStream>) {
trace!("on_add_stream");
let mut handler = self.on_add_stream_handler.lock().unwrap();
if let Some(f) = handler.as_mut() {
// TODO(theomonnom)
}
}
fn on_remove_stream(&self, stream: SharedPtr<sys_ms::ffi::MediaStream>) {
trace!("on_remove_stream");
let mut handler = self.on_remove_stream_handler.lock().unwrap();
if let Some(f) = handler.as_mut() {
// TODO(theomonnom)
}
}
fn on_data_channel(&self, data_channel: UniquePtr<sys_dc::ffi::DataChannel>) {
trace!("on_data_channel");
let mut handler = self.on_data_channel_handler.lock().unwrap();
if let Some(f) = handler.as_mut() {
f(DataChannel::new(data_channel));
}
}
fn on_renegotiation_needed(&self) {
trace!("on_renegotiation_needed");
let mut handler = self.on_renegotiation_needed_handler.lock().unwrap();
if let Some(f) = handler.as_mut() {
f();
}
}
fn on_negotiation_needed_event(&self, event: u32) {
trace!("on_negotiation_needed_event");
let mut handler = self.on_negotiation_needed_event_handler.lock().unwrap();
if let Some(f) = handler.as_mut() {
f(event);
}
}
fn on_ice_connection_change(&self, new_state: IceConnectionState) {
trace!("on_ice_connection_change (new_state: {:?})", new_state);
let mut handler = self.on_ice_connection_change_handler.lock().unwrap();
if let Some(f) = handler.as_mut() {
f(new_state);
}
}
fn on_standardized_ice_connection_change(&self, new_state: IceConnectionState) {
trace!(
"on_standardized_ice_connection_change (new_state: {:?}",
new_state
);
let mut handler = self
.on_standardized_ice_connection_change_handler
.lock()
.unwrap();
if let Some(f) = handler.as_mut() {
f(new_state);
}
}
fn on_connection_change(&self, new_state: PeerConnectionState) {
trace!("on_connection_change (new_state: {:?})", new_state);
let mut handler = self.on_connection_change_handler.lock().unwrap();
if let Some(f) = handler.as_mut() {
f(new_state);
}
}
fn on_ice_gathering_change(&self, new_state: IceGatheringState) {
trace!("on_ice_gathering_change (new_state: {:?}", new_state);
let mut handler = self.on_ice_gathering_change_handler.lock().unwrap();
if let Some(f) = handler.as_mut() {
f(new_state);
}
}
fn on_ice_candidate(&self, candidate: SharedPtr<sys_jsep::ffi::IceCandidate>) {
trace!("on_ice_candidate");
let mut handler = self.on_ice_candidate_handler.lock().unwrap();
if let Some(f) = handler.as_mut() {
f(IceCandidate::new(candidate));
}
}
fn on_ice_candidate_error(
&self,
address: String,
port: i32,
url: String,
error_code: i32,
error_text: String,
) {
trace!("on_ice_candidate_error");
let mut handler = self.on_ice_candidate_error_handler.lock().unwrap();
if let Some(f) = handler.as_mut() {
f(address, port, url, error_code, error_text);
}
}
fn on_ice_candidates_removed(&self, removed: Vec<SharedPtr<sys_ca::ffi::Candidate>>) {
trace!("on_ice_candidates_removed");
let mut handler = self.on_ice_candidates_removed_handler.lock().unwrap();
if let Some(f) = handler.as_mut() {
// TODO(theomonnom)
}
}
fn on_ice_connection_receiving_change(&self, receiving: bool) {
trace!("on_ice_connection_receiving_change");
let mut handler = self
.on_ice_connection_receiving_change_handler
.lock()
.unwrap();
if let Some(f) = handler.as_mut() {
f(receiving);
}
}
fn on_ice_selected_candidate_pair_changed(&self, event: sys_pc::ffi::CandidatePairChangeEvent) {
trace!("on_ice_selected_candidate_pair_changed");
let mut handler = self
.on_ice_selected_candidate_pair_changed_handler
.lock()
.unwrap();
if let Some(f) = handler.as_mut() {
f(event);
}
}
fn on_add_track(
&self,
receiver: SharedPtr<sys_rr::ffi::RtpReceiver>,
streams: Vec<SharedPtr<sys_ms::ffi::MediaStream>>,
) {
trace!("on_add_track");
let mut handler = self.on_add_track_handler.lock().unwrap();
if let Some(f) = handler.as_mut() {
let streams = streams.into_iter().map(MediaStream::new).collect();
f(RtpReceiver::new(receiver), streams)
}
}
fn on_track(&self, transceiver: SharedPtr<sys_rt::ffi::RtpTransceiver>) {
trace!("on_track");
let mut handler = self.on_track_handler.lock().unwrap();
if let Some(f) = handler.as_mut() {
// TODO(theomonnom)
}
}
fn on_remove_track(&self, receiver: SharedPtr<sys_rr::ffi::RtpReceiver>) {
trace!("on_remove_track");
let mut handler = self.on_remove_track_handler.lock().unwrap();
if let Some(f) = handler.as_mut() {
// TODO(theomonnom)
}
}
fn on_interesting_usage(&self, usage_pattern: i32) {
trace!("on_interesting_usage");
let mut handler = self.on_interesting_usage_handler.lock().unwrap();
if let Some(f) = handler.as_mut() {
f(usage_pattern);
}
}
}
#[cfg(test)]
mod tests {
use log::trace;
use tokio::sync::mpsc;
use webrtc_sys::peer_connection::ffi::RTCOfferAnswerOptions;
use webrtc_sys::peer_connection_factory::ffi::{ContinualGatheringPolicy, IceTransportsType};
use crate::data_channel::{DataChannel, DataChannelInit};
use crate::jsep::IceCandidate;
use crate::peer_connection_factory::{ICEServer, PeerConnectionFactory, RTCConfiguration};
use crate::webrtc::RTCRuntime;
fn init_log() {
let _ = env_logger::builder().is_test(true).try_init();
}
#[tokio::test]
async fn create_pc() {
init_log();
let rtc_runtime = RTCRuntime::new();
let factory = PeerConnectionFactory::new(rtc_runtime);
let config = RTCConfiguration {
ice_servers: vec![ICEServer {
urls: vec!["stun:stun1.l.google.com:19302".to_string()],
username: "".into(),
password: "".into(),
}],
continual_gathering_policy: ContinualGatheringPolicy::GatherOnce,
ice_transport_type: IceTransportsType::All,
};
let mut bob = factory.create_peer_connection(config.clone()).unwrap();
let mut alice = factory.create_peer_connection(config.clone()).unwrap();
let (bob_ice_tx, mut bob_ice_rx) = mpsc::channel::<IceCandidate>(16);
let (alice_ice_tx, mut alice_ice_rx) = mpsc::channel::<IceCandidate>(16);
let (alice_dc_tx, mut alice_dc_rx) = mpsc::channel::<DataChannel>(16);
bob.on_ice_candidate(Box::new(move |candidate| {
bob_ice_tx.blocking_send(candidate).unwrap();
}));
alice.on_ice_candidate(Box::new(move |candidate| {
alice_ice_tx.blocking_send(candidate).unwrap();
}));
alice.on_data_channel(Box::new(move |dc| {
alice_dc_tx.blocking_send(dc).unwrap();
}));
let mut bob_dc = bob
.create_data_channel("test_dc", DataChannelInit::default())
.unwrap();
let offer = bob
.create_offer(RTCOfferAnswerOptions::default())
.await
.unwrap();
trace!("Bob offer: {:?}", offer);
bob.set_local_description(offer.clone()).await.unwrap();
alice.set_remote_description(offer).await.unwrap();
let answer = alice
.create_answer(RTCOfferAnswerOptions::default())
.await
.unwrap();
trace!("Alice answer: {:?}", answer);
alice.set_local_description(answer.clone()).await.unwrap();
bob.set_remote_description(answer).await.unwrap();
let bob_ice = bob_ice_rx.recv().await.unwrap();
let alice_ice = alice_ice_rx.recv().await.unwrap();
bob.add_ice_candidate(alice_ice).await.unwrap();
alice.add_ice_candidate(bob_ice).await.unwrap();
let (data_tx, mut data_rx) = mpsc::channel::<String>(1);
let mut alice_dc = alice_dc_rx.recv().await.unwrap();
alice_dc.on_message(Box::new(move |data, _| {
data_tx
.blocking_send(String::from_utf8_lossy(data).to_string())
.unwrap();
}));
bob_dc.send(b"This is a test", true).unwrap();
assert_eq!(data_rx.recv().await.unwrap(), "This is a test");
alice.close();
bob.close();
impl Debug for PeerConnection {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("PeerConnection")
.field("state", &self.connection_state())
.field("ice_state", &self.ice_connection_state())
.finish()
}
}
livekit-webrtc/src/peer_connection_factory.rs
+63 -36
View File
@@ -1,51 +1,78 @@
use cxx::UniquePtr;
use crate::imp::peer_connection_factory as imp_pcf;
use crate::peer_connection::PeerConnection;
use crate::rtp_parameters::RtpCapabilities;
use crate::MediaType;
use crate::RtcError;
use std::fmt::Debug;
pub use sys_factory::ffi::{
ContinualGatheringPolicy, ICEServer, IceTransportsType, RTCConfiguration,
};
use webrtc_sys::peer_connection as sys_pc;
use webrtc_sys::peer_connection_factory as sys_factory;
#[derive(Debug, Clone)]
pub struct IceServer {
pub urls: Vec<String>,
pub username: String,
pub password: String,
}
use crate::peer_connection::{InternalObserver, PeerConnection};
use crate::rtc_error::RTCError;
use crate::webrtc::RTCRuntime;
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum ContinualGatheringPolicy {
GatherOnce,
GatherContinually,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum IceTransportsType {
None,
Relay,
NoHost,
All,
}
#[derive(Debug, Clone)]
pub struct RtcConfiguration {
pub ice_servers: Vec<IceServer>,
pub continual_gathering_policy: ContinualGatheringPolicy,
pub ice_transport_type: IceTransportsType,
}
#[derive(Clone, Default)]
pub struct PeerConnectionFactory {
cxx_handle: UniquePtr<sys_factory::ffi::PeerConnectionFactory>,
rtc_runtime: RTCRuntime,
pub(crate) handle: imp_pcf::PeerConnectionFactory,
}
impl Debug for PeerConnectionFactory {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
f.debug_struct("PeerConnectionFactory").finish()
}
}
impl PeerConnectionFactory {
pub fn new(rtc_runtime: RTCRuntime) -> Self {
Self {
cxx_handle: sys_factory::ffi::create_peer_connection_factory(
rtc_runtime.clone().release(),
),
rtc_runtime,
}
}
pub fn create_peer_connection(
&self,
config: RTCConfiguration,
) -> Result<PeerConnection, RTCError> {
let native_config = sys_factory::ffi::create_rtc_configuration(config);
config: RtcConfiguration,
) -> Result<PeerConnection, RtcError> {
self.handle.create_peer_connection(config)
}
unsafe {
let mut observer = Box::new(InternalObserver::default());
let mut native_observer = sys_pc::ffi::create_native_peer_connection_observer(
self.rtc_runtime.clone().release(),
Box::new(sys_pc::PeerConnectionObserverWrapper::new(&mut *observer)),
);
pub fn get_rtp_sender_capabilities(&self, media_type: MediaType) -> RtpCapabilities {
self.handle.get_rtp_sender_capabilities(media_type)
}
let res = self
.cxx_handle
.create_peer_connection(native_config, native_observer.pin_mut());
pub fn get_rtp_receiver_capabilities(&self, media_type: MediaType) -> RtpCapabilities {
self.handle.get_rtp_receiver_capabilities(media_type)
}
}
match res {
Ok(cxx_handle) => Ok(PeerConnection::new(cxx_handle, observer, native_observer)),
Err(e) => Err(RTCError::from(e.what())),
}
pub mod native {
use super::PeerConnectionFactory;
use crate::media_stream::RtcVideoTrack;
use crate::video_source::native::NativeVideoSource;
pub trait PeerConnectionFactoryExt {
fn create_video_track(&self, label: &str, source: NativeVideoSource) -> RtcVideoTrack;
}
impl PeerConnectionFactoryExt for PeerConnectionFactory {
fn create_video_track(&self, label: &str, source: NativeVideoSource) -> RtcVideoTrack {
self.handle.create_video_track(label, source)
}
}
}
livekit-webrtc/src/prelude.rs
+15 -13
View File
@@ -1,22 +1,24 @@
pub use crate::data_channel::{DataChannel, DataChannelInit, DataState};
pub use crate::jsep::{IceCandidate, SessionDescription};
pub use crate::data_channel::{
DataBuffer, DataChannel, DataChannelError, DataChannelInit, DataState,
};
pub use crate::ice_candidate::IceCandidate;
pub use crate::media_stream::{
AudioTrack, MediaStream, MediaStreamTrackHandle, MediaStreamTrackTrait,
OnConstraintsChangedHandler, OnDiscardedFrameHandler, OnFrameHandler, VideoTrack,
MediaStream, MediaStreamTrack, RtcAudioTrack, RtcTrackState, RtcVideoTrack,
};
pub use crate::peer_connection::{
IceConnectionState, IceGatheringState, PeerConnection, PeerConnectionState,
RTCOfferAnswerOptions, SignalingState,
AnswerOptions, IceConnectionState, IceGatheringState, OfferOptions, PeerConnection,
PeerConnectionState, SignalingState,
};
pub use crate::peer_connection_factory::{
ContinualGatheringPolicy, ICEServer, IceTransportsType, PeerConnectionFactory, RTCConfiguration,
ContinualGatheringPolicy, IceServer, IceTransportsType, PeerConnectionFactory, RtcConfiguration,
};
pub use crate::rtc_error::RTCError;
pub use crate::rtp_parameters::*;
pub use crate::rtp_receiver::RtpReceiver;
pub use crate::rtp_sender::RtpSender;
pub use crate::rtp_transceiver::{RtpTransceiver, RtpTransceiverInit};
pub use crate::video_frame::{VideoFrame, VideoRotation};
pub use crate::video_frame_buffer::*;
pub use crate::webrtc::*;
pub use crate::yuv_helper::ConvertError;
pub use crate::rtp_transceiver::{RtpTransceiver, RtpTransceiverDirection, RtpTransceiverInit};
pub use crate::session_description::{SdpType, SessionDescription};
pub use crate::video_frame::{
BoxVideoFrame, I010Buffer, I420ABuffer, I420Buffer, I422Buffer, I444Buffer, NV12Buffer,
VideoFormatType, VideoFrame, VideoFrameBuffer, VideoFrameBufferType, VideoRotation,
};
pub use crate::{RtcError, RtcErrorType};
livekit-webrtc/src/rtc_error.rs
-2
View File
@@ -1,2 +0,0 @@
// TODO(theomonnom) Wrap the RTCError ffi so we can use Option(u16)
pub use webrtc_sys::rtc_error::ffi::RTCError;
livekit-webrtc/src/rtp_parameters.rs
+60 -431
View File
@@ -1,462 +1,91 @@
use crate::prelude::*;
use std::collections::HashMap;
use std::vec::Vec;
use webrtc_sys::rtp_parameters as ps_sys;
use crate::rtp_transceiver::RtpTransceiverDirection;
// Don't exporting structs here (only enum), cxx doesn't support Option and HashMap
pub use ps_sys::ffi::{
DegradationPreference, FecMechanism, RtcpFeedbackMessageType, RtcpFeedbackType,
RtpExtensionFilter,
};
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum Priority {
VeryLow,
Low,
Medium,
High,
}
#[derive(Debug, Clone)]
pub struct RtcpFeedback {
pub feedback_type: RtcpFeedbackType,
pub message_type: Option<RtcpFeedbackMessageType>,
pub struct RtpHeaderExtensionParameters {
pub uri: String,
pub id: i32,
pub encrypted: bool,
}
#[derive(Debug, Clone, Default)]
pub struct RtpParameters {
pub codecs: Vec<RtpCodecParameters>,
pub header_extensions: Vec<RtpHeaderExtensionParameters>,
pub rtcp: RtcpParameters,
}
#[derive(Debug, Clone)]
pub struct RtpCodecParameters {
pub payload_type: u8,
pub mime_type: String, // read-only
pub clock_rate: Option<u64>,
pub channels: Option<u16>,
}
#[derive(Debug, Clone, Default)]
pub struct RtcpParameters {
pub cname: String,
pub reduced_size: bool,
}
#[derive(Debug, Clone)]
pub struct RtpEncodingParameters {
pub active: bool,
pub max_bitrate: Option<u64>,
pub max_framerate: Option<f64>,
pub priority: Priority,
pub rid: String,
pub scale_resolution_down_by: Option<f64>,
}
#[derive(Debug, Clone)]
pub struct RtpCodecCapability {
pub channels: Option<u16>,
pub clock_rate: Option<u64>,
pub mime_type: String,
pub name: String,
pub kind: MediaType,
pub clock_rate: Option<i32>,
pub preferred_payload_type: Option<i32>,
pub max_ptime: Option<i32>,
pub ptime: Option<i32>,
pub num_channels: Option<i32>,
pub rtcp_feedback: Vec<RtcpFeedback>,
pub parameters: HashMap<String, String>,
pub options: HashMap<String, String>,
pub max_temporal_layer_extensions: i32,
pub max_spatial_layer_extensions: i32,
pub svc_multi_stream_support: bool,
pub sdp_fmtp_line: Option<String>,
}
#[derive(Debug, Clone)]
pub struct RtpHeaderExtensionCapability {
pub uri: String,
pub preferred_id: Option<i32>,
pub preferred_encrypt: bool,
pub direction: RtpTransceiverDirection,
}
#[derive(Debug, Clone)]
pub struct RtpExtension {
pub uri: String,
pub id: i32,
pub encrypt: bool,
}
#[derive(Debug, Clone)]
pub struct RtpFecParameters {
pub ssrc: Option<u32>,
pub mechanism: FecMechanism,
}
#[derive(Debug, Clone)]
pub struct RtpRtxParameters {
pub ssrc: Option<u32>,
}
#[derive(Debug, Clone)]
pub struct RtpEncodingParameters {
pub ssrc: Option<u32>,
pub bitrate_priority: f64,
pub network_priority: Priority,
pub max_bitrate_bps: Option<i32>,
pub min_bitrate_bps: Option<i32>,
pub max_framerate: Option<f64>,
pub num_temporal_layers: Option<i32>,
pub scale_resolution_down_by: Option<f64>,
pub scalability_mode: Option<String>,
pub active: bool,
pub rid: String,
pub adaptive_ptime: bool,
}
#[derive(Debug, Clone)]
pub struct RtpCodecParameters {
pub mime_type: String,
pub name: String,
pub kind: MediaType,
pub payload_type: i32,
pub clock_rate: Option<i32>,
pub num_channels: Option<i32>,
pub max_ptime: Option<i32>,
pub ptime: Option<i32>,
pub rtcp_feedback: Vec<RtcpFeedback>,
pub parameters: HashMap<String, String>,
}
#[derive(Debug, Clone)]
pub struct RtpCapabilities {
pub codecs: Vec<RtpCodecCapability>,
pub header_extensions: Vec<RtpHeaderExtensionCapability>,
pub fec: Vec<FecMechanism>,
}
#[derive(Debug, Clone)]
pub struct RtcpParameters {
pub ssrc: Option<u32>,
pub cname: String,
pub reduced_size: bool,
pub mux: bool,
}
#[derive(Debug, Clone)]
pub struct RtpParameters {
pub transaction_id: String,
pub mid: String,
pub codecs: Vec<RtpCodecParameters>,
pub header_extensions: Vec<RtpExtension>,
pub encodings: Vec<RtpEncodingParameters>,
pub rtcp: RtcpParameters,
pub degradation_preference: Option<DegradationPreference>,
}
fn into_map(vec: Vec<ps_sys::ffi::StringKeyValue>) -> HashMap<String, String> {
let mut map = HashMap::with_capacity(vec.len());
for pair in vec {
map.insert(pair.key, pair.value);
}
map
}
impl From<ps_sys::ffi::RtcpFeedback> for RtcpFeedback {
fn from(value: ps_sys::ffi::RtcpFeedback) -> Self {
impl Default for RtpCodecParameters {
fn default() -> Self {
Self {
feedback_type: value.feedback_type,
message_type: value.has_message_type.then_some(value.message_type),
payload_type: 0,
mime_type: String::default(),
clock_rate: None,
channels: None,
}
}
}
impl From<ps_sys::ffi::RtpCodecCapability> for RtpCodecCapability {
fn from(value: ps_sys::ffi::RtpCodecCapability) -> Self {
impl Default for RtpEncodingParameters {
fn default() -> Self {
Self {
mime_type: value.mime_type,
name: value.name,
kind: value.kind,
clock_rate: value.has_clock_rate.then_some(value.clock_rate),
preferred_payload_type: value
.has_preferred_payload_type
.then_some(value.preferred_payload_type),
max_ptime: value.has_max_ptime.then_some(value.max_ptime),
ptime: value.has_ptime.then_some(value.ptime),
num_channels: value.has_num_channels.then_some(value.num_channels),
rtcp_feedback: value.rtcp_feedback.into_iter().map(Into::into).collect(),
parameters: into_map(value.parameters),
options: into_map(value.options),
max_temporal_layer_extensions: value.max_temporal_layer_extensions,
max_spatial_layer_extensions: value.max_spatial_layer_extensions,
svc_multi_stream_support: value.svc_multi_stream_support,
}
}
}
impl From<ps_sys::ffi::RtpHeaderExtensionCapability> for RtpHeaderExtensionCapability {
fn from(value: ps_sys::ffi::RtpHeaderExtensionCapability) -> Self {
Self {
uri: value.uri,
preferred_id: value.has_preferred_id.then_some(value.preferred_id),
preferred_encrypt: value.preferred_encrypt,
direction: value.direction,
}
}
}
impl From<ps_sys::ffi::RtpExtension> for RtpExtension {
fn from(value: ps_sys::ffi::RtpExtension) -> Self {
Self {
uri: value.uri,
id: value.id,
encrypt: value.encrypt,
}
}
}
impl From<ps_sys::ffi::RtpFecParameters> for RtpFecParameters {
fn from(value: ps_sys::ffi::RtpFecParameters) -> Self {
Self {
ssrc: value.has_ssrc.then_some(value.ssrc),
mechanism: value.mechanism,
}
}
}
impl From<ps_sys::ffi::RtpRtxParameters> for RtpRtxParameters {
fn from(value: ps_sys::ffi::RtpRtxParameters) -> Self {
Self {
ssrc: value.has_ssrc.then_some(value.ssrc),
}
}
}
impl From<ps_sys::ffi::RtpEncodingParameters> for RtpEncodingParameters {
fn from(value: ps_sys::ffi::RtpEncodingParameters) -> Self {
Self {
ssrc: value.has_ssrc.then_some(value.ssrc),
bitrate_priority: value.bitrate_priority,
network_priority: value.network_priority,
max_bitrate_bps: value.has_max_bitrate_bps.then_some(value.max_bitrate_bps),
min_bitrate_bps: value.has_min_bitrate_bps.then_some(value.min_bitrate_bps),
max_framerate: value.has_max_framerate.then_some(value.max_framerate),
num_temporal_layers: value
.has_num_temporal_layers
.then_some(value.num_temporal_layers),
scale_resolution_down_by: value
.has_scale_resolution_down_by
.then_some(value.scale_resolution_down_by),
scalability_mode: value.has_scalability_mode.then_some(value.scalability_mode),
active: value.active,
rid: value.rid,
adaptive_ptime: value.adaptive_ptime,
}
}
}
impl From<ps_sys::ffi::RtpCodecParameters> for RtpCodecParameters {
fn from(value: ps_sys::ffi::RtpCodecParameters) -> Self {
Self {
mime_type: value.mime_type,
name: value.name,
kind: value.kind,
payload_type: value.payload_type,
clock_rate: value.has_clock_rate.then_some(value.clock_rate),
num_channels: value.has_num_channels.then_some(value.num_channels),
max_ptime: value.has_max_ptime.then_some(value.max_ptime),
ptime: value.has_ptime.then_some(value.ptime),
rtcp_feedback: value.rtcp_feedback.into_iter().map(Into::into).collect(),
parameters: into_map(value.parameters),
}
}
}
impl From<ps_sys::ffi::RtpCapabilities> for RtpCapabilities {
fn from(value: ps_sys::ffi::RtpCapabilities) -> Self {
Self {
codecs: value.codecs.into_iter().map(Into::into).collect(),
header_extensions: value
.header_extensions
.into_iter()
.map(Into::into)
.collect(),
fec: value.fec.into_iter().map(Into::into).collect(),
}
}
}
impl From<ps_sys::ffi::RtcpParameters> for RtcpParameters {
fn from(value: ps_sys::ffi::RtcpParameters) -> Self {
Self {
ssrc: value.has_ssrc.then_some(value.ssrc),
cname: value.cname,
reduced_size: value.reduced_size,
mux: value.mux,
}
}
}
impl From<ps_sys::ffi::RtpParameters> for RtpParameters {
fn from(value: ps_sys::ffi::RtpParameters) -> Self {
Self {
transaction_id: value.transaction_id,
mid: value.mid,
codecs: value.codecs.into_iter().map(Into::into).collect(),
header_extensions: value
.header_extensions
.into_iter()
.map(Into::into)
.collect(),
encodings: value.encodings.into_iter().map(Into::into).collect(),
rtcp: value.rtcp.into(),
degradation_preference: value
.has_degradation_preference
.then_some(value.degradation_preference),
}
}
}
// Ignore the value inside unwrap_or for the following implementations
fn into_vec(map: HashMap<String, String>) -> Vec<ps_sys::ffi::StringKeyValue> {
let mut vec = Vec::with_capacity(map.len());
for (key, value) in map {
vec.push(ps_sys::ffi::StringKeyValue { key, value })
}
vec
}
impl From<RtcpFeedback> for ps_sys::ffi::RtcpFeedback {
fn from(value: RtcpFeedback) -> Self {
Self {
feedback_type: value.feedback_type,
has_message_type: value.message_type.is_some(),
message_type: value
.message_type
.unwrap_or(RtcpFeedbackMessageType::GenericNACK),
}
}
}
impl From<RtpCodecCapability> for ps_sys::ffi::RtpCodecCapability {
fn from(value: RtpCodecCapability) -> Self {
Self {
mime_type: value.mime_type,
name: value.name,
kind: value.kind,
has_clock_rate: value.clock_rate.is_some(),
clock_rate: value.clock_rate.unwrap_or(0),
has_preferred_payload_type: value.preferred_payload_type.is_some(),
preferred_payload_type: value.preferred_payload_type.unwrap_or(0),
has_max_ptime: value.max_ptime.is_some(),
max_ptime: value.max_ptime.unwrap_or(0),
has_ptime: value.ptime.is_some(),
ptime: value.ptime.unwrap_or(0),
has_num_channels: value.num_channels.is_some(),
num_channels: value.num_channels.unwrap_or(0),
rtcp_feedback: value.rtcp_feedback.into_iter().map(Into::into).collect(),
parameters: into_vec(value.parameters),
options: into_vec(value.options),
max_temporal_layer_extensions: value.max_temporal_layer_extensions,
max_spatial_layer_extensions: value.max_spatial_layer_extensions,
svc_multi_stream_support: value.svc_multi_stream_support,
}
}
}
impl From<RtpHeaderExtensionCapability> for ps_sys::ffi::RtpHeaderExtensionCapability {
fn from(value: RtpHeaderExtensionCapability) -> Self {
Self {
uri: value.uri,
has_preferred_id: value.preferred_id.is_some(),
preferred_id: value.preferred_id.unwrap_or(0),
preferred_encrypt: value.preferred_encrypt,
direction: value.direction,
}
}
}
impl From<RtpExtension> for ps_sys::ffi::RtpExtension {
fn from(value: RtpExtension) -> Self {
Self {
uri: value.uri,
id: value.id,
encrypt: value.encrypt,
}
}
}
impl From<RtpFecParameters> for ps_sys::ffi::RtpFecParameters {
fn from(value: RtpFecParameters) -> Self {
Self {
has_ssrc: value.ssrc.is_some(),
ssrc: value.ssrc.unwrap_or(0),
mechanism: value.mechanism,
}
}
}
impl From<RtpRtxParameters> for ps_sys::ffi::RtpRtxParameters {
fn from(value: RtpRtxParameters) -> Self {
Self {
has_ssrc: value.ssrc.is_some(),
ssrc: value.ssrc.unwrap_or(0),
}
}
}
impl From<RtpEncodingParameters> for ps_sys::ffi::RtpEncodingParameters {
fn from(value: RtpEncodingParameters) -> Self {
Self {
has_ssrc: value.ssrc.is_some(),
ssrc: value.ssrc.unwrap_or(0),
bitrate_priority: value.bitrate_priority,
network_priority: value.network_priority,
has_max_bitrate_bps: value.max_bitrate_bps.is_some(),
max_bitrate_bps: value.max_bitrate_bps.unwrap_or(0),
has_min_bitrate_bps: value.min_bitrate_bps.is_some(),
min_bitrate_bps: value.min_bitrate_bps.unwrap_or(0),
has_max_framerate: value.max_framerate.is_some(),
max_framerate: value.max_framerate.unwrap_or(0.0),
has_num_temporal_layers: value.num_temporal_layers.is_some(),
num_temporal_layers: value.num_temporal_layers.unwrap_or(0),
has_scale_resolution_down_by: value.scale_resolution_down_by.is_some(),
scale_resolution_down_by: value.scale_resolution_down_by.unwrap_or(0.0),
has_scalability_mode: value.scalability_mode.is_some(),
scalability_mode: value.scalability_mode.unwrap_or(String::new()),
active: value.active,
rid: value.rid,
adaptive_ptime: value.adaptive_ptime,
}
}
}
impl From<RtpCodecParameters> for ps_sys::ffi::RtpCodecParameters {
fn from(value: RtpCodecParameters) -> Self {
Self {
mime_type: value.mime_type,
name: value.name,
kind: value.kind,
payload_type: value.payload_type,
has_clock_rate: value.clock_rate.is_some(),
clock_rate: value.clock_rate.unwrap_or(0),
has_num_channels: value.num_channels.is_some(),
num_channels: value.num_channels.unwrap_or(0),
has_max_ptime: value.max_ptime.is_some(),
max_ptime: value.max_ptime.unwrap_or(0),
has_ptime: value.ptime.is_some(),
ptime: value.ptime.unwrap_or(0),
rtcp_feedback: value.rtcp_feedback.into_iter().map(Into::into).collect(),
parameters: into_vec(value.parameters),
}
}
}
impl From<RtpCapabilities> for ps_sys::ffi::RtpCapabilities {
fn from(value: RtpCapabilities) -> Self {
Self {
codecs: value.codecs.into_iter().map(Into::into).collect(),
header_extensions: value
.header_extensions
.into_iter()
.map(Into::into)
.collect(),
fec: value.fec.into_iter().map(Into::into).collect(),
}
}
}
impl From<RtcpParameters> for ps_sys::ffi::RtcpParameters {
fn from(value: RtcpParameters) -> Self {
Self {
has_ssrc: value.ssrc.is_some(),
ssrc: value.ssrc.unwrap_or(0),
cname: value.cname,
reduced_size: value.reduced_size,
mux: value.mux,
}
}
}
impl From<RtpParameters> for ps_sys::ffi::RtpParameters {
fn from(value: RtpParameters) -> Self {
Self {
transaction_id: value.transaction_id,
mid: value.mid,
codecs: value.codecs.into_iter().map(Into::into).collect(),
header_extensions: value
.header_extensions
.into_iter()
.map(Into::into)
.collect(),
encodings: value.encodings.into_iter().map(Into::into).collect(),
rtcp: value.rtcp.into(),
has_degradation_preference: value.degradation_preference.is_some(),
degradation_preference: value
.degradation_preference
.unwrap_or(DegradationPreference::Balanced),
active: true,
max_bitrate: None,
max_framerate: None,
priority: Priority::Low,
rid: String::default(),
scale_resolution_down_by: None,
}
}
}
livekit-webrtc/src/rtp_receiver.rs
+15 -53
View File
@@ -1,67 +1,29 @@
use crate::media_stream::{MediaStream, MediaStreamTrackHandle};
use crate::rtp_parameters::RtpParameters;
use cxx::SharedPtr;
use std::fmt::{Debug, Formatter};
use webrtc_sys::rtp_receiver as sys_rec;
use webrtc_sys::webrtc as sys_webrtc;
use std::fmt::Debug;
pub use sys_webrtc::ffi::MediaType;
use crate::{
imp::rtp_receiver as imp_rr, media_stream::MediaStreamTrack, rtp_parameters::RtpParameters,
};
#[derive(Clone)]
pub struct RtpReceiver {
cxx_handle: SharedPtr<sys_rec::ffi::RtpReceiver>,
}
impl Debug for RtpReceiver {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
f.debug_struct("RtpReceiver")
.field("track", &self.track())
.field("media_type", &self.media_type())
.field("id", &self.id())
.finish()
}
pub(crate) handle: imp_rr::RtpReceiver,
}
impl RtpReceiver {
pub(crate) fn new(cxx_handle: SharedPtr<sys_rec::ffi::RtpReceiver>) -> Self {
Self { cxx_handle }
}
pub(crate) fn cxx_handle(&self) -> SharedPtr<sys_rec::ffi::RtpReceiver> {
self.cxx_handle.clone()
}
pub fn track(&self) -> MediaStreamTrackHandle {
MediaStreamTrackHandle::new(self.cxx_handle.track())
}
pub fn stream_ids(&self) -> Vec<String> {
self.cxx_handle.stream_ids()
}
pub fn streams(&self) -> Vec<MediaStream> {
let ptrs = self.cxx_handle.streams();
let mut vec = Vec::with_capacity(ptrs.len());
for stream in ptrs {
vec.push(MediaStream::new(stream.ptr));
}
vec
}
pub fn media_type(&self) -> MediaType {
self.cxx_handle.media_type()
}
pub fn id(&self) -> String {
self.cxx_handle.id()
pub fn track(&self) -> Option<MediaStreamTrack> {
self.handle.track()
}
pub fn parameters(&self) -> RtpParameters {
self.cxx_handle.get_parameters().into()
self.handle.parameters()
}
}
pub fn set_jitter_buffer_minimum_delay(&self, delay_seconds: Option<f64>) {
self.cxx_handle
.set_jitter_buffer_minimum_delay(delay_seconds.is_some(), delay_seconds.unwrap_or(0.0));
impl Debug for RtpReceiver {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RtpReceiver")
.field("track", &self.track())
.field("cname", &self.parameters().rtcp.cname)
.finish()
}
}
livekit-webrtc/src/rtp_sender.rs
+21 -65
View File
@@ -1,81 +1,37 @@
use crate::media_stream::{MediaStream, MediaStreamTrackHandle};
use crate::prelude::*;
use crate::rtp_parameters::{RtpEncodingParameters, RtpParameters};
use cxx::SharedPtr;
use std::fmt::{Debug, Formatter};
use webrtc_sys::rtp_sender as sys_rs;
use webrtc_sys::webrtc as sys_webrtc;
use std::fmt::Debug;
pub use sys_webrtc::ffi::MediaType;
use crate::{
imp::rtp_sender as imp_rs, media_stream::MediaStreamTrack, rtp_parameters::RtpParameters,
RtcError,
};
#[derive(Clone)]
pub struct RtpSender {
cxx_handle: SharedPtr<sys_rs::ffi::RtpSender>,
}
impl Debug for RtpSender {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
f.debug_struct("RtpSender")
.field("track", &self.track())
.field("media_type", &self.media_type())
.field("ssrc", &self.ssrc())
.field("id", &self.id())
.finish()
}
pub(crate) handle: imp_rs::RtpSender,
}
impl RtpSender {
pub(crate) fn new(cxx_handle: SharedPtr<sys_rs::ffi::RtpSender>) -> Self {
Self { cxx_handle }
pub fn track(&self) -> Option<MediaStreamTrack> {
self.handle.track()
}
pub(crate) fn cxx_handle(&self) -> SharedPtr<sys_rs::ffi::RtpSender> {
self.cxx_handle.clone()
}
pub fn set_track(&self, track: MediaStreamTrackHandle) -> bool {
self.cxx_handle.set_track(track.cxx_handle())
}
pub fn track(&self) -> MediaStreamTrackHandle {
MediaStreamTrackHandle::new(self.cxx_handle.track())
}
pub fn ssrc(&self) -> u32 {
self.cxx_handle.ssrc()
}
pub fn media_type(&self) -> MediaType {
self.cxx_handle.media_type()
}
pub fn id(&self) -> String {
self.cxx_handle.id()
}
pub fn stream_ids(&self) -> Vec<String> {
self.cxx_handle.stream_ids()
}
pub fn set_streams(&self, stream_ids: &Vec<String>) {
self.cxx_handle.set_streams(stream_ids);
}
pub fn init_send_encodings(&self) -> Vec<RtpEncodingParameters> {
self.cxx_handle
.init_send_encodings()
.into_iter()
.map(Into::into)
.collect()
pub fn set_track(&self, track: Option<MediaStreamTrack>) -> Result<(), RtcError> {
self.handle.set_track(track)
}
pub fn parameters(&self) -> RtpParameters {
self.cxx_handle.get_parameters().into()
self.handle.parameters()
}
pub fn set_parameters(&self, params: RtpParameters) -> Result<(), RTCError> {
self.cxx_handle
.set_parameters(params.into())
.map_err(|e| unsafe { RTCError::from(e.what()) })
pub fn set_parameters(&self, parameters: RtpParameters) -> Result<(), RtcError> {
self.handle.set_parameters(parameters)
}
}
impl Debug for RtpSender {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RtpReceiver")
.field("cname", &self.parameters().rtcp.cname)
.finish()
}
}
livekit-webrtc/src/rtp_transceiver.rs
+38 -89
View File
@@ -1,119 +1,68 @@
use crate::prelude::*;
use cxx::SharedPtr;
use std::fmt::{Debug, Formatter};
use webrtc_sys::rtp_transceiver as sys_rt;
use crate::imp::rtp_transceiver as imp_rt;
use crate::rtp_parameters::{RtpCodecCapability, RtpEncodingParameters};
use crate::rtp_receiver::RtpReceiver;
use crate::rtp_sender::RtpSender;
use crate::RtcError;
use std::fmt::Debug;
#[derive(Debug)]
#[derive(Debug, Clone)]
pub struct RtpTransceiverInit {
pub direction: RtpTransceiverDirection,
pub stream_ids: Vec<String>,
pub send_encodings: Vec<RtpEncodingParameters>,
}
impl From<RtpTransceiverInit> for sys_rt::ffi::RtpTransceiverInit {
fn from(value: RtpTransceiverInit) -> Self {
Self {
direction: value.direction,
stream_ids: value.stream_ids,
send_encodings: value.send_encodings.into_iter().map(Into::into).collect()
}
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum RtpTransceiverDirection {
SendRecv,
SendOnly,
RecvOnly,
Inactive,
Stopped,
}
#[derive(Clone)]
pub struct RtpTransceiver {
cxx_handle: SharedPtr<sys_rt::ffi::RtpTransceiver>,
}
impl Debug for RtpTransceiver {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
f.debug_struct("RtpTransceiver")
.field("media_type", &self.media_type())
.field("mid", &self.mid())
.field("direction", &self.direction())
.field("stopped", &self.stopped())
.field("stopping", &self.stopping())
.finish()
}
pub(crate) handle: imp_rt::RtpTransceiver,
}
impl RtpTransceiver {
pub(crate) fn new(cxx_handle: SharedPtr<sys_rt::ffi::RtpTransceiver>) -> Self {
Self { cxx_handle }
}
pub(crate) fn cxx_handle(&self) -> SharedPtr<sys_rt::ffi::RtpTransceiver> {
self.cxx_handle.clone()
}
pub fn media_type(&self) -> MediaType {
self.cxx_handle.media_type()
}
pub fn mid(&self) -> Option<String> {
self.cxx_handle.mid().ok()
}
pub fn sender(&self) -> RtpSender {
RtpSender::new(self.cxx_handle.sender())
}
pub fn receiver(&self) -> RtpReceiver {
RtpReceiver::new(self.cxx_handle.receiver())
}
pub fn stopped(&self) -> bool {
self.cxx_handle.stopped()
}
pub fn stopping(&self) -> bool {
self.cxx_handle.stopping()
}
pub fn direction(&self) -> RtpTransceiverDirection {
self.cxx_handle.direction()
}
pub fn set_direction(&self, direction: RtpTransceiverDirection) -> Result<(), RTCError> {
self.cxx_handle.set_direction(direction)
.map_err(|e| unsafe { RTCError::from(e.what()) })
self.handle.mid()
}
pub fn current_direction(&self) -> Option<RtpTransceiverDirection> {
self.cxx_handle.current_direction().ok()
self.handle.current_direction()
}
pub fn fired_direction(&self) -> Option<RtpTransceiverDirection> {
self.cxx_handle.fired_direction().ok()
pub fn direction(&self) -> RtpTransceiverDirection {
self.handle.direction()
}
pub fn stop_standard(&self) -> Result<(), RTCError> {
self.cxx_handle.stop_standard()
.map_err(|e| unsafe { RTCError::from(e.what()) })
pub fn sender(&self) -> RtpSender {
self.handle.sender()
}
pub fn set_codec_preferences(&self, codecs: Vec<RtpCodecCapability>) -> Result<(), RTCError> {
let ffi_codecs = codecs.into_iter().map(Into::into).collect();
self.cxx_handle.set_codec_preferences(ffi_codecs)
.map_err(|e| unsafe { RTCError::from(e.what()) })
pub fn receiver(&self) -> RtpReceiver {
self.handle.receiver()
}
pub fn codec_preferences(&self) -> Vec<RtpCodecCapability> {
self.cxx_handle.codec_preferences().into_iter().map(Into::into).collect()
pub fn set_codec_preferences(&self, codecs: Vec<RtpCodecCapability>) -> Result<(), RtcError> {
self.handle.set_codec_preferences(codecs)
}
pub fn header_extensions_to_offer(&self) -> Vec<RtpHeaderExtensionCapability> {
self.cxx_handle.header_extensions_to_offer().into_iter().map(Into::into).collect()
pub fn stop(&self) -> Result<(), RtcError> {
self.handle.stop()
}
}
impl Debug for RtpTransceiver {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RtpTransceiver")
.field("mid", &self.mid())
.field("direction", &self.direction())
.field("sender", &self.sender())
.field("receiver", &self.receiver())
.finish()
}
pub fn header_extensions_negotiated(&self) -> Vec<RtpHeaderExtensionCapability> {
self.cxx_handle.header_extensions_negotiated().into_iter().map(Into::into).collect()
}
pub fn set_offered_rtp_header_extensions(&self, headers: Vec<RtpHeaderExtensionCapability>) -> Result<(), RTCError> {
let ffi_headers = headers.into_iter().map(Into::into).collect();
self.cxx_handle.set_offered_rtp_header_extensions(ffi_headers)
.map_err(|e| unsafe { RTCError::from(e.what()) })
}
}
livekit-webrtc/src/session_description.rs
+61
View File
@@ -0,0 +1,61 @@
use crate::imp::session_description as sd_imp;
use std::{fmt::Debug, str::FromStr};
use thiserror::Error;
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum SdpType {
Offer,
PrAnswer,
Answer,
Rollback,
}
impl FromStr for SdpType {
type Err = &'static str;
fn from_str(sdp_type: &str) -> Result<Self, Self::Err> {
match sdp_type {
"offer" => Ok(Self::Offer),
"pranswer" => Ok(Self::PrAnswer),
"answer" => Ok(Self::Answer),
"rollback" => Ok(Self::Rollback),
_ => Err("invalid SdpType"),
}
}
}
#[derive(Clone)]
pub struct SessionDescription {
pub(crate) handle: sd_imp::SessionDescription,
}
#[derive(Clone, Error, Debug)]
#[error("Failed to parse sdp: {line} - {description}")]
pub struct SdpParseError {
pub line: String,
pub description: String,
}
impl SessionDescription {
pub fn parse(sdp: &str, sdp_type: SdpType) -> Result<Self, SdpParseError> {
sd_imp::SessionDescription::parse(sdp, sdp_type)
}
pub fn sdp_type(&self) -> SdpType {
self.handle.sdp_type()
}
}
impl ToString for SessionDescription {
fn to_string(&self) -> String {
self.handle.to_string()
}
}
impl Debug for SessionDescription {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("SessionDescription")
.field("sdp_type", &self.sdp_type())
.finish()
}
}
livekit-webrtc/src/video_frame.rs
+452 -109
View File
@@ -1,9 +1,14 @@
use crate::video_frame_buffer::VideoFrameBuffer;
use cxx::UniquePtr;
use std::fmt::{Debug, Formatter};
use webrtc_sys::video_frame as vf_sys;
use crate::imp::video_frame as vf_imp;
use std::fmt::Debug;
use thiserror::Error;
#[derive(Debug)]
#[derive(Debug, Error)]
pub enum SinkError {
#[error("platform error: {0}")]
Platform(String),
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum VideoRotation {
VideoRotation0 = 0,
VideoRotation90 = 90,
@@ -11,141 +16,479 @@ pub enum VideoRotation {
VideoRotation270 = 270,
}
impl From<vf_sys::ffi::VideoRotation> for VideoRotation {
fn from(rotation: vf_sys::ffi::VideoRotation) -> Self {
match rotation {
vf_sys::ffi::VideoRotation::VideoRotation0 => Self::VideoRotation0,
vf_sys::ffi::VideoRotation::VideoRotation90 => Self::VideoRotation90,
vf_sys::ffi::VideoRotation::VideoRotation180 => Self::VideoRotation180,
vf_sys::ffi::VideoRotation::VideoRotation270 => Self::VideoRotation270,
_ => unreachable!(),
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum VideoFormatType {
ARGB,
BGRA,
ABGR,
RGBA,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum VideoFrameBufferType {
Native,
I420,
I420A,
I422,
I444,
I010,
NV12,
WebGl,
}
#[derive(Debug)]
pub struct VideoFrame<T>
where
T: VideoFrameBuffer,
{
pub rotation: VideoRotation,
pub timestamp: i64, // When the frame was captured
pub buffer: T,
}
pub type BoxVideoFrame = VideoFrame<Box<dyn VideoFrameBuffer + Send + Sync>>;
macro_rules! new_buffer_type {
($type:ident, $variant:ident, $as:ident) => {
pub struct $type {
pub(crate) handle: vf_imp::$type,
}
impl $crate::video_frame::internal::BufferInternal for $type {
#[cfg(not(target_arch = "wasm32"))]
fn sys_handle(&self) -> &webrtc_sys::video_frame_buffer::ffi::VideoFrameBuffer {
self.handle.sys_handle()
}
#[cfg(not(target_arch = "wasm32"))]
fn to_i420(&self) -> I420Buffer {
I420Buffer {
handle: self.handle.to_i420(),
}
}
#[cfg(not(target_arch = "wasm32"))]
fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
stride: i32,
width: i32,
height: i32,
) -> Result<(), $crate::video_frame::native::ConvertError> {
self.handle.to_argb(format, dst, stride, width, height)
}
}
impl VideoFrameBuffer for $type {
fn width(&self) -> i32 {
self.handle.width()
}
fn height(&self) -> i32 {
self.handle.height()
}
fn buffer_type(&self) -> VideoFrameBufferType {
VideoFrameBufferType::$variant
}
fn $as(&self) -> Option<&$type> {
Some(self)
}
}
impl Debug for $type {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct(stringify!($type))
.field("width", &self.width())
.field("height", &self.height())
.finish()
}
}
};
}
pub(crate) mod internal {
use super::{I420Buffer, VideoFormatType};
pub trait BufferInternal {
#[cfg(not(target_arch = "wasm32"))]
fn sys_handle(&self) -> &webrtc_sys::video_frame_buffer::ffi::VideoFrameBuffer;
#[cfg(not(target_arch = "wasm32"))]
fn to_i420(&self) -> I420Buffer;
#[cfg(not(target_arch = "wasm32"))]
fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: i32,
dst_width: i32,
dst_height: i32,
) -> Result<(), super::native::ConvertError>;
}
}
pub trait VideoFrameBuffer: internal::BufferInternal + Debug {
fn width(&self) -> i32;
fn height(&self) -> i32;
fn buffer_type(&self) -> VideoFrameBufferType;
#[cfg(not(target_arch = "wasm32"))]
fn as_native(&self) -> Option<&native::NativeBuffer> {
None
}
fn as_i420(&self) -> Option<&I420Buffer> {
None
}
fn as_i420a(&self) -> Option<&I420ABuffer> {
None
}
fn as_i422(&self) -> Option<&I422Buffer> {
None
}
fn as_i444(&self) -> Option<&I444Buffer> {
None
}
fn as_i010(&self) -> Option<&I010Buffer> {
None
}
fn as_nv12(&self) -> Option<&NV12Buffer> {
None
}
}
new_buffer_type!(I420Buffer, I420, as_i420);
new_buffer_type!(I420ABuffer, I420A, as_i420a);
new_buffer_type!(I422Buffer, I422, as_i422);
new_buffer_type!(I444Buffer, I444, as_i444);
new_buffer_type!(I010Buffer, I010, as_i010);
new_buffer_type!(NV12Buffer, NV12, as_nv12);
impl I420Buffer {
pub fn chroma_width(&self) -> i32 {
self.handle.chroma_width()
}
pub fn chroma_height(&self) -> i32 {
self.handle.chroma_height()
}
pub fn stride_y(&self) -> i32 {
self.handle.stride_y()
}
pub fn stride_u(&self) -> i32 {
self.handle.stride_u()
}
pub fn stride_v(&self) -> i32 {
self.handle.stride_v()
}
pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
self.handle.data()
}
pub fn data_mut(&mut self) -> (&mut [u8], &mut [u8], &mut [u8]) {
let (data_y, data_u, data_v) = self.handle.data();
unsafe {
(
std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()),
std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u8, data_u.len()),
std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u8, data_v.len()),
)
}
}
}
impl From<VideoRotation> for vf_sys::ffi::VideoRotation {
fn from(rotation: VideoRotation) -> Self {
match rotation {
VideoRotation::VideoRotation0 => Self::VideoRotation0,
VideoRotation::VideoRotation90 => Self::VideoRotation90,
VideoRotation::VideoRotation180 => Self::VideoRotation180,
VideoRotation::VideoRotation270 => Self::VideoRotation270,
impl I420ABuffer {
pub fn chroma_width(&self) -> i32 {
self.handle.chroma_width()
}
pub fn chroma_height(&self) -> i32 {
self.handle.chroma_height()
}
pub fn stride_y(&self) -> i32 {
self.handle.stride_y()
}
pub fn stride_u(&self) -> i32 {
self.handle.stride_u()
}
pub fn stride_v(&self) -> i32 {
self.handle.stride_v()
}
pub fn stride_a(&self) -> i32 {
self.handle.stride_a()
}
pub fn data(&self) -> (&[u8], &[u8], &[u8], Option<&[u8]>) {
self.handle.data()
}
pub fn data_mut(&self) -> (&mut [u8], &mut [u8], &mut [u8], Option<&mut [u8]>) {
let (data_y, data_u, data_v, data_a) = self.handle.data();
unsafe {
(
std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()),
std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u8, data_u.len()),
std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u8, data_v.len()),
data_a.map(|data_a| {
std::slice::from_raw_parts_mut(data_a.as_ptr() as *mut u8, data_a.len())
}),
)
}
}
}
pub struct VideoFrame {
cxx_handle: UniquePtr<vf_sys::ffi::VideoFrame>,
}
impl Debug for VideoFrame {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
f.debug_struct("VideoFrame")
.field("width", &self.width())
.field("height", &self.height())
.field("id", &self.id())
.field("rotation", &self.rotation())
.field("timestamp", &self.timestamp())
.finish()
}
}
impl VideoFrame {
pub(crate) fn new(cxx_handle: UniquePtr<vf_sys::ffi::VideoFrame>) -> Self {
Self { cxx_handle }
impl I422Buffer {
pub fn chroma_width(&self) -> i32 {
self.handle.chroma_width()
}
pub fn width(&self) -> i32 {
self.cxx_handle.width()
pub fn chroma_height(&self) -> i32 {
self.handle.chroma_height()
}
pub fn height(&self) -> i32 {
self.cxx_handle.height()
pub fn stride_y(&self) -> i32 {
self.handle.stride_y()
}
pub fn size(&self) -> u32 {
self.cxx_handle.size()
pub fn stride_u(&self) -> i32 {
self.handle.stride_u()
}
pub fn id(&self) -> u16 {
self.cxx_handle.id()
pub fn stride_v(&self) -> i32 {
self.handle.stride_v()
}
pub fn timestamp_us(&self) -> i64 {
self.cxx_handle.timestamp_us()
pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
self.handle.data()
}
pub fn ntp_time_ms(&self) -> i64 {
self.cxx_handle.ntp_time_ms()
}
pub fn transport_frame_id(&self) -> u32 {
self.cxx_handle.transport_frame_id()
}
pub fn timestamp(&self) -> u32 {
self.cxx_handle.timestamp()
}
pub fn rotation(&self) -> VideoRotation {
self.cxx_handle.rotation().into()
}
/// # Safety
/// Must be called only once, this function create the safe Rust
/// wrapper around a VideoFrameBuffer.
/// Only one wrapper musts exist at a time.
pub(crate) unsafe fn video_frame_buffer(&self) -> VideoFrameBuffer {
VideoFrameBuffer::new(self.cxx_handle.video_frame_buffer())
}
pub fn builder() -> VideoFrameBuilder {
VideoFrameBuilder::default()
}
}
pub struct VideoFrameBuilder {
cxx_handle: UniquePtr<vf_sys::ffi::VideoFrameBuilder>,
}
impl Debug for VideoFrameBuilder {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
f.debug_struct("VideoFrameBuilder").finish()
}
}
impl Default for VideoFrameBuilder {
fn default() -> Self {
Self {
cxx_handle: vf_sys::ffi::create_video_frame_builder(),
pub fn data_mut(&mut self) -> (&mut [u8], &mut [u8], &mut [u8]) {
let (data_y, data_u, data_v) = self.handle.data();
unsafe {
(
std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()),
std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u8, data_u.len()),
std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u8, data_v.len()),
)
}
}
}
impl VideoFrameBuilder {
pub fn set_video_frame_buffer(mut self, buffer: VideoFrameBuffer) -> Self {
self.cxx_handle
.pin_mut()
.set_video_frame_buffer(buffer.release());
self
impl I444Buffer {
pub fn chroma_width(&self) -> i32 {
self.handle.chroma_width()
}
pub fn set_timestamp_us(mut self, ts_us: i64) -> Self {
self.cxx_handle.pin_mut().set_timestamp_us(ts_us);
self
pub fn chroma_height(&self) -> i32 {
self.handle.chroma_height()
}
pub fn set_rotation(mut self, rotation: VideoRotation) -> Self {
self.cxx_handle.pin_mut().set_rotation(rotation.into());
self
pub fn stride_y(&self) -> i32 {
self.handle.stride_y()
}
pub fn set_id(mut self, id: u16) -> Self {
self.cxx_handle.pin_mut().set_id(id);
self
pub fn stride_u(&self) -> i32 {
self.handle.stride_u()
}
pub fn build(mut self) -> VideoFrame {
VideoFrame::new(self.cxx_handle.pin_mut().build())
pub fn stride_v(&self) -> i32 {
self.handle.stride_v()
}
pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
self.handle.data()
}
pub fn data_mut(&mut self) -> (&mut [u8], &mut [u8], &mut [u8]) {
let (data_y, data_u, data_v) = self.handle.data();
unsafe {
(
std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()),
std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u8, data_u.len()),
std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u8, data_v.len()),
)
}
}
}
impl I010Buffer {
pub fn chroma_width(&self) -> i32 {
self.handle.chroma_width()
}
pub fn chroma_height(&self) -> i32 {
self.handle.chroma_height()
}
pub fn stride_y(&self) -> i32 {
self.handle.stride_y()
}
pub fn stride_u(&self) -> i32 {
self.handle.stride_u()
}
pub fn stride_v(&self) -> i32 {
self.handle.stride_v()
}
pub fn data(&self) -> (&[u16], &[u16], &[u16]) {
self.handle.data()
}
pub fn data_mut(&mut self) -> (&mut [u16], &mut [u16], &mut [u16]) {
let (data_y, data_u, data_v) = self.handle.data();
unsafe {
(
std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u16, data_y.len()),
std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u16, data_u.len()),
std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u16, data_v.len()),
)
}
}
}
impl NV12Buffer {
pub fn chroma_width(&self) -> i32 {
self.handle.chroma_width()
}
pub fn chroma_height(&self) -> i32 {
self.handle.chroma_height()
}
pub fn stride_y(&self) -> i32 {
self.handle.stride_y()
}
pub fn stride_uv(&self) -> i32 {
self.handle.stride_uv()
}
pub fn data(&self) -> (&[u8], &[u8]) {
self.handle.data()
}
pub fn data_mut(&mut self) -> (&mut [u8], &mut [u8]) {
let (data_y, data_uv) = self.handle.data();
unsafe {
(
std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()),
std::slice::from_raw_parts_mut(data_uv.as_ptr() as *mut u8, data_uv.len()),
)
}
}
}
#[cfg(not(target_arch = "wasm32"))]
pub mod native {
use super::{vf_imp, I420Buffer, VideoFormatType, VideoFrameBuffer, VideoFrameBufferType};
use std::fmt::Debug;
pub use crate::imp::yuv_helper::ConvertError;
new_buffer_type!(NativeBuffer, Native, as_native);
pub trait I420BufferExt {
fn new(width: u32, height: u32) -> I420Buffer;
}
impl I420BufferExt for I420Buffer {
fn new(width: u32, height: u32) -> I420Buffer {
vf_imp::I420Buffer::new(width, height)
}
}
pub trait VideoFrameBufferExt: VideoFrameBuffer {
fn to_i420(&self) -> I420Buffer;
fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: i32,
dst_width: i32,
dst_height: i32,
) -> Result<(), ConvertError>;
}
impl<T: VideoFrameBuffer> VideoFrameBufferExt for T {
fn to_i420(&self) -> I420Buffer {
self.to_i420()
}
fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: i32,
dst_width: i32,
dst_height: i32,
) -> Result<(), ConvertError> {
self.to_argb(format, dst, dst_stride, dst_width, dst_height)
}
}
}
impl<T: VideoFrameBuffer + ?Sized> internal::BufferInternal for Box<T> {
fn sys_handle(&self) -> &webrtc_sys::video_frame_buffer::ffi::VideoFrameBuffer {
self.as_ref().sys_handle()
}
fn to_i420(&self) -> I420Buffer {
self.as_ref().to_i420()
}
fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: i32,
dst_width: i32,
dst_height: i32,
) -> Result<(), self::native::ConvertError> {
self.as_ref()
.to_argb(format, dst, dst_stride, dst_width, dst_height)
}
}
impl<T: VideoFrameBuffer + ?Sized> VideoFrameBuffer for Box<T> {
fn width(&self) -> i32 {
self.as_ref().width()
}
fn height(&self) -> i32 {
self.as_ref().height()
}
fn buffer_type(&self) -> VideoFrameBufferType {
self.as_ref().buffer_type()
}
}
#[cfg(target_arch = "wasm32")]
pub mod web {
use super::VideoFrameBuffer;
#[derive(Debug)]
pub struct WebGlBuffer {}
impl VideoFrameBuffer for WebGlBuffer {}
}
livekit-webrtc/src/video_frame_buffer.rs
-578
View File
@@ -1,578 +0,0 @@
use cxx::UniquePtr;
use livekit_utils::enum_dispatch;
use std::pin::Pin;
use std::slice;
use webrtc_sys::video_frame_buffer as vfb_sys;
use crate::yuv_helper::{self, ConvertError};
macro_rules! recursive_cast {
($ptr:expr $(, $fnc:ident)*) => {
{
let ptr = $ptr;
$(
let ptr = unsafe { vfb_sys::ffi::$fnc(ptr) };
)*
ptr
}
};
}
#[derive(Debug)]
pub enum VideoFrameBufferType {
Native,
I420,
I420A,
I422,
I444,
I010,
NV12,
}
// types to convert to
#[derive(Debug)]
pub enum VideoFormatType {
ARGB,
BGRA,
ABGR,
RGBA,
}
impl From<vfb_sys::ffi::VideoFrameBufferType> for VideoFrameBufferType {
fn from(buffer_type: vfb_sys::ffi::VideoFrameBufferType) -> Self {
match buffer_type {
vfb_sys::ffi::VideoFrameBufferType::Native => Self::Native,
vfb_sys::ffi::VideoFrameBufferType::I420 => Self::I420,
vfb_sys::ffi::VideoFrameBufferType::I420A => Self::I420A,
vfb_sys::ffi::VideoFrameBufferType::I422 => Self::I422,
vfb_sys::ffi::VideoFrameBufferType::I444 => Self::I444,
vfb_sys::ffi::VideoFrameBufferType::I010 => Self::I010,
vfb_sys::ffi::VideoFrameBufferType::NV12 => Self::NV12,
_ => unreachable!(),
}
}
}
pub trait VideoFrameBufferTrait {
fn buffer_type(&self) -> VideoFrameBufferType; // Useful for the FFI
fn width(&self) -> i32;
fn height(&self) -> i32;
fn to_i420(self) -> I420Buffer;
}
pub trait PlanarYuvBuffer: VideoFrameBufferTrait {
fn chroma_width(&self) -> i32;
fn chroma_height(&self) -> i32;
fn stride_y(&self) -> i32;
fn stride_u(&self) -> i32;
fn stride_v(&self) -> i32;
}
pub trait PlanarYuv8Buffer: PlanarYuvBuffer {
fn data_y(&self) -> &[u8];
fn data_u(&self) -> &[u8];
fn data_v(&self) -> &[u8];
}
pub trait PlanarYuv16BBuffer: PlanarYuvBuffer {
fn data_y(&self) -> &[u16];
fn data_u(&self) -> &[u16];
fn data_v(&self) -> &[u16];
}
pub trait BiplanarYuvBuffer: VideoFrameBufferTrait {
fn chroma_width(&self) -> i32;
fn chroma_height(&self) -> i32;
fn stride_y(&self) -> i32;
fn stride_uv(&self) -> i32;
}
pub trait BiplanarYuv8Buffer: BiplanarYuvBuffer {
fn data_y(&self) -> &[u8];
fn data_uv(&self) -> &[u8];
}
pub enum VideoFrameBuffer {
Native(NativeBuffer),
I420(I420Buffer),
I420A(I420ABuffer),
I422(I422Buffer),
I444(I444Buffer),
I010(I010Buffer),
NV12(NV12Buffer),
}
impl VideoFrameBuffer {
pub(crate) fn new(mut cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>) -> Self {
unsafe {
match cxx_handle.buffer_type().into() {
VideoFrameBufferType::Native => Self::Native(NativeBuffer::from(cxx_handle)),
VideoFrameBufferType::I420 => {
Self::I420(I420Buffer::from(cxx_handle.pin_mut().get_i420()))
}
VideoFrameBufferType::I420A => {
Self::I420A(I420ABuffer::from(cxx_handle.pin_mut().get_i420a()))
}
VideoFrameBufferType::I422 => {
Self::I422(I422Buffer::from(cxx_handle.pin_mut().get_i422()))
}
VideoFrameBufferType::I444 => {
Self::I444(I444Buffer::from(cxx_handle.pin_mut().get_i444()))
}
VideoFrameBufferType::I010 => {
Self::I010(I010Buffer::from(cxx_handle.pin_mut().get_i010()))
}
VideoFrameBufferType::NV12 => {
Self::NV12(NV12Buffer::from(cxx_handle.pin_mut().get_nv12()))
}
}
}
}
#[allow(unused_unsafe)]
pub(crate) fn release(self) -> UniquePtr<vfb_sys::ffi::VideoFrameBuffer> {
unsafe {
match self {
VideoFrameBuffer::Native(native) => native.release(),
VideoFrameBuffer::I420(i420) => UniquePtr::from_raw(recursive_cast!(
i420.release().into_raw(),
i420_to_yuv8,
yuv8_to_yuv,
yuv_to_vfb
) as *mut _),
VideoFrameBuffer::I420A(i420a) => UniquePtr::from_raw(recursive_cast!(
i420a.release().into_raw(),
i420a_to_yuv8,
yuv8_to_yuv,
yuv_to_vfb
) as *mut _),
VideoFrameBuffer::I422(i422) => UniquePtr::from_raw(recursive_cast!(
i422.release().into_raw(),
i422_to_yuv8,
yuv8_to_yuv,
yuv_to_vfb
) as *mut _),
VideoFrameBuffer::I444(i444) => UniquePtr::from_raw(recursive_cast!(
i444.release().into_raw(),
i444_to_yuv8,
yuv8_to_yuv,
yuv_to_vfb
) as *mut _),
VideoFrameBuffer::I010(i010) => UniquePtr::from_raw(recursive_cast!(
i010.release().into_raw(),
i010_to_yuv16b,
yuv16b_to_yuv,
yuv_to_vfb
) as *mut _),
VideoFrameBuffer::NV12(nv12) => UniquePtr::from_raw(recursive_cast!(
nv12.release().into_raw(),
nv12_to_biyuv8,
biyuv8_to_biyuv,
biyuv_to_vfb
) as *mut _),
}
}
}
pub fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: i32,
dst_width: i32,
dst_height: i32,
) -> Result<(), ConvertError> {
match self {
Self::I420(i420) => match format {
VideoFormatType::ARGB => yuv_helper::i420_to_argb(
i420.data_y(),
i420.stride_y(),
i420.data_u(),
i420.stride_u(),
i420.data_v(),
i420.stride_v(),
dst,
dst_stride,
dst_width,
dst_height,
)?,
VideoFormatType::BGRA => yuv_helper::i420_to_bgra(
i420.data_y(),
i420.stride_y(),
i420.data_u(),
i420.stride_u(),
i420.data_v(),
i420.stride_v(),
dst,
dst_stride,
dst_width,
dst_height,
)?,
VideoFormatType::ABGR => yuv_helper::i420_to_abgr(
i420.data_y(),
i420.stride_y(),
i420.data_u(),
i420.stride_u(),
i420.data_v(),
i420.stride_v(),
dst,
dst_stride,
dst_width,
dst_height,
)?,
VideoFormatType::RGBA => yuv_helper::i420_to_rgba(
i420.data_y(),
i420.stride_y(),
i420.data_u(),
i420.stride_u(),
i420.data_v(),
i420.stride_v(),
dst,
dst_stride,
dst_width,
dst_height,
)?,
},
_ => {
// TODO(theomonnom): Support other buffer types
}
};
Ok(())
}
}
impl VideoFrameBufferTrait for VideoFrameBuffer {
enum_dispatch!(
[Native, I420, I420A, I422, I444, I010, NV12]
fnc!(buffer_type, &Self, [], VideoFrameBufferType);
fnc!(width, &Self, [], i32);
fnc!(height, &Self, [], i32);
fnc!(to_i420, Self, [], I420Buffer);
);
}
macro_rules! impl_video_frame_buffer {
($x:ty $(, $cast:ident)*) => {
// Allow unused_unsafe when we don't do any cast ( e.g. NativeBuffer )
#[allow(unused_unsafe)]
impl VideoFrameBufferTrait for $x {
fn buffer_type(&self) -> VideoFrameBufferType {
let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
unsafe {
(*ptr).buffer_type().into()
}
}
fn width(&self) -> i32 {
let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
unsafe {
(*ptr).width()
}
}
fn height(&self) -> i32 {
let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
unsafe {
(*ptr).height()
}
}
// Require ownership because libwebrtc uses the same pointers
fn to_i420(self) -> I420Buffer {
let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*)
as *const vfb_sys::ffi::VideoFrameBuffer
as *mut vfb_sys::ffi::VideoFrameBuffer;
unsafe {
I420Buffer::from(Pin::new_unchecked(&mut *ptr).to_i420())
}
}
}
};
}
macro_rules! impl_yuv_buffer {
($x:ty $(, $cast:ident)*) => {
impl PlanarYuvBuffer for $x {
fn chroma_width(&self) -> i32 {
let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
unsafe {
(*ptr).chroma_width()
}
}
fn chroma_height(&self) -> i32 {
let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
unsafe {
(*ptr).chroma_height()
}
}
fn stride_y(&self) -> i32 {
let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
unsafe {
(*ptr).stride_y()
}
}
fn stride_u(&self) -> i32 {
let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
unsafe {
(*ptr).stride_u()
}
}
fn stride_v(&self) -> i32 {
let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
unsafe {
(*ptr).stride_v()
}
}
}
};
}
macro_rules! impl_yuv8_buffer {
($x:ty $(, $cast:ident)*) => {
impl PlanarYuv8Buffer for $x {
fn data_y(&self) -> &[u8] {
let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
unsafe {
slice::from_raw_parts((*ptr).data_y(), (self.width() * self.height()) as usize)
}
}
fn data_u(&self) -> &[u8] {
let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
unsafe {
let chroma_height = (self.height() + 1) / 2;
slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * chroma_height) as usize)
}
}
fn data_v(&self) -> &[u8] {
let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
unsafe {
let chroma_height = (self.height() + 1) / 2;
slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * chroma_height) as usize)
}
}
}
};
}
macro_rules! impl_yuv16_buffer {
($x:ty $(, $cast:ident)*) => {
impl PlanarYuv16BBuffer for $x {
fn data_y(&self) -> &[u16] {
let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
unsafe {
slice::from_raw_parts((*ptr).data_y(), (self.width() * self.height()) as usize)
}
}
fn data_u(&self) -> &[u16] {
let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
unsafe {
let chroma_height = (self.height() + 1) / 2;
slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * chroma_height) as usize)
}
}
fn data_v(&self) -> &[u16] {
let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
unsafe {
let chroma_height = (self.height() + 1) / 2;
slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * chroma_height) as usize)
}
}
}
};
}
macro_rules! impl_biyuv_buffer {
($x:ty $(, $cast:ident)*) => {
impl BiplanarYuvBuffer for $x {
fn chroma_width(&self) -> i32 {
let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
unsafe {
(*ptr).chroma_width()
}
}
fn chroma_height(&self) -> i32 {
let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
unsafe {
(*ptr).chroma_height()
}
}
fn stride_y(&self) -> i32 {
let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
unsafe {
(*ptr).stride_y()
}
}
fn stride_uv(&self) -> i32 {
let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
unsafe {
(*ptr).stride_uv()
}
}
}
};
}
macro_rules! impl_biyuv8_buffer {
($x:ty $(, $cast:ident)*) => {
impl BiplanarYuv8Buffer for $x {
fn data_y(&self) -> &[u8] {
let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
unsafe {
slice::from_raw_parts((*ptr).data_y(), (self.width() * self.height()) as usize)
}
}
fn data_uv(&self) -> &[u8] {
let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
unsafe {
let chroma_height = (self.height() + 1) / 2;
slice::from_raw_parts((*ptr).data_uv(), (self.stride_uv() * chroma_height) as usize)
}
}
}
};
}
pub struct NativeBuffer {
cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>,
}
pub struct I420Buffer {
cxx_handle: UniquePtr<vfb_sys::ffi::I420Buffer>,
}
pub struct I420ABuffer {
cxx_handle: UniquePtr<vfb_sys::ffi::I420ABuffer>,
}
pub struct I422Buffer {
cxx_handle: UniquePtr<vfb_sys::ffi::I422Buffer>,
}
pub struct I444Buffer {
cxx_handle: UniquePtr<vfb_sys::ffi::I444Buffer>,
}
pub struct I010Buffer {
cxx_handle: UniquePtr<vfb_sys::ffi::I010Buffer>,
}
pub struct NV12Buffer {
cxx_handle: UniquePtr<vfb_sys::ffi::NV12Buffer>,
}
impl_video_frame_buffer!(NativeBuffer);
impl_video_frame_buffer!(I420Buffer, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
impl_video_frame_buffer!(I420ABuffer, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
impl_video_frame_buffer!(I422Buffer, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
impl_video_frame_buffer!(I444Buffer, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
impl_video_frame_buffer!(I010Buffer, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb);
impl_video_frame_buffer!(NV12Buffer, nv12_to_biyuv8, biyuv8_to_biyuv, biyuv_to_vfb);
impl_yuv_buffer!(I420Buffer, i420_to_yuv8, yuv8_to_yuv);
impl_yuv_buffer!(I420ABuffer, i420a_to_yuv8, yuv8_to_yuv);
impl_yuv_buffer!(I422Buffer, i422_to_yuv8, yuv8_to_yuv);
impl_yuv_buffer!(I444Buffer, i444_to_yuv8, yuv8_to_yuv);
impl_yuv_buffer!(I010Buffer, i010_to_yuv16b, yuv16b_to_yuv);
impl_yuv8_buffer!(I420Buffer, i420_to_yuv8);
impl_yuv8_buffer!(I420ABuffer, i420a_to_yuv8);
impl_yuv8_buffer!(I422Buffer, i422_to_yuv8);
impl_yuv8_buffer!(I444Buffer, i444_to_yuv8);
impl_yuv16_buffer!(I010Buffer, i010_to_yuv16b);
impl_biyuv_buffer!(NV12Buffer, nv12_to_biyuv8, biyuv8_to_biyuv);
impl_biyuv8_buffer!(NV12Buffer, nv12_to_biyuv8);
impl NativeBuffer {
fn from(cxx_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>) -> Self {
Self { cxx_handle }
}
pub(crate) fn release(self) -> UniquePtr<vfb_sys::ffi::VideoFrameBuffer> {
self.cxx_handle
}
}
impl I420Buffer {
pub fn new(width: u32, height: u32) -> Self {
Self::from(vfb_sys::ffi::create_i420_buffer(
width as i32,
height as i32,
))
}
fn from(cxx_handle: UniquePtr<vfb_sys::ffi::I420Buffer>) -> Self {
Self { cxx_handle }
}
pub(crate) fn release(self) -> UniquePtr<vfb_sys::ffi::I420Buffer> {
self.cxx_handle
}
}
impl I420ABuffer {
fn from(cxx_handle: UniquePtr<vfb_sys::ffi::I420ABuffer>) -> Self {
Self { cxx_handle }
}
pub(crate) fn release(self) -> UniquePtr<vfb_sys::ffi::I420ABuffer> {
self.cxx_handle
}
}
impl I422Buffer {
fn from(cxx_handle: UniquePtr<vfb_sys::ffi::I422Buffer>) -> Self {
Self { cxx_handle }
}
pub(crate) fn release(self) -> UniquePtr<vfb_sys::ffi::I422Buffer> {
self.cxx_handle
}
}
impl I444Buffer {
fn from(cxx_handle: UniquePtr<vfb_sys::ffi::I444Buffer>) -> Self {
Self { cxx_handle }
}
pub(crate) fn release(self) -> UniquePtr<vfb_sys::ffi::I444Buffer> {
self.cxx_handle
}
}
impl I010Buffer {
fn from(cxx_handle: UniquePtr<vfb_sys::ffi::I010Buffer>) -> Self {
Self { cxx_handle }
}
pub(crate) fn release(self) -> UniquePtr<vfb_sys::ffi::I010Buffer> {
self.cxx_handle
}
}
impl NV12Buffer {
fn from(cxx_handle: UniquePtr<vfb_sys::ffi::NV12Buffer>) -> Self {
Self { cxx_handle }
}
pub(crate) fn release(self) -> UniquePtr<vfb_sys::ffi::NV12Buffer> {
self.cxx_handle
}
}
livekit-webrtc/src/video_source.rs
+28
View File
@@ -0,0 +1,28 @@
use crate::imp::video_source as vs_imp;
#[cfg(not(target_arch = "wasm32"))]
pub mod native {
use super::vs_imp;
use crate::video_frame::{VideoFrame, VideoFrameBuffer};
use std::fmt::{Debug, Formatter};
#[derive(Default, Clone)]
pub struct NativeVideoSource {
pub(crate) handle: vs_imp::NativeVideoSource,
}
impl Debug for NativeVideoSource {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
f.debug_struct("NativeVideoSource").finish()
}
}
impl NativeVideoSource {
pub fn capture_frame<T: VideoFrameBuffer>(&self, frame: &VideoFrame<T>) {
self.handle.capture_frame(frame)
}
}
}
#[cfg(target_arch = "wasm32")]
pub mod web {}
livekit-webrtc/src/video_stream.rs
+54
View File
@@ -0,0 +1,54 @@
use crate::imp::video_stream as stream_imp;
// There is no shared sink between native and web platforms.
// Each platform requires different configuration (e.g: WebGlContext, ..)
#[cfg(not(target_arch = "wasm32"))]
pub mod native {
use super::stream_imp;
use crate::media_stream::RtcVideoTrack;
use crate::video_frame::BoxVideoFrame;
use futures::stream::Stream;
use std::fmt::Debug;
use std::pin::Pin;
use std::task::{Context, Poll};
pub struct NativeVideoStream {
pub(crate) handle: stream_imp::NativeVideoStream,
}
impl Debug for NativeVideoStream {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
f.debug_struct("NativeVideoStream")
.field("track", &self.track())
.finish()
}
}
impl NativeVideoStream {
pub fn new(video_track: RtcVideoTrack) -> Self {
Self {
handle: stream_imp::NativeVideoStream::new(video_track),
}
}
pub fn track(&self) -> RtcVideoTrack {
self.handle.track()
}
pub fn close(&mut self) {
self.handle.close();
}
}
impl Stream for NativeVideoStream {
type Item = BoxVideoFrame;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
Pin::new(&mut self.get_mut().handle).poll_next(cx)
}
}
}
#[cfg(target_arch = "wasm32")]
pub mod web {}
livekit-webrtc/src/web/data_channel.rs
+103
View File
@@ -0,0 +1,103 @@
use core::str;
use std::cell::RefCell;
use std::rc::Rc;
use wasm_bindgen::prelude::*;
use web_sys::{MessageEvent, RtcDataChannelEvent, RtcDataChannelState};
use crate::data_channel::{
DataChannelError, DataChannelTrait, DataState, OnBufferedAmountChange, OnMessage, OnStateChange,
};
impl From<RtcDataChannelState> for DataState {
fn from(value: RtcDataChannelState) -> Self {
match value {
RtcDataChannelState::Connecting => Self::Connecting,
RtcDataChannelState::Open => Self::Open,
RtcDataChannelState::Closing => Self::Closing,
RtcDataChannelState::Closed => Self::Closed,
_ => panic!("unknown data channel state"),
}
}
}
#[derive(Clone)]
pub struct DataChannel {
sys_handle: web_sys::RtcDataChannel,
on_closing: Rc<RefCell<Option<JsValue>>>,
}
impl DataChannelTrait for DataChannel {
fn send(&self, data: &[u8], binary: bool) -> Result<(), DataChannelError> {
if binary {
self.sys_handle
.send_with_u8_array(data)
.map_err(|_| DataChannelError::Send)
} else {
let utf8 = str::from_utf8(data)?;
self.sys_handle
.send_with_str(utf8)
.map_err(|_| DataChannelError::Send)
}
}
fn label(&self) -> String {
self.sys_handle.label()
}
fn state(&self) -> DataState {
self.sys_handle.ready_state().into()
}
fn close(&self) {
self.sys_handle.close();
}
fn on_state_change(&self, callback: Option<OnStateChange>) {
if let Some(mut callback) = callback {
let dc = self.clone();
let js_callback = Closure::new(move |_: RtcDataChannelEvent| {
callback(dc.state());
});
let js_callback = js_callback.into_js_value();
self.sys_handle
.set_onopen(Some(js_callback.unchecked_ref()));
self.sys_handle
.set_onclose(Some(js_callback.unchecked_ref()));
self.sys_handle
.add_event_listener_with_callback("closing", js_callback.unchecked_ref())
.unwrap();
self.on_closing.replace(Some(js_callback));
} else {
self.sys_handle.set_onopen(None);
self.sys_handle.set_onclose(None);
if let Some(on_closing) = self.on_closing.take() {
self.sys_handle
.remove_event_listener_with_callback("closing", on_closing.unchecked_ref())
.unwrap();
}
self.on_closing.replace(None);
}
}
fn on_message(&self, callback: Option<OnMessage>) {
let js_callback = callback.map(|mut callback| {
Closure::new(move |event: MessageEvent| {
if let Some(str) = event.as_string() {
callback(str.as_bytes(), false);
}
})
.into_js_value()
});
self.sys_handle.set_onmessage(
js_callback
.as_ref()
.map(|callback| callback.unchecked_ref()),
);
}
fn on_buffered_amount_change(&self, _callback: Option<OnBufferedAmountChange>) {
todo!("onbufferedamountlow instead?")
}
}
livekit-webrtc/src/web/media_stream.rs
+358
View File
@@ -0,0 +1,358 @@
use super::VideoTrack;
use crate::{
media_stream::{
BiplanarYuv8Buffer, BiplanarYuvBuffer, PlanarYuv16BBuffer, PlanarYuv8Buffer,
PlanarYuvBuffer, VideoFrameBuffer,
},
video_frame::{BiplanarYuv8Buffer, I420Buffer, SinkError, VideoFrame, VideoFrameBuffer},
I010Buffer, I420ABuffer, I422Buffer, I444Buffer, NV12Buffer,
};
use std::sync::mpsc;
use web_sys::{WebGlRenderingContext, WebGlTexture};
#[derive(Debug)]
pub struct WebGlVideoSink {
track: Arc<VideoTrack>,
gl_ctx: WebGlRenderingContext,
tex: WebGlTexture,
}
/// Create a new WebGL texture and update it inside requestAnimationFrame
impl WebGlVideoSink {
pub fn new(
track: Arc<VideoTrack>,
gl_ctx: WebGlRenderingContext,
) -> Result<(Self, mpsc::Receiver<VideoFrame<WebGlBuffer>>), SinkError> {
let (sender, receiver) = mpsc::channel();
let tex = gl_ctx.create_texture()?;
Ok((Self { track, gl_ctx, tex }, receiver))
}
}
#[derive(Debug, Clone)]
pub struct WebGlBuffer {
width: i32,
height: i32,
tex: WebGlTexture,
}
impl VideoFrameBuffer for WebGlBuffer {
fn width(&self) -> i32 {
self.width
}
fn height(&self) -> i32 {
self.height
}
}
/// The following types could be implemented if we want
/// to support VideoFrame with WebCodecs
#[derive(Debug)]
pub struct I420Buffer {}
#[derive(Debug)]
pub struct I420ABuffer {}
#[derive(Debug)]
pub struct I422Buffer {}
#[derive(Debug)]
pub struct I444Buffer {}
#[derive(Debug)]
pub struct I010Buffer {}
#[derive(Debug)]
pub struct NV12Buffer {}
impl VideoFrameBuffer for I420Buffer {
fn width(&self) -> i32 {
unimplemented!()
}
fn height(&self) -> i32 {
unimplemented!()
}
}
impl VideoFrameBuffer for I420ABuffer {
fn width(&self) -> i32 {
unimplemented!()
}
fn height(&self) -> i32 {
unimplemented!()
}
}
impl VideoFrameBuffer for I422Buffer {
fn width(&self) -> i32 {
unimplemented!()
}
fn height(&self) -> i32 {
unimplemented!()
}
}
impl VideoFrameBuffer for I444Buffer {
fn width(&self) -> i32 {
unimplemented!()
}
fn height(&self) -> i32 {
unimplemented!()
}
}
impl VideoFrameBuffer for I010Buffer {
fn width(&self) -> i32 {
unimplemented!()
}
fn height(&self) -> i32 {
unimplemented!()
}
}
impl VideoFrameBuffer for NV12Buffer {
fn width(&self) -> i32 {
unimplemented!()
}
fn height(&self) -> i32 {
unimplemented!()
}
}
impl PlanarYuvBuffer for I420Buffer {
fn chroma_width(&self) -> i32 {
unimplemented!()
}
fn chroma_height(&self) -> i32 {
unimplemented!()
}
fn stride_y(&self) -> i32 {
unimplemented!()
}
fn stride_u(&self) -> i32 {
unimplemented!()
}
fn stride_v(&self) -> i32 {
unimplemented!()
}
}
impl PlanarYuvBuffer for I420ABuffer {
fn chroma_width(&self) -> i32 {
unimplemented!()
}
fn chroma_height(&self) -> i32 {
unimplemented!()
}
fn stride_y(&self) -> i32 {
unimplemented!()
}
fn stride_u(&self) -> i32 {
unimplemented!()
}
fn stride_v(&self) -> i32 {
unimplemented!()
}
}
impl PlanarYuvBuffer for I422Buffer {
fn chroma_width(&self) -> i32 {
unimplemented!()
}
fn chroma_height(&self) -> i32 {
unimplemented!()
}
fn stride_y(&self) -> i32 {
unimplemented!()
}
fn stride_u(&self) -> i32 {
unimplemented!()
}
fn stride_v(&self) -> i32 {
unimplemented!()
}
}
impl PlanarYuvBuffer for I444Buffer {
fn chroma_width(&self) -> i32 {
unimplemented!()
}
fn chroma_height(&self) -> i32 {
unimplemented!()
}
fn stride_y(&self) -> i32 {
unimplemented!()
}
fn stride_u(&self) -> i32 {
unimplemented!()
}
fn stride_v(&self) -> i32 {
unimplemented!()
}
}
impl PlanarYuvBuffer for I010Buffer {
fn chroma_width(&self) -> i32 {
unimplemented!()
}
fn chroma_height(&self) -> i32 {
unimplemented!()
}
fn stride_y(&self) -> i32 {
unimplemented!()
}
fn stride_u(&self) -> i32 {
unimplemented!()
}
fn stride_v(&self) -> i32 {
unimplemented!()
}
}
impl PlanarYuvBuffer for NV12Buffer {
fn chroma_width(&self) -> i32 {
unimplemented!()
}
fn chroma_height(&self) -> i32 {
unimplemented!()
}
fn stride_y(&self) -> i32 {
unimplemented!()
}
fn stride_u(&self) -> i32 {
unimplemented!()
}
fn stride_v(&self) -> i32 {
unimplemented!()
}
}
impl PlanarYuv8Buffer for I420Buffer {
fn data_y(&self) -> &[u8] {
unimplemented!()
}
fn data_u(&self) -> &[u8] {
unimplemented!()
}
fn data_v(&self) -> &[u8] {
unimplemented!()
}
}
impl PlanarYuv8Buffer for I420ABuffer {
fn data_y(&self) -> &[u8] {
unimplemented!()
}
fn data_u(&self) -> &[u8] {
unimplemented!()
}
fn data_v(&self) -> &[u8] {
unimplemented!()
}
}
impl PlanarYuv8Buffer for I422Buffer {
fn data_y(&self) -> &[u8] {
unimplemented!()
}
fn data_u(&self) -> &[u8] {
unimplemented!()
}
fn data_v(&self) -> &[u8] {
unimplemented!()
}
}
impl PlanarYuv8Buffer for I444Buffer {
fn data_y(&self) -> &[u8] {
unimplemented!()
}
fn data_u(&self) -> &[u8] {
unimplemented!()
}
fn data_v(&self) -> &[u8] {
unimplemented!()
}
}
impl PlanarYuv16BBuffer for I010Buffer {
fn data_y(&self) -> &[u16] {
unimplemented!()
}
fn data_u(&self) -> &[u16] {
unimplemented!()
}
fn data_v(&self) -> &[u16] {
unimplemented!()
}
}
impl BiplanarYuvBuffer for NV12Buffer {
fn chroma_width(&self) -> i32 {
unimplemented!()
}
fn chroma_height(&self) -> i32 {
unimplemented!()
}
fn stride_y(&self) -> i32 {
unimplemented!()
}
fn stride_uv(&self) -> i32 {
unimplemented!()
}
}
impl BiplanarYuv8Buffer for NV12Buffer {
fn data_y(&self) -> &[u8] {
unimplemented!()
}
fn data_uv(&self) -> &[u8] {
unimplemented!()
}
}
livekit-webrtc/src/web/mod.rs
+1
View File
@@ -0,0 +1 @@
unimplemented!()
livekit-webrtc/src/webrtc.rs
-24
View File
@@ -1,24 +0,0 @@
use cxx::SharedPtr;
use webrtc_sys::webrtc as sys_rtc;
pub use sys_rtc::ffi::MediaType;
pub use sys_rtc::ffi::Priority;
pub use sys_rtc::ffi::RtpTransceiverDirection;
#[derive(Clone)]
pub struct RTCRuntime {
cxx_handle: SharedPtr<sys_rtc::ffi::RTCRuntime>,
}
impl RTCRuntime {
pub fn new() -> Self {
Self {
cxx_handle: sys_rtc::ffi::create_rtc_runtime(),
}
}
pub(crate) fn release(self) -> SharedPtr<sys_rtc::ffi::RTCRuntime> {
self.cxx_handle
}
}
livekit-webrtc/src/yuv_helper.rs
-98
View File
@@ -1,98 +0,0 @@
use thiserror::Error;
use webrtc_sys::yuv_helper as yuv_sys;
#[derive(Error, Debug)]
pub enum ConvertError {
#[error("conversion failed: {0}")]
Convert(&'static str),
}
fn i420_safety(
src_y: &[u8],
src_stride_y: i32,
src_u: &[u8],
src_stride_u: i32,
src_v: &[u8],
src_stride_v: i32,
dst: &mut [u8],
dst_stride: i32,
_width: i32,
height: i32,
) -> Result<(), ConvertError> {
let chroma_height = (height + 1) / 2;
let min_y = (src_stride_y * height) as usize;
let min_u = (src_stride_u * chroma_height) as usize;
let min_v = (src_stride_v * chroma_height) as usize;
let min_dst = (dst_stride * height) as usize;
if src_y.len() < min_y {
return Err(ConvertError::Convert("src_y isn't large enough"));
}
if src_u.len() < min_u {
return Err(ConvertError::Convert("src_u isn't large enough"));
}
if src_v.len() < min_v {
return Err(ConvertError::Convert("src_v isn't large enough"));
}
if dst.len() < min_dst {
return Err(ConvertError::Convert("dst isn't large enough"));
}
Ok(())
}
macro_rules! i420_to_x {
($x:ident) => {
pub fn $x(
src_y: &[u8],
src_stride_y: i32,
src_u: &[u8],
src_stride_u: i32,
src_v: &[u8],
src_stride_v: i32,
dst: &mut [u8],
dst_stride: i32,
width: i32,
height: i32,
) -> Result<(), ConvertError> {
i420_safety(
src_y,
src_stride_y,
src_u,
src_stride_u,
src_v,
src_stride_v,
dst,
dst_stride,
width,
height,
)?;
unsafe {
yuv_sys::ffi::$x(
src_y.as_ptr(),
src_stride_y,
src_u.as_ptr(),
src_stride_u,
src_v.as_ptr(),
src_stride_v,
dst.as_mut_ptr(),
dst_stride,
width,
height,
);
}
Ok(())
}
};
}
i420_to_x!(i420_to_argb);
i420_to_x!(i420_to_bgra);
i420_to_x!(i420_to_abgr);
i420_to_x!(i420_to_rgba);
livekit/Cargo.toml
+2 -2
View File
@@ -11,9 +11,9 @@ livekit-webrtc = { path = "../livekit-webrtc", version = "0.1.1" }
livekit-utils = { path = "../livekit-utils", version = "0.1.1" }
serde = { version = "1", features = ["derive"] }
serde_json = "1.0"
tokio-tungstenite = { version = "0.17.2", features = ["native-tls"] }
tokio-tungstenite = { version = "0.18", features = ["native-tls"] }
tokio = { version = "1", features = ["full"] }
futures = "0.3"
tokio-stream = "0.1"
parking_lot = { version = "0.12.1", features = ["send_guard"] }
url = "2.2.2"
futures-util = "0.3.23"
livekit/src/lib.rs
+2 -5
View File
@@ -1,12 +1,9 @@
extern crate core;
pub mod proto {
include!(concat!(env!("OUT_DIR"), "/livekit.rs"));
}
pub mod proto;
mod room;
mod rtc_engine;
mod signal_client;
mod room;
pub mod webrtc {
pub use livekit_webrtc::*;
livekit/src/prelude.rs
+5 -24
View File
@@ -1,31 +1,12 @@
pub use crate::participant::{
LocalParticipant, Participant, ParticipantEvent, ParticipantTrait, RemoteParticipant,
};
pub use crate::participant::{LocalParticipant, Participant, ParticipantEvent, RemoteParticipant};
pub use crate::{ConnectionState, Room, RoomError, RoomEvent, RoomSession};
pub use crate::{ConnectionState, Room, RoomError, RoomEvent, RoomResult, RoomSession};
pub use crate::publication::{
LocalTrackPublication, RemoteTrackPublication, TrackPublication, TrackPublicationTrait,
};
pub use crate::publication::{LocalTrackPublication, RemoteTrackPublication, TrackPublication};
pub use crate::track::{
AudioTrackHandle, LocalAudioTrack, LocalTrackHandle, LocalVideoTrack, RemoteAudioTrack,
RemoteTrackHandle, RemoteVideoTrack, StreamState, TrackEvent, TrackHandle, TrackKind,
TrackSource, TrackTrait, VideoTrackHandle,
AudioTrack, LocalAudioTrack, LocalTrack, LocalVideoTrack, RemoteAudioTrack, RemoteTrack,
RemoteVideoTrack, StreamState, Track, TrackEvent, TrackKind, TrackSource, VideoTrack,
};
pub use crate::id::*;
pub use crate::webrtc::{
data_channel::DataChannel,
media_stream::{
AudioTrack, MediaStream, MediaStreamTrackHandle, MediaStreamTrackTrait,
OnConstraintsChangedHandler, OnDiscardedFrameHandler, OnFrameHandler, VideoTrack,
},
rtp_receiver::RtpReceiver,
rtp_transceiver::RtpTransceiver,
video_frame::{VideoFrame, VideoRotation},
video_frame_buffer::{
VideoFormatType, VideoFrameBuffer, VideoFrameBufferTrait, VideoFrameBufferType,
},
};
livekit/src/proto.rs
+49
View File
@@ -0,0 +1,49 @@
use crate::track;
include!(concat!(env!("OUT_DIR"), "/livekit.rs"));
// Conversions
impl TryFrom<TrackType> for track::TrackKind {
type Error = &'static str;
fn try_from(r#type: TrackType) -> Result<Self, Self::Error> {
match r#type {
TrackType::Audio => Ok(Self::Audio),
TrackType::Video => Ok(Self::Video),
TrackType::Data => Err("data tracks are not implemented yet"),
}
}
}
impl From<track::TrackKind> for TrackType {
fn from(kind: track::TrackKind) -> Self {
match kind {
track::TrackKind::Audio => Self::Audio,
track::TrackKind::Video => Self::Video,
}
}
}
impl From<TrackSource> for track::TrackSource {
fn from(source: TrackSource) -> Self {
match source {
TrackSource::Camera => Self::Camera,
TrackSource::Microphone => Self::Microphone,
TrackSource::ScreenShare => Self::Screenshare,
TrackSource::ScreenShareAudio => Self::ScreenshareAudio,
TrackSource::Unknown => Self::Unknown,
}
}
}
impl From<track::TrackSource> for TrackSource {
fn from(source: track::TrackSource) -> Self {
match source {
track::TrackSource::Camera => Self::Camera,
track::TrackSource::Microphone => Self::Microphone,
track::TrackSource::Screenshare => Self::ScreenShare,
track::TrackSource::ScreenshareAudio => Self::ScreenShareAudio,
track::TrackSource::Unknown => Self::Unknown,
}
}
}
livekit/src/room/mod.rs
+14 -10
View File
@@ -1,3 +1,4 @@
use self::track::RemoteTrack;
use crate::participant::ConnectionQuality;
use crate::prelude::*;
use crate::proto;
@@ -10,6 +11,7 @@ use tokio::sync::mpsc;
pub use crate::rtc_engine::SimulateScenario;
pub mod id;
pub mod options;
pub mod participant;
pub mod publication;
pub mod room_session;
@@ -25,34 +27,36 @@ pub enum RoomError {
Engine(#[from] EngineError),
#[error("room failure: {0}")]
Internal(String),
#[error("this track or a track of the same source is already published")]
TrackAlreadyPublished,
}
#[derive(Clone, Debug)]
pub enum RoomEvent {
ParticipantConnected(Arc<RemoteParticipant>),
ParticipantDisconnected(Arc<RemoteParticipant>),
ParticipantConnected(RemoteParticipant),
ParticipantDisconnected(RemoteParticipant),
TrackSubscribed {
track: RemoteTrackHandle,
track: RemoteTrack,
publication: RemoteTrackPublication,
participant: Arc<RemoteParticipant>,
participant: RemoteParticipant,
},
TrackPublished {
publication: RemoteTrackPublication,
participant: Arc<RemoteParticipant>,
participant: RemoteParticipant,
},
TrackUnpublished {
publication: RemoteTrackPublication,
participant: Arc<RemoteParticipant>,
participant: RemoteParticipant,
},
TrackUnsubscribed {
track: RemoteTrackHandle,
track: RemoteTrack,
publication: RemoteTrackPublication,
participant: Arc<RemoteParticipant>,
participant: RemoteParticipant,
},
TrackSubscriptionFailed {
error: track::TrackError,
sid: TrackSid,
participant: Arc<RemoteParticipant>,
participant: RemoteParticipant,
},
TrackMuted {
participant: Participant,
@@ -72,7 +76,7 @@ pub enum RoomEvent {
DataReceived {
payload: Arc<Vec<u8>>,
kind: proto::data_packet::Kind,
participant: Arc<RemoteParticipant>,
participant: RemoteParticipant,
},
ConnectionStateChanged(ConnectionState),
Connected,
livekit/src/room/observer.rs
+1
View File
@@ -0,0 +1 @@
livekit/src/room/options.rs
+372
View File
@@ -0,0 +1,372 @@
use crate::prelude::*;
use crate::proto;
use livekit_webrtc::prelude::*;
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum VideoCodec {
VP8,
H264,
AV1,
}
impl VideoCodec {
pub fn as_str(&self) -> &'static str {
match self {
VideoCodec::VP8 => "vp8",
VideoCodec::H264 => "h264",
VideoCodec::AV1 => "av1",
}
}
}
#[derive(Debug, Clone)]
pub struct VideoResolution {
pub width: u32,
pub height: u32,
pub frame_rate: f64,
pub aspect_ratio: f32,
}
#[derive(Debug, Clone)]
pub struct VideoEncoding {
pub max_bitrate: u64,
pub max_framerate: f64,
}
#[derive(Debug, Clone)]
pub struct VideoPreset {
pub encoding: VideoEncoding,
pub width: u32,
pub height: u32,
}
#[derive(Debug, Clone)]
pub struct AudioPreset {
pub max_bitrate: u32,
}
impl AudioPreset {
pub const fn new(max_bitrate: u32) -> Self {
Self { max_bitrate }
}
}
#[derive(Clone, Debug)]
pub struct VideoCaptureOptions {
pub preset: VideoPreset,
}
impl Default for VideoCaptureOptions {
fn default() -> Self {
Self {
preset: video::H720,
}
}
}
#[derive(Clone, Debug)]
pub struct TrackPublishOptions {
pub dynacast: bool,
pub video_codec: VideoCodec,
pub dtx: bool,
pub red: bool,
pub simulcast: bool,
pub screenshare: bool,
pub name: String,
pub source: TrackSource,
}
impl Default for TrackPublishOptions {
fn default() -> Self {
Self {
dynacast: false,
video_codec: VideoCodec::VP8,
dtx: true,
red: true,
simulcast: true,
screenshare: false,
name: "unnamed track".to_owned(),
source: TrackSource::Unknown,
}
}
}
impl VideoPreset {
pub const fn new(width: u32, height: u32, max_bitrate: u64, max_framerate: f64) -> Self {
Self {
width,
height,
encoding: VideoEncoding {
max_bitrate,
max_framerate,
},
}
}
pub fn resolution(&self) -> VideoResolution {
VideoResolution {
width: self.width,
height: self.height,
frame_rate: self.encoding.max_framerate,
aspect_ratio: self.width as f32 / self.height as f32,
}
}
}
/// Compute appropriate RtpEncodingParameters from the video resolution.
/// TrackPublishOptions helps to find the most appropriate encodings
pub fn compute_video_encodings(
width: u32,
height: u32,
options: &TrackPublishOptions,
) -> Vec<RtpEncodingParameters> {
let encoding = compute_appropriate_encoding(options.screenshare, width, height);
let initial_preset = VideoPreset {
width,
height,
encoding: VideoEncoding {
max_bitrate: encoding.max_bitrate,
max_framerate: encoding.max_framerate,
},
};
if !options.simulcast {
return into_rtp_encodings(width, height, &[initial_preset]);
}
let mut simulcast_presets =
compute_default_simulcast_presets(options.screenshare, &initial_preset);
let mid_preset = simulcast_presets.pop();
let low_preset = simulcast_presets.pop();
let size = u32::max(width, height);
if size >= 960 && low_preset.is_some() {
return into_rtp_encodings(
width,
height,
&[low_preset.unwrap(), mid_preset.unwrap(), initial_preset],
);
} else if size >= 480 {
return into_rtp_encodings(width, height, &[mid_preset.unwrap(), initial_preset]);
}
// Other layers not needed
into_rtp_encodings(width, height, &[initial_preset])
}
/// Return an appropriate VideoEncdoding for the specified resolution based on our presets
pub fn compute_appropriate_encoding(
is_screenshare: bool,
width: u32,
height: u32,
) -> VideoEncoding {
let presets = compute_presets_for_resolution(is_screenshare, width, height);
let size = u32::max(width, height);
for preset in presets {
if preset.width >= size {
return preset.encoding.clone();
}
}
unreachable!()
}
pub fn compute_presets_for_resolution(
is_screenshare: bool,
width: u32,
height: u32,
) -> &'static [VideoPreset] {
if is_screenshare {
return screenshare::PRESETS;
}
// Check how close width & height are from 16/9 or 4/3
let ar = landscape_aspect_ratio(width, height);
if f32::abs(ar - 16.0 / 9.0) < f32::abs(ar - 4.0 / 3.0) {
return video::PRESETS;
}
video43::PRESETS
}
/// Returns our most appropriate default presets
pub fn compute_default_simulcast_presets(
is_screenshare: bool,
initial: &VideoPreset,
) -> Vec<VideoPreset> {
if is_screenshare {
return vec![screenshare::compute_default_simulcast_preset(initial)];
}
let ar = landscape_aspect_ratio(initial.width, initial.height);
if f32::abs(ar - 16.0 / 9.0) < f32::abs(ar - 4.0 / 3.0) {
return video::DEFAULT_SIMULCAST_PRESETS.to_owned();
}
video43::DEFAULT_SIMULCAST_PRESETS.to_owned()
}
pub fn landscape_aspect_ratio(width: u32, height: u32) -> f32 {
if width > height {
width as f32 / height as f32
} else {
height as f32 / width as f32
}
}
/// Presets must be ordered
pub fn into_rtp_encodings(
initial_width: u32,
initial_height: u32,
presets: &[VideoPreset],
) -> Vec<RtpEncodingParameters> {
let mut encodings = Vec::with_capacity(presets.len());
let size = u32::min(initial_width, initial_height);
for (i, preset) in presets.iter().enumerate() {
encodings.push(RtpEncodingParameters {
rid: VIDEO_RIDS[i].to_string(),
scale_resolution_down_by: Some(f64::max(
1.0,
size as f64 / u32::min(preset.width, preset.height) as f64,
)),
max_bitrate: Some(preset.encoding.max_bitrate),
max_framerate: Some(preset.encoding.max_framerate),
..Default::default()
})
}
encodings
}
pub fn video_quality_for_rid(rid: &str) -> Option<proto::VideoQuality> {
match rid {
"f" => Some(proto::VideoQuality::High),
"h" => Some(proto::VideoQuality::Medium),
"q" => Some(proto::VideoQuality::Low),
_ => None,
}
}
pub fn video_layers_from_encodings(
width: u32,
height: u32,
encodings: &[RtpEncodingParameters],
) -> Vec<proto::VideoLayer> {
if encodings.is_empty() {
return vec![proto::VideoLayer {
quality: proto::VideoQuality::High as i32,
width,
height,
bitrate: 0,
ssrc: 0,
}];
}
let mut layers = Vec::with_capacity(encodings.len());
for encoding in encodings {
let scale = encoding.scale_resolution_down_by.unwrap_or(1.0);
let quality = video_quality_for_rid(&encoding.rid).unwrap_or(proto::VideoQuality::High);
layers.push(proto::VideoLayer {
quality: quality as i32,
width: (width as f64 / scale) as u32,
height: (height as f64 / scale) as u32,
bitrate: encoding.max_bitrate.unwrap_or(0) as u32,
ssrc: 0,
});
}
layers
}
const VIDEO_RIDS: &[char] = &['q', 'h', 'f'];
pub mod audio {
use super::AudioPreset;
pub const TELEPHONE: AudioPreset = AudioPreset::new(12_000);
pub const SPEECH: AudioPreset = AudioPreset::new(20_000);
pub const MUSIC: AudioPreset = AudioPreset::new(32_000);
pub const MUSIC_STEREO: AudioPreset = AudioPreset::new(48_000);
pub const MUSIC_HIGH_QUALITY: AudioPreset = AudioPreset::new(64_000);
pub const MUSIC_HIGH_QUALITY_STEREO: AudioPreset = AudioPreset::new(96_000);
pub const PRESETS: &[AudioPreset] = &[
TELEPHONE,
SPEECH,
MUSIC,
MUSIC_STEREO,
MUSIC_HIGH_QUALITY,
MUSIC_HIGH_QUALITY_STEREO,
];
}
pub mod video {
use super::VideoPreset;
pub const H90: VideoPreset = VideoPreset::new(160, 90, 60_000, 15.0);
pub const H180: VideoPreset = VideoPreset::new(320, 180, 120_000, 15.0);
pub const H216: VideoPreset = VideoPreset::new(384, 216, 180_000, 15.0);
pub const H360: VideoPreset = VideoPreset::new(640, 360, 300_000, 20.0);
pub const H540: VideoPreset = VideoPreset::new(960, 540, 600_000, 25.0);
pub const H720: VideoPreset = VideoPreset::new(1280, 720, 1_700_000, 30.0);
pub const H1080: VideoPreset = VideoPreset::new(1920, 1080, 3_000_000, 30.0);
pub const H1440: VideoPreset = VideoPreset::new(2560, 1440, 5_000_000, 30.0);
pub const H2160: VideoPreset = VideoPreset::new(3840, 2160, 8_000_000, 30.0);
pub const PRESETS: &[VideoPreset] = &[H90, H180, H216, H360, H540, H720, H1080, H1440, H2160];
pub const DEFAULT_SIMULCAST_PRESETS: &[VideoPreset] = &[H180, H360];
}
pub mod video43 {
use super::VideoPreset;
pub const H120: VideoPreset = VideoPreset::new(160, 120, 80_000, 15.0);
pub const H180: VideoPreset = VideoPreset::new(240, 180, 100_000, 15.0);
pub const H240: VideoPreset = VideoPreset::new(320, 240, 150_000, 15.0);
pub const H360: VideoPreset = VideoPreset::new(480, 360, 225_000, 20.0);
pub const H480: VideoPreset = VideoPreset::new(640, 480, 300_000, 20.0);
pub const H540: VideoPreset = VideoPreset::new(720, 540, 450_000, 25.0);
pub const H720: VideoPreset = VideoPreset::new(960, 720, 1_500_000, 30.0);
pub const H1080: VideoPreset = VideoPreset::new(1440, 1080, 2_500_000, 30.0);
pub const H1440: VideoPreset = VideoPreset::new(1920, 1440, 3_500_000, 30.0);
pub const PRESETS: &[VideoPreset] = &[H120, H180, H240, H360, H480, H540, H720, H1080, H1440];
pub const DEFAULT_SIMULCAST_PRESETS: &[VideoPreset] = &[H180, H360];
}
pub mod screenshare {
/// The screenshare presets are optimized for quality.
/// When simulcasting, we prefer to reduce the FPS.
use super::VideoPreset;
pub const H360_FPS3: VideoPreset = VideoPreset::new(640, 360, 200_000, 3.0);
pub const H720_FPS5: VideoPreset = VideoPreset::new(1280, 720, 400_000, 5.0);
pub const H720_FPS15: VideoPreset = VideoPreset::new(1280, 720, 1_000_000, 15.0);
pub const H1080_FPS15: VideoPreset = VideoPreset::new(1920, 1080, 1_500_000, 15.0);
pub const H1080_FPS30: VideoPreset = VideoPreset::new(1920, 1080, 3_000_000, 30.0);
pub const PRESETS: &[VideoPreset] =
&[H360_FPS3, H720_FPS5, H720_FPS15, H1080_FPS15, H1080_FPS30];
/// Only one additional layer for screenshares. (Prioritize quality)
pub fn compute_default_simulcast_preset(initial: &VideoPreset) -> VideoPreset {
const SCALE_DOWN_FACTOR: u32 = 2;
const FPS: f64 = 3.0;
VideoPreset::new(
initial.width / SCALE_DOWN_FACTOR,
initial.height / SCALE_DOWN_FACTOR,
u64::max(
150_000,
initial.encoding.max_bitrate as u64
/ (SCALE_DOWN_FACTOR.pow(2) as u64
* (initial.encoding.max_framerate / FPS) as u64),
),
FPS,
)
}
}
livekit/src/room/participant/local_participant.rs
+173 -24
View File
@@ -1,37 +1,130 @@
use super::{
impl_participant_trait, ConnectionQuality, ParticipantInternalTrait, ParticipantShared,
};
use super::{ConnectionQuality, ParticipantInner};
use crate::options::compute_video_encodings;
use crate::options::video_layers_from_encodings;
use crate::options::TrackPublishOptions;
use crate::prelude::*;
use crate::proto;
use crate::publication::TrackPublication;
use crate::rtc_engine::RTCEngine;
use crate::rtc_engine::RtcEngine;
use parking_lot::RwLockReadGuard;
use std::collections::HashMap;
use std::sync::atomic::Ordering;
use std::sync::Arc;
use tokio::sync::mpsc;
use tracing::{debug, trace};
#[derive(Debug)]
#[derive(Debug, Clone)]
pub struct LocalParticipant {
shared: ParticipantShared,
rtc_engine: Arc<RTCEngine>,
inner: Arc<ParticipantInner>,
rtc_engine: Arc<RtcEngine>,
}
impl LocalParticipant {
pub(crate) fn new(
rtc_engine: Arc<RTCEngine>,
rtc_engine: Arc<RtcEngine>,
sid: ParticipantSid,
identity: ParticipantIdentity,
name: String,
metadata: String,
) -> Self {
Self {
shared: ParticipantShared::new(sid, identity, name, metadata),
inner: Arc::new(ParticipantInner::new(sid, identity, name, metadata)),
rtc_engine,
}
}
pub async fn publish_track(&self) {}
pub async fn publish_track(
&self,
track: LocalTrack,
options: TrackPublishOptions,
) -> RoomResult<LocalTrackPublication> {
let mut req = proto::AddTrackRequest {
cid: track.rtc_track().id(),
name: options.name.clone(),
r#type: proto::TrackType::from(track.kind()) as i32,
muted: track.muted(),
source: proto::TrackSource::from(options.source) as i32,
disable_dtx: !options.dtx,
disable_red: !options.red,
..Default::default()
};
let mut encodings = Vec::default();
match &track {
LocalTrack::Video(video_track) => {
// Get the video dimension
// TODO(theomonnom): Use MediaStreamTrack::getSettings() on web
let capture_options = video_track.capture_options();
req.width = capture_options.preset.width;
req.height = capture_options.preset.height;
encodings = compute_video_encodings(req.width, req.height, &options);
req.layers = video_layers_from_encodings(req.width, req.height, &encodings);
}
LocalTrack::Audio(_audio_track) => {}
}
let track_info = self.rtc_engine.add_track(req).await?;
let publication =
LocalTrackPublication::new(track_info.clone(), track.clone(), options.clone());
track.update_info(track_info); // Update SID + Source
debug!("publishing track with cid {:?}", track.rtc_track().id());
let transceiver = self
.rtc_engine
.create_sender(track.clone(), options, encodings)
.await?;
track.update_transceiver(Some(transceiver));
track.start();
tokio::spawn({
let rtc_engine = self.rtc_engine.clone();
async move {
let _ = rtc_engine.negotiate_publisher().await;
}
});
self.inner
.add_track_publication(TrackPublication::Local(publication.clone()));
self.inner
.dispatcher
.dispatch(&ParticipantEvent::LocalTrackPublished {
publication: publication.clone(),
});
Ok(publication)
}
pub async fn unpublish_track(
&self,
track: TrackSid,
stop_on_unpublish: bool,
) -> RoomResult<LocalTrackPublication> {
let mut tracks = self.inner.tracks.write();
if let Some(TrackPublication::Local(publication)) = tracks.remove(&track) {
let track = publication.track().unwrap();
let sender = track.transceiver().unwrap().sender();
self.rtc_engine.remove_track(sender).await?;
track.update_transceiver(None);
self.inner
.dispatcher
.dispatch(&ParticipantEvent::LocalTrackUnpublished {
publication: publication.clone(),
});
publication.update_track(None);
tokio::spawn({
let rtc_engine = self.rtc_engine.clone();
async move {
let _ = rtc_engine.negotiate_publisher().await;
}
});
Ok(publication)
} else {
Err(RoomError::Internal("track not found".to_string()))
}
}
pub async fn publish_data(
&self,
@@ -52,24 +145,80 @@ impl LocalParticipant {
.await
.map_err(Into::into)
}
}
impl ParticipantInternalTrait for LocalParticipant {
fn update_info(self: &Arc<Self>, info: proto::ParticipantInfo, _emit_events: bool) {
self.shared.update_info(info);
#[inline]
pub fn get_track_publication(&self, sid: &TrackSid) -> Option<LocalTrackPublication> {
self.inner.tracks.read().get(sid).map(|track| {
if let TrackPublication::Local(local) = track {
return local.clone();
}
unreachable!()
})
}
fn set_speaking(&self, speaking: bool) {
self.shared.set_speaking(speaking);
#[inline]
pub fn sid(&self) -> ParticipantSid {
self.inner.sid()
}
fn set_audio_level(&self, level: f32) {
self.shared.set_audio_level(level);
#[inline]
pub fn identity(&self) -> ParticipantIdentity {
self.inner.identity()
}
fn set_connection_quality(&self, quality: ConnectionQuality) {
self.shared.set_connection_quality(quality);
#[inline]
pub fn name(&self) -> String {
self.inner.name()
}
#[inline]
pub fn metadata(&self) -> String {
self.inner.metadata()
}
#[inline]
pub fn is_speaking(&self) -> bool {
self.inner.is_speaking()
}
#[inline]
pub fn tracks(&self) -> RwLockReadGuard<HashMap<TrackSid, TrackPublication>> {
self.inner.tracks()
}
#[inline]
pub fn audio_level(&self) -> f32 {
self.inner.audio_level()
}
#[inline]
pub fn connection_quality(&self) -> ConnectionQuality {
self.inner.connection_quality()
}
#[inline]
pub fn register_observer(&self) -> mpsc::UnboundedReceiver<ParticipantEvent> {
self.inner.register_observer()
}
#[inline]
pub(crate) fn update_info(self: &Self, info: proto::ParticipantInfo) {
self.inner.update_info(info);
}
#[inline]
pub(crate) fn set_speaking(&self, speaking: bool) {
self.inner.set_speaking(speaking);
}
#[inline]
pub(crate) fn set_audio_level(&self, level: f32) {
self.inner.set_audio_level(level);
}
#[inline]
pub(crate) fn set_connection_quality(&self, quality: ConnectionQuality) {
self.inner.set_connection_quality(quality);
}
}
impl_participant_trait!(LocalParticipant);
livekit/src/room/participant/mod.rs
+89 -120
View File
@@ -24,11 +24,11 @@ pub enum ParticipantEvent {
publication: RemoteTrackPublication,
},
TrackSubscribed {
track: RemoteTrackHandle,
track: RemoteTrack,
publication: RemoteTrackPublication,
},
TrackUnsubscribed {
track: RemoteTrackHandle,
track: RemoteTrack,
publication: RemoteTrackPublication,
},
TrackSubscriptionFailed {
@@ -51,6 +51,12 @@ pub enum ParticipantEvent {
ConnectionQualityChanged {
quality: ConnectionQuality,
},
LocalTrackPublished {
publication: LocalTrackPublication,
},
LocalTrackUnpublished {
publication: LocalTrackPublication,
},
}
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
@@ -83,21 +89,48 @@ impl From<proto::ConnectionQuality> for ConnectionQuality {
}
}
#[derive(Debug)]
pub(super) struct ParticipantShared {
pub(super) sid: Mutex<ParticipantSid>,
pub(super) identity: Mutex<ParticipantIdentity>,
pub(super) name: Mutex<String>,
pub(super) metadata: Mutex<String>,
pub(super) tracks: RwLock<HashMap<TrackSid, TrackPublication>>,
pub(super) speaking: AtomicBool,
pub(super) audio_level: AtomicU32,
pub(super) connection_quality: AtomicU8,
pub(super) dispatcher: Mutex<Dispatcher<ParticipantEvent>>,
#[derive(Debug, Clone)]
pub enum Participant {
Local(LocalParticipant),
Remote(RemoteParticipant),
}
impl ParticipantShared {
pub(super) fn new(
impl Participant {
enum_dispatch!(
[Local, Remote];
pub fn sid(self: &Self) -> ParticipantSid;
pub fn identity(self: &Self) -> ParticipantIdentity;
pub fn name(self: &Self) -> String;
pub fn metadata(self: &Self) -> String;
pub fn is_speaking(self: &Self) -> bool;
pub fn audio_level(self: &Self) -> f32;
pub fn connection_quality(self: &Self) -> ConnectionQuality;
pub fn tracks(self: &Self) -> RwLockReadGuard<HashMap<TrackSid, TrackPublication>>;
pub fn register_observer(self: &Self) -> mpsc::UnboundedReceiver<ParticipantEvent>;
// Internal functions
pub(crate) fn set_speaking(self: &Self, speaking: bool) -> ();
pub(crate) fn set_audio_level(self: &Self, level: f32) -> ();
pub(crate) fn set_connection_quality(self: &Self, quality: ConnectionQuality) -> ();
pub(crate) fn update_info(self: &Self, info: proto::ParticipantInfo) -> ();
);
}
#[derive(Debug)]
pub(crate) struct ParticipantInner {
pub sid: Mutex<ParticipantSid>,
pub identity: Mutex<ParticipantIdentity>,
pub name: Mutex<String>,
pub metadata: Mutex<String>,
pub speaking: AtomicBool,
pub tracks: RwLock<HashMap<TrackSid, TrackPublication>>,
pub audio_level: AtomicU32,
pub connection_quality: AtomicU8,
pub dispatcher: Dispatcher<ParticipantEvent>,
}
impl ParticipantInner {
pub fn new(
sid: ParticipantSid,
identity: ParticipantIdentity,
name: String,
@@ -116,128 +149,64 @@ impl ParticipantShared {
}
}
pub(crate) fn update_info(&self, info: proto::ParticipantInfo) {
pub fn sid(&self) -> ParticipantSid {
self.sid.lock().clone()
}
pub fn identity(&self) -> ParticipantIdentity {
self.identity.lock().clone()
}
pub fn name(&self) -> String {
self.name.lock().clone()
}
pub fn metadata(&self) -> String {
self.metadata.lock().clone()
}
pub fn is_speaking(&self) -> bool {
self.speaking.load(Ordering::SeqCst)
}
pub fn tracks(&self) -> RwLockReadGuard<HashMap<TrackSid, TrackPublication>> {
self.tracks.read()
}
pub fn audio_level(&self) -> f32 {
f32::from_bits(self.audio_level.load(Ordering::SeqCst))
}
pub fn connection_quality(&self) -> ConnectionQuality {
self.connection_quality.load(Ordering::SeqCst).into()
}
pub fn register_observer(&self) -> mpsc::UnboundedReceiver<ParticipantEvent> {
self.dispatcher.register()
}
pub fn update_info(&self, info: proto::ParticipantInfo) {
*self.sid.lock() = info.sid.into();
*self.identity.lock() = info.identity.into();
*self.name.lock() = info.name;
*self.metadata.lock() = info.metadata; // TODO(theomonnom): callback MetadataChanged
}
pub(crate) fn set_speaking(&self, speaking: bool) {
pub fn set_speaking(&self, speaking: bool) {
self.speaking.store(speaking, Ordering::SeqCst);
}
pub(crate) fn set_audio_level(&self, audio_level: f32) {
pub fn set_audio_level(&self, audio_level: f32) {
self.audio_level
.store(audio_level.to_bits(), Ordering::SeqCst)
}
pub(crate) fn register_observer(&self) -> mpsc::UnboundedReceiver<ParticipantEvent> {
self.dispatcher.lock().register()
}
pub(crate) fn set_connection_quality(&self, quality: ConnectionQuality) {
pub fn set_connection_quality(&self, quality: ConnectionQuality) {
self.connection_quality
.store(quality as u8, Ordering::SeqCst);
}
pub(crate) fn add_track_publication(&self, publication: TrackPublication) {
pub fn add_track_publication(&self, publication: TrackPublication) {
self.tracks.write().insert(publication.sid(), publication);
}
}
pub(crate) trait ParticipantInternalTrait {
fn set_speaking(&self, speaking: bool);
fn set_audio_level(&self, level: f32);
fn set_connection_quality(&self, quality: ConnectionQuality);
fn update_info(self: &Arc<Self>, info: proto::ParticipantInfo, emit_events: bool);
}
pub trait ParticipantTrait {
fn sid(&self) -> ParticipantSid;
fn identity(&self) -> ParticipantIdentity;
fn name(&self) -> String;
fn metadata(&self) -> String;
fn is_speaking(&self) -> bool;
fn audio_level(&self) -> f32;
fn connection_quality(&self) -> ConnectionQuality;
fn tracks(&self) -> RwLockReadGuard<HashMap<TrackSid, TrackPublication>>;
fn register_observer(&self) -> mpsc::UnboundedReceiver<ParticipantEvent>;
}
#[derive(Debug, Clone)]
pub enum Participant {
Local(Arc<LocalParticipant>),
Remote(Arc<RemoteParticipant>),
}
// TODO(theomonnom): Should I provide a WeakParticipant here ?
impl Participant {
enum_dispatch!(
[Local, Remote]
fnc!(pub(crate), update_info, &Self, [info: proto::ParticipantInfo, emit_events: bool], ());
fnc!(pub(crate), set_speaking, &Self, [speaking: bool], ());
fnc!(pub(crate), set_audio_level, &Self, [audio_level: f32], ());
fnc!(pub(crate), set_connection_quality, &Self, [quality: ConnectionQuality], ());
);
}
impl ParticipantTrait for Participant {
enum_dispatch!(
[Local, Remote]
fnc!(sid, &Self, [], ParticipantSid);
fnc!(identity, &Self, [], ParticipantIdentity);
fnc!(name, &Self, [], String);
fnc!(metadata, &Self, [], String);
fnc!(is_speaking, &Self, [], bool);
fnc!(audio_level, &Self, [], f32);
fnc!(connection_quality, &Self, [], ConnectionQuality);
fnc!(tracks, &Self, [], RwLockReadGuard<HashMap<TrackSid, TrackPublication>>);
fnc!(register_observer, &Self, [], mpsc::UnboundedReceiver<ParticipantEvent>);
);
}
macro_rules! impl_participant_trait {
($x:ty) => {
impl crate::room::participant::ParticipantTrait for $x {
fn sid(&self) -> ParticipantSid {
self.shared.sid.lock().clone()
}
fn identity(&self) -> ParticipantIdentity {
self.shared.identity.lock().clone()
}
fn name(&self) -> String {
self.shared.name.lock().clone()
}
fn metadata(&self) -> String {
self.shared.metadata.lock().clone()
}
fn is_speaking(&self) -> bool {
self.shared.speaking.load(Ordering::SeqCst)
}
fn audio_level(&self) -> f32 {
f32::from_bits(self.shared.audio_level.load(Ordering::SeqCst))
}
fn connection_quality(&self) -> ConnectionQuality {
self.shared.connection_quality.load(Ordering::SeqCst).into()
}
fn tracks(&self) -> RwLockReadGuard<HashMap<TrackSid, TrackPublication>> {
self.shared.tracks.read()
}
fn register_observer(&self) -> mpsc::UnboundedReceiver<ParticipantEvent> {
self.shared.register_observer()
}
}
};
}
pub(super) use impl_participant_trait;
livekit/src/room/participant/remote_participant.rs
+104 -65
View File
@@ -1,15 +1,11 @@
use super::{
impl_participant_trait, ConnectionQuality, ParticipantInternalTrait, ParticipantShared,
};
use super::{ConnectionQuality, ParticipantInner};
use crate::prelude::*;
use crate::proto;
use crate::publication::TrackPublicationInternalTrait;
use crate::track::TrackError;
use livekit_webrtc::prelude::*;
use livekit_webrtc as rtc;
use parking_lot::RwLockReadGuard;
use std::collections::HashMap;
use std::collections::HashSet;
use std::sync::atomic::Ordering;
use rtc::prelude::MediaStreamTrack;
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::mpsc;
@@ -18,9 +14,9 @@ use tracing::{debug, error, instrument, Level};
const ADD_TRACK_TIMEOUT: Duration = Duration::from_secs(5);
#[derive(Debug)]
#[derive(Debug, Clone)]
pub struct RemoteParticipant {
shared: ParticipantShared,
inner: Arc<ParticipantInner>,
}
impl RemoteParticipant {
@@ -31,26 +27,25 @@ impl RemoteParticipant {
metadata: String,
) -> Self {
Self {
shared: ParticipantShared::new(sid, identity, name, metadata),
inner: Arc::new(ParticipantInner::new(sid, identity, name, metadata)),
}
}
fn get_track_publication(&self, sid: &TrackSid) -> Option<RemoteTrackPublication> {
self.shared.tracks.read().get(sid).map(|track| {
#[inline]
pub fn get_track_publication(&self, sid: &TrackSid) -> Option<RemoteTrackPublication> {
self.inner.tracks.read().get(sid).map(|track| {
if let TrackPublication::Remote(remote) = track {
remote.clone()
} else {
unreachable!()
return remote.clone();
}
unreachable!()
})
}
/// Called by the RoomSession when receiving data by the RTCSession
/// Called by the RoomSession when receiving data from the RrcSession
/// It is just used to emit the Data event on the participant dispatcher.
pub(crate) fn on_data_received(&self, data: Arc<Vec<u8>>, kind: proto::data_packet::Kind) {
self.shared
self.inner
.dispatcher
.lock()
.dispatch(&ParticipantEvent::DataReceived {
payload: data,
kind,
@@ -59,9 +54,9 @@ impl RemoteParticipant {
#[instrument(level = Level::DEBUG)]
pub(crate) async fn add_subscribed_media_track(
self: Arc<Self>,
&self,
sid: TrackSid,
media_track: MediaStreamTrackHandle,
media_track: rtc::media_stream::MediaStreamTrack,
) {
let wait_publication = {
let participant = self.clone();
@@ -73,7 +68,7 @@ impl RemoteParticipant {
return publication;
}
tokio::task::yield_now().await;
tokio::task::yield_now().await; // Remove yield
}
}
};
@@ -81,25 +76,25 @@ impl RemoteParticipant {
if let Ok(remote_publication) = timeout(ADD_TRACK_TIMEOUT, wait_publication).await {
let track = match remote_publication.kind() {
TrackKind::Audio => {
if let MediaStreamTrackHandle::Audio(rtc_track) = media_track {
if let MediaStreamTrack::Audio(rtc_track) = media_track {
let audio_track = RemoteAudioTrack::new(
remote_publication.sid().into(),
remote_publication.name(),
rtc_track,
);
RemoteTrackHandle::Audio(Arc::new(audio_track))
RemoteTrack::Audio(audio_track)
} else {
unreachable!();
}
}
TrackKind::Video => {
if let MediaStreamTrackHandle::Video(rtc_track) = media_track {
if let MediaStreamTrack::Video(rtc_track) = media_track {
let video_track = RemoteVideoTrack::new(
remote_publication.sid().into(),
remote_publication.name(),
rtc_track,
);
RemoteTrackHandle::Video(Arc::new(video_track))
RemoteTrack::Video(video_track)
} else {
unreachable!()
}
@@ -110,23 +105,30 @@ impl RemoteParticipant {
debug!("starting track: {:?}", sid);
remote_publication.update_track(Some(track.clone().into()));
self.shared
track.set_muted(remote_publication.muted());
track.update_info(proto::TrackInfo {
sid: remote_publication.sid().to_string(),
name: remote_publication.name().to_string(),
r#type: proto::TrackType::from(remote_publication.kind()) as i32,
source: proto::TrackSource::from(remote_publication.source()) as i32,
..Default::default()
});
self.inner
.add_track_publication(TrackPublication::Remote(remote_publication.clone()));
track.start();
self.shared
self.inner
.dispatcher
.lock()
.dispatch(&ParticipantEvent::TrackSubscribed {
track: track,
track,
publication: remote_publication,
});
} else {
error!("could not find published track with sid: {:?}", sid);
self.shared
self.inner
.dispatcher
.lock()
.dispatch(&ParticipantEvent::TrackSubscriptionFailed {
sid: sid.clone(),
error: TrackError::TrackNotFound(sid.clone().to_string()),
@@ -134,81 +136,118 @@ impl RemoteParticipant {
}
}
pub(crate) fn unpublish_track(self: &Arc<Self>, sid: &TrackSid, emit_events: bool) {
pub(crate) fn unpublish_track(&self, sid: &TrackSid) {
if let Some(publication) = self.get_track_publication(sid) {
// Unsubscribe to the track if needed
if let Some(track) = publication.track() {
track.stop();
self.shared
self.inner
.dispatcher
.lock()
.dispatch(&ParticipantEvent::TrackUnsubscribed {
track: track.clone(),
publication: publication.clone(),
});
}
if emit_events {
self.shared
.dispatcher
.lock()
.dispatch(&ParticipantEvent::TrackUnpublished {
publication: publication.clone(),
});
}
self.inner
.dispatcher
.dispatch(&ParticipantEvent::TrackUnpublished {
publication: publication.clone(),
});
publication.update_track(None);
}
}
}
impl ParticipantInternalTrait for RemoteParticipant {
fn update_info(self: &Arc<Self>, info: proto::ParticipantInfo, emit_events: bool) {
self.shared.update_info(info.clone());
pub(crate) fn update_info(&self, info: proto::ParticipantInfo) {
self.inner.update_info(info.clone());
let mut valid_tracks = HashSet::<TrackSid>::new();
for track in info.tracks {
if let Some(publication) = self.get_track_publication(&track.sid.clone().into()) {
publication.update_info(track.clone());
} else {
let publication = RemoteTrackPublication::new(track.clone(), self.sid(), None);
self.shared
let publication = RemoteTrackPublication::new(track.clone(), None);
self.inner
.add_track_publication(TrackPublication::Remote(publication.clone()));
// This is a new track, dispatch publish event
if emit_events {
self.shared
.dispatcher
.lock()
.dispatch(&ParticipantEvent::TrackPublished { publication });
}
self.inner
.dispatcher
.dispatch(&ParticipantEvent::TrackPublished { publication });
}
valid_tracks.insert(track.sid.into());
}
// remove tracks that are no longer valid
for (sid, _) in self.shared.tracks.read().iter() {
for (sid, _) in self.inner.tracks.read().iter() {
if valid_tracks.contains(sid) {
continue;
}
self.unpublish_track(sid, emit_events);
self.unpublish_track(sid);
}
}
fn set_speaking(&self, speaking: bool) {
self.shared.set_speaking(speaking);
#[inline]
pub fn sid(&self) -> ParticipantSid {
self.inner.sid()
}
fn set_audio_level(&self, level: f32) {
self.shared.set_audio_level(level);
#[inline]
pub fn identity(&self) -> ParticipantIdentity {
self.inner.identity()
}
fn set_connection_quality(&self, quality: ConnectionQuality) {
self.shared.set_connection_quality(quality);
#[inline]
pub fn name(&self) -> String {
self.inner.name()
}
#[inline]
pub fn metadata(&self) -> String {
self.inner.metadata()
}
#[inline]
pub fn is_speaking(&self) -> bool {
self.inner.is_speaking()
}
#[inline]
pub fn tracks(&self) -> RwLockReadGuard<HashMap<TrackSid, TrackPublication>> {
self.inner.tracks()
}
#[inline]
pub fn audio_level(&self) -> f32 {
self.inner.audio_level()
}
#[inline]
pub fn connection_quality(&self) -> ConnectionQuality {
self.inner.connection_quality()
}
#[inline]
pub fn register_observer(&self) -> mpsc::UnboundedReceiver<ParticipantEvent> {
self.inner.register_observer()
}
#[inline]
pub(crate) fn set_speaking(&self, speaking: bool) {
self.inner.set_speaking(speaking);
}
#[inline]
pub(crate) fn set_audio_level(&self, level: f32) {
self.inner.set_audio_level(level);
}
#[inline]
pub(crate) fn set_connection_quality(&self, quality: ConnectionQuality) {
self.inner.set_connection_quality(quality);
}
}
impl_participant_trait!(RemoteParticipant);
livekit/src/room/publication/local.rs
+91
View File
@@ -0,0 +1,91 @@
use super::TrackPublicationInner;
use crate::id::TrackSid;
use crate::options::TrackPublishOptions;
use crate::proto;
use crate::track::{LocalTrack, Track, TrackDimension, TrackKind, TrackSource};
use parking_lot::Mutex;
use std::sync::Arc;
#[derive(Debug)]
struct LocalTrackPublicationInner {
publication_inner: TrackPublicationInner,
options: Mutex<TrackPublishOptions>,
}
#[derive(Clone, Debug)]
pub struct LocalTrackPublication {
inner: Arc<LocalTrackPublicationInner>,
}
impl LocalTrackPublication {
pub(crate) fn new(
info: proto::TrackInfo,
track: LocalTrack,
options: TrackPublishOptions,
) -> Self {
Self {
inner: Arc::new(LocalTrackPublicationInner {
publication_inner: TrackPublicationInner::new(info, Some(track.into())),
options: Mutex::new(options),
}),
}
}
#[inline]
pub fn sid(&self) -> TrackSid {
self.inner.publication_inner.sid()
}
#[inline]
pub fn name(&self) -> String {
self.inner.publication_inner.name()
}
#[inline]
pub fn kind(&self) -> TrackKind {
self.inner.publication_inner.kind()
}
#[inline]
pub fn source(&self) -> TrackSource {
self.inner.publication_inner.source()
}
#[inline]
pub fn simulcasted(&self) -> bool {
self.inner.publication_inner.simulcasted()
}
#[inline]
pub fn dimension(&self) -> TrackDimension {
self.inner.publication_inner.dimension()
}
#[inline]
pub fn track(&self) -> Option<LocalTrack> {
self.inner
.publication_inner
.track()
.map(|track| track.try_into().unwrap())
}
#[inline]
pub fn mime_type(&self) -> String {
self.inner.publication_inner.mime_type()
}
#[inline]
pub fn muted(&self) -> bool {
self.inner.publication_inner.muted()
}
#[inline]
pub(crate) fn update_track(&self, track: Option<Track>) {
self.inner.publication_inner.update_track(track);
}
#[inline]
pub(crate) fn update_info(&self, info: proto::TrackInfo) {
self.inner.publication_inner.update_info(info);
}
}
livekit/src/room/publication/mod.rs
+95 -205
View File
@@ -1,58 +1,46 @@
use super::track::{TrackDimension, TrackEvent};
use crate::prelude::*;
use crate::proto;
use crate::track::Track;
use futures_util::stream::StreamExt;
use livekit_utils::enum_dispatch;
use livekit_utils::observer::Dispatcher;
use parking_lot::Mutex;
use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
use std::sync::Arc;
use tokio::sync::{mpsc, oneshot};
use tokio::sync::Notify;
use tokio_stream::wrappers::UnboundedReceiverStream;
use super::track::{TrackDimension, TrackEvent};
mod local;
pub use local::*;
pub(crate) trait TrackPublicationInternalTrait {
fn update_track(&self, track: Option<TrackHandle>);
fn update_info(&self, info: proto::TrackInfo);
}
pub trait TrackPublicationTrait {
fn name(&self) -> String;
fn sid(&self) -> TrackSid;
fn kind(&self) -> TrackKind;
fn source(&self) -> TrackSource;
fn simulcasted(&self) -> bool;
fn dimension(&self) -> TrackDimension;
fn mime_type(&self) -> String;
fn muted(&self) -> bool;
}
mod remote;
pub use remote::*;
#[derive(Debug)]
pub(super) struct TrackPublicationShared {
pub(super) track: Mutex<Option<TrackHandle>>,
pub(super) name: Mutex<String>,
pub(super) sid: Mutex<TrackSid>,
pub(super) kind: AtomicU8, // Casted to TrackKind
pub(super) source: AtomicU8, // Casted to TrackSource
pub(super) simulcasted: AtomicBool,
pub(super) dimension: Mutex<TrackDimension>,
pub(super) mime_type: Mutex<String>,
pub(super) muted: AtomicBool,
pub(super) participant: ParticipantSid,
pub(super) dispatcher: Mutex<Dispatcher<TrackEvent>>,
pub(super) close_sender: Mutex<Option<oneshot::Sender<()>>>,
pub(crate) struct TrackPublicationInner {
track: Mutex<Option<Track>>,
name: Mutex<String>,
sid: Mutex<TrackSid>,
kind: AtomicU8, // Casted to TrackKind
source: AtomicU8, // Casted to TrackSource
simulcasted: AtomicBool,
dimension: Mutex<TrackDimension>,
mime_type: Mutex<String>,
muted: AtomicBool,
dispatcher: Dispatcher<TrackEvent>,
close_notifier: Arc<Notify>,
}
impl TrackPublicationShared {
pub fn new(
info: proto::TrackInfo,
participant: ParticipantSid,
track: Option<TrackHandle>,
) -> Arc<Self> {
Arc::new(Self {
impl TrackPublicationInner {
pub fn new(info: proto::TrackInfo, track: Option<Track>) -> Self {
Self {
track: Mutex::new(track),
name: Mutex::new(info.name),
sid: Mutex::new(info.sid.into()),
kind: AtomicU8::new(
TrackKind::from(proto::TrackType::from_i32(info.r#type).unwrap()) as u8,
TrackKind::try_from(proto::TrackType::from_i32(info.r#type).unwrap()).unwrap()
as u8,
),
source: AtomicU8::new(TrackSource::from(
proto::TrackSource::from_i32(info.source).unwrap(),
@@ -62,46 +50,32 @@ impl TrackPublicationShared {
mime_type: Mutex::new(info.mime_type),
muted: AtomicBool::new(info.muted),
dispatcher: Default::default(),
close_sender: Default::default(),
participant,
})
close_notifier: Default::default(),
}
}
pub fn update_track(self: &Arc<Self>, track: Option<TrackHandle>) {
pub fn update_track(&self, track: Option<Track>) {
let mut old_track = self.track.lock();
if let Some(close_sender) = self.close_sender.lock().take() {
let _ = close_sender.send(());
}
*old_track = track.clone();
if let Some(track) = track {
let (close_sender, close_receiver) = oneshot::channel();
self.close_sender.lock().replace(close_sender);
let track_receiver = track.register_observer();
tokio::spawn(
self.clone()
.publication_task(close_receiver, track_receiver),
);
}
}
self.close_notifier.notify_waiters();
/// Task used to forward TrackHandle's events to the TrackPublications's dispatcher
async fn publication_task(
self: Arc<Self>,
mut close_receiver: oneshot::Receiver<()>,
mut track_receiver: mpsc::UnboundedReceiver<TrackEvent>,
) {
loop {
tokio::select! {
Some(event) = track_receiver.recv() => {
self.dispatcher.lock().dispatch(&event);
if let Some(track) = track.as_ref() {
let track_stream = UnboundedReceiverStream::new(track.register_observer());
tokio::spawn({
let dispatcher = self.dispatcher.clone();
let notifier = self.close_notifier.clone();
async move {
let notified = notifier.notified();
futures_util::pin_mut!(notified);
futures_util::future::select(
track_stream.map(Ok).forward(dispatcher),
notified,
)
.await;
}
_ = &mut close_receiver => {
break;
}
}
});
}
}
@@ -111,7 +85,7 @@ impl TrackPublicationShared {
*self.dimension.lock() = TrackDimension(info.width, info.height);
*self.mime_type.lock() = info.mime_type;
self.kind.store(
TrackKind::from(proto::TrackType::from_i32(info.r#type).unwrap()) as u8,
TrackKind::try_from(proto::TrackType::from_i32(info.r#type).unwrap()).unwrap() as u8,
Ordering::SeqCst,
);
self.source.store(
@@ -125,13 +99,41 @@ impl TrackPublicationShared {
track.set_muted(info.muted);
}
}
}
impl Drop for TrackPublicationShared {
fn drop(&mut self) {
if let Some(close_sender) = self.close_sender.lock().take() {
let _ = close_sender.send(());
}
pub fn sid(&self) -> TrackSid {
self.sid.lock().clone()
}
pub fn name(&self) -> String {
self.name.lock().clone()
}
pub fn kind(&self) -> TrackKind {
self.kind.load(Ordering::SeqCst).try_into().unwrap()
}
pub fn source(&self) -> TrackSource {
self.source.load(Ordering::SeqCst).into()
}
pub fn simulcasted(&self) -> bool {
self.simulcasted.load(Ordering::Relaxed)
}
pub fn dimension(&self) -> TrackDimension {
self.dimension.lock().clone()
}
pub fn mime_type(&self) -> String {
self.mime_type.lock().clone()
}
pub fn track(&self) -> Option<Track> {
self.track.lock().clone()
}
pub fn muted(&self) -> bool {
self.muted.load(Ordering::Relaxed)
}
}
@@ -142,134 +144,22 @@ pub enum TrackPublication {
}
impl TrackPublication {
pub fn track(&self) -> Option<TrackHandle> {
// Not calling Local/Remote function here, we don't need "cast"
enum_dispatch!(
[Local, Remote];
pub fn sid(self: &Self) -> TrackSid;
pub fn name(self: &Self) -> String;
pub fn kind(self: &Self) -> TrackKind;
pub fn source(self: &Self) -> TrackSource;
pub fn simulcasted(self: &Self) -> bool;
pub fn dimension(self: &Self) -> TrackDimension;
pub fn mime_type(self: &Self) -> String;
pub fn muted(self: &Self) -> bool;
);
pub fn track(&self) -> Option<Track> {
match self {
TrackPublication::Local(p) => p.shared.track.lock().clone(),
TrackPublication::Remote(p) => p.shared.track.lock().clone(),
TrackPublication::Local(p) => p.track().map(Into::into),
TrackPublication::Remote(p) => p.track().map(Into::into),
}
}
}
impl TrackPublicationInternalTrait for TrackPublication {
enum_dispatch!(
[Local, Remote]
fnc!(update_track, &Self, [track: Option<TrackHandle>], ());
fnc!(update_info, &Self, [info: proto::TrackInfo], ());
);
}
impl TrackPublicationTrait for TrackPublication {
enum_dispatch!(
[Local, Remote]
fnc!(sid, &Self, [], TrackSid);
fnc!(name, &Self, [], String);
fnc!(kind, &Self, [], TrackKind);
fnc!(source, &Self, [], TrackSource);
fnc!(simulcasted, &Self, [], bool);
fnc!(dimension, &Self, [], TrackDimension);
fnc!(mime_type, &Self, [], String);
fnc!(muted, &Self, [], bool);
);
}
macro_rules! impl_publication_trait {
($x:ident) => {
impl TrackPublicationTrait for $x {
fn name(&self) -> String {
self.shared.name.lock().clone()
}
fn sid(&self) -> TrackSid {
self.shared.sid.lock().clone()
}
fn kind(&self) -> TrackKind {
self.shared.kind.load(Ordering::SeqCst).into()
}
fn source(&self) -> TrackSource {
self.shared.source.load(Ordering::SeqCst).into()
}
fn simulcasted(&self) -> bool {
self.shared.simulcasted.load(Ordering::SeqCst)
}
fn dimension(&self) -> TrackDimension {
self.shared.dimension.lock().clone()
}
fn mime_type(&self) -> String {
self.shared.mime_type.lock().clone()
}
fn muted(&self) -> bool {
self.shared.muted.load(Ordering::SeqCst)
}
}
};
}
#[derive(Clone, Debug)]
pub struct LocalTrackPublication {
shared: Arc<TrackPublicationShared>,
}
impl LocalTrackPublication {
pub fn track(&self) -> Option<LocalTrackHandle> {
self.shared
.track
.lock()
.clone()
.map(|local_track| local_track.try_into().unwrap())
}
}
impl TrackPublicationInternalTrait for LocalTrackPublication {
fn update_track(&self, track: Option<TrackHandle>) {
self.shared.update_track(track);
}
fn update_info(&self, info: proto::TrackInfo) {
self.shared.update_info(info);
}
}
#[derive(Clone, Debug)]
pub struct RemoteTrackPublication {
shared: Arc<TrackPublicationShared>,
}
impl RemoteTrackPublication {
pub fn new(
info: proto::TrackInfo,
participant: ParticipantSid,
track: Option<TrackHandle>,
) -> Self {
Self {
shared: TrackPublicationShared::new(info, participant, track),
}
}
pub fn track(&self) -> Option<RemoteTrackHandle> {
self.shared
.track
.lock()
.clone()
.map(|track| track.try_into().unwrap())
}
}
impl TrackPublicationInternalTrait for RemoteTrackPublication {
fn update_track(&self, track: Option<TrackHandle>) {
self.shared.update_track(track);
}
fn update_info(&self, info: proto::TrackInfo) {
self.shared.update_info(info);
}
}
impl_publication_trait!(LocalTrackPublication);
impl_publication_trait!(RemoteTrackPublication);
livekit/src/room/publication/remote.rs
+73
View File
@@ -0,0 +1,73 @@
use super::TrackPublicationInner;
use crate::id::TrackSid;
use crate::proto;
use crate::track::{RemoteTrack, Track, TrackDimension, TrackKind, TrackSource};
use std::sync::Arc;
#[derive(Clone, Debug)]
pub struct RemoteTrackPublication {
inner: Arc<TrackPublicationInner>,
}
impl RemoteTrackPublication {
pub(crate) fn new(info: proto::TrackInfo, track: Option<RemoteTrack>) -> Self {
Self {
inner: Arc::new(TrackPublicationInner::new(info, track.map(Into::into))),
}
}
#[inline]
pub fn sid(&self) -> TrackSid {
self.inner.sid()
}
#[inline]
pub fn name(&self) -> String {
self.inner.name()
}
#[inline]
pub fn kind(&self) -> TrackKind {
self.inner.kind()
}
#[inline]
pub fn source(&self) -> TrackSource {
self.inner.source()
}
#[inline]
pub fn simulcasted(&self) -> bool {
self.inner.simulcasted()
}
#[inline]
pub fn dimension(&self) -> TrackDimension {
self.inner.dimension()
}
#[inline]
pub fn track(&self) -> Option<RemoteTrack> {
self.inner.track().map(|track| track.try_into().unwrap())
}
#[inline]
pub fn mime_type(&self) -> String {
self.inner.mime_type()
}
#[inline]
pub fn muted(&self) -> bool {
self.inner.muted()
}
#[inline]
pub(crate) fn update_track(&self, track: Option<Track>) {
self.inner.update_track(track);
}
#[inline]
pub(crate) fn update_info(&self, info: proto::TrackInfo) {
self.inner.update_info(info);
}
}
livekit/src/room/room_session.rs
+57 -82
View File
@@ -1,7 +1,7 @@
use crate::participant::{ConnectionQuality, ParticipantInternalTrait};
use crate::participant::ConnectionQuality;
use crate::prelude::*;
use crate::proto;
use crate::rtc_engine::{EngineEvent, EngineEvents, EngineResult, RTCEngine};
use crate::rtc_engine::{EngineEvent, EngineEvents, EngineResult, RtcEngine};
use crate::signal_client::SignalOptions;
use crate::{RoomError, RoomEvent, RoomResult, SimulateScenario};
use livekit_utils::observer::Dispatcher;
@@ -9,8 +9,7 @@ use parking_lot::{Mutex, RwLock, RwLockReadGuard};
use std::collections::HashMap;
use std::sync::atomic::{AtomicU8, Ordering};
use std::sync::Arc;
use tokio::sync::mpsc;
use tokio::sync::oneshot;
use tokio::sync::{mpsc, oneshot};
use tokio::task::JoinHandle;
use tracing::{error, info, instrument, Level};
@@ -22,17 +21,6 @@ pub enum ConnectionState {
Unknown,
}
impl From<u8> for ConnectionState {
fn from(value: u8) -> Self {
match value {
0 => ConnectionState::Disconnected,
1 => ConnectionState::Connected,
2 => ConnectionState::Reconnecting,
_ => ConnectionState::Unknown,
}
}
}
/// Internal representation of a RoomSession
#[derive(Debug)]
struct SessionInner {
@@ -40,12 +28,12 @@ struct SessionInner {
sid: Mutex<RoomSid>,
name: Mutex<String>,
metadata: Mutex<String>,
participants: RwLock<HashMap<ParticipantSid, Arc<RemoteParticipant>>>,
participants: RwLock<HashMap<ParticipantSid, RemoteParticipant>>,
participants_tasks: RwLock<HashMap<ParticipantSid, (JoinHandle<()>, oneshot::Sender<()>)>>,
active_speakers: RwLock<Vec<Participant>>,
rtc_engine: Arc<RTCEngine>,
local_participant: Arc<LocalParticipant>,
dispatcher: Mutex<Dispatcher<RoomEvent>>,
rtc_engine: Arc<RtcEngine>,
local_participant: LocalParticipant,
dispatcher: Dispatcher<RoomEvent>,
}
#[derive(Debug)]
@@ -64,7 +52,7 @@ pub struct RoomSession {
impl SessionHandle {
pub async fn connect(url: &str, token: &str) -> RoomResult<Self> {
let (rtc_engine, engine_events) = RTCEngine::new();
let (rtc_engine, engine_events) = RtcEngine::new();
let rtc_engine = Arc::new(rtc_engine);
rtc_engine
.connect(url, token, SignalOptions::default())
@@ -72,13 +60,13 @@ impl SessionHandle {
let join_response = rtc_engine.join_response().unwrap();
let pi = join_response.participant.unwrap().clone();
let local_participant = Arc::new(LocalParticipant::new(
let local_participant = LocalParticipant::new(
rtc_engine.clone(),
pi.sid.into(),
pi.identity.into(),
pi.name,
pi.metadata,
));
);
let room_info = join_response.room.unwrap();
let inner = Arc::new(SessionInner {
@@ -99,7 +87,7 @@ impl SessionHandle {
let pi = pi.clone();
inner.create_participant(pi.sid.into(), pi.identity.into(), pi.name, pi.metadata)
};
participant.update_info(pi.clone(), false);
participant.update_info(pi.clone());
}
let (close_emitter, close_receiver) = oneshot::channel();
@@ -108,7 +96,7 @@ impl SessionHandle {
inner.update_connection_state(ConnectionState::Connected);
let session = Self {
session: RoomSession::from(inner),
session: RoomSession { inner },
session_task,
close_emitter,
};
@@ -122,7 +110,7 @@ impl SessionHandle {
}
pub fn subscribe(&self) -> mpsc::UnboundedReceiver<RoomEvent> {
self.session.inner.dispatcher.lock().register()
self.session.inner.dispatcher.register()
}
pub fn session(&self) -> RoomSession {
@@ -131,10 +119,6 @@ impl SessionHandle {
}
impl RoomSession {
fn from(inner: Arc<SessionInner>) -> Self {
Self { inner }
}
pub fn sid(&self) -> RoomSid {
self.inner.sid.lock().clone()
}
@@ -147,7 +131,7 @@ impl RoomSession {
self.inner.metadata.lock().clone()
}
pub fn local_participant(&self) -> Arc<LocalParticipant> {
pub fn local_participant(&self) -> LocalParticipant {
self.inner.local_participant.clone()
}
@@ -155,7 +139,7 @@ impl RoomSession {
self.inner.state.load(Ordering::Acquire).try_into().unwrap()
}
pub fn participants(&self) -> RwLockReadGuard<HashMap<ParticipantSid, Arc<RemoteParticipant>>> {
pub fn participants(&self) -> RwLockReadGuard<HashMap<ParticipantSid, RemoteParticipant>> {
self.inner.participants.read()
}
@@ -226,36 +210,30 @@ impl SessionInner {
if let Participant::Remote(remote_participant) = participant {
match event {
ParticipantEvent::TrackPublished { publication } => {
self.dispatcher.lock().dispatch(&RoomEvent::TrackPublished {
self.dispatcher.dispatch(&RoomEvent::TrackPublished {
participant: remote_participant.clone(),
publication,
});
}
ParticipantEvent::TrackUnpublished { publication } => {
self.dispatcher
.lock()
.dispatch(&RoomEvent::TrackUnpublished {
participant: remote_participant.clone(),
publication,
});
self.dispatcher.dispatch(&RoomEvent::TrackUnpublished {
participant: remote_participant.clone(),
publication,
});
}
ParticipantEvent::TrackSubscribed { track, publication } => {
self.dispatcher
.lock()
.dispatch(&RoomEvent::TrackSubscribed {
participant: remote_participant.clone(),
track,
publication,
});
self.dispatcher.dispatch(&RoomEvent::TrackSubscribed {
participant: remote_participant.clone(),
track,
publication,
});
}
ParticipantEvent::TrackUnsubscribed { track, publication } => {
self.dispatcher
.lock()
.dispatch(&RoomEvent::TrackUnsubscribed {
participant: remote_participant.clone(),
track,
publication,
});
self.dispatcher.dispatch(&RoomEvent::TrackUnsubscribed {
participant: remote_participant.clone(),
track,
publication,
});
}
_ => {}
};
@@ -303,12 +281,12 @@ impl SessionInner {
}
EngineEvent::Resuming => {
if self.update_connection_state(ConnectionState::Reconnecting) {
self.dispatcher.lock().dispatch(&RoomEvent::Reconnecting);
self.dispatcher.dispatch(&RoomEvent::Reconnecting);
}
}
EngineEvent::Resumed => {
self.update_connection_state(ConnectionState::Connected);
self.dispatcher.lock().dispatch(&RoomEvent::Reconnected);
self.dispatcher.dispatch(&RoomEvent::Reconnected);
// TODO(theomonnom): Update subscriptions settings
// TODO(theomonnom): Send sync state
@@ -323,7 +301,7 @@ impl SessionInner {
} => {
let payload = Arc::new(payload);
if let Some(participant) = self.get_participant(&participant_sid.into()) {
self.dispatcher.lock().dispatch(&RoomEvent::DataReceived {
self.dispatcher.dispatch(&RoomEvent::DataReceived {
payload: payload.clone(),
kind,
participant: participant.clone(),
@@ -357,7 +335,6 @@ impl SessionInner {
self.state.store(state as u8, Ordering::Release);
self.dispatcher
.lock()
.dispatch(&RoomEvent::ConnectionStateChanged(state));
return true;
}
@@ -368,11 +345,10 @@ impl SessionInner {
#[instrument(level = Level::DEBUG)]
fn handle_participant_update(self: &Arc<Self>, updates: Vec<proto::ParticipantInfo>) {
for pi in updates {
info!("test");
if pi.sid == self.local_participant.sid()
|| pi.identity == self.local_participant.identity()
{
self.local_participant.clone().update_info(pi, true);
self.local_participant.clone().update_info(pi);
continue;
}
@@ -386,7 +362,7 @@ impl SessionInner {
.handle_participant_disconnect(remote_participant)
} else {
// Participant is already connected, update the it
remote_participant.update_info(pi.clone(), true);
remote_participant.update_info(pi.clone());
}
} else {
// Create a new participant
@@ -398,10 +374,9 @@ impl SessionInner {
let _ = self
.dispatcher
.lock()
.dispatch(&RoomEvent::ParticipantConnected(remote_participant.clone()));
remote_participant.update_info(pi.clone(), true);
remote_participant.update_info(pi.clone());
}
}
}
@@ -438,7 +413,6 @@ impl SessionInner {
let _ = self
.dispatcher
.lock()
.dispatch(&RoomEvent::ActiveSpeakersChanged { speakers });
}
@@ -466,7 +440,6 @@ impl SessionInner {
participant.set_connection_quality(quality);
self.dispatcher
.lock()
.dispatch(&RoomEvent::ConnectionQualityChanged {
participant,
quality,
@@ -483,7 +456,7 @@ impl SessionInner {
}
if self.update_connection_state(ConnectionState::Reconnecting) {
self.dispatcher.lock().dispatch(&RoomEvent::Reconnecting);
self.dispatcher.dispatch(&RoomEvent::Reconnecting);
}
}
@@ -493,10 +466,10 @@ impl SessionInner {
let join_response = self.rtc_engine.join_response().unwrap();
self.update_connection_state(ConnectionState::Connected);
self.dispatcher.lock().dispatch(&RoomEvent::Reconnected);
self.dispatcher.dispatch(&RoomEvent::Reconnected);
if let Some(pi) = join_response.participant {
self.local_participant.update_info(pi, true); // The sid may have changed
self.local_participant.update_info(pi); // The sid may have changed
}
self.handle_participant_update(join_response.other_participants);
@@ -511,7 +484,7 @@ impl SessionInner {
}
self.update_connection_state(ConnectionState::Disconnected);
self.dispatcher.lock().dispatch(&RoomEvent::Disconnected);
self.dispatcher.dispatch(&RoomEvent::Disconnected);
}
/// Create a new participant
@@ -523,13 +496,8 @@ impl SessionInner {
identity: ParticipantIdentity,
name: String,
metadata: String,
) -> Arc<RemoteParticipant> {
let participant = Arc::new(RemoteParticipant::new(
sid.clone(),
identity,
name,
metadata,
));
) -> RemoteParticipant {
let participant = RemoteParticipant::new(sid.clone(), identity, name, metadata);
// Create the participant task
let (close_tx, close_rx) = oneshot::channel();
@@ -549,10 +517,10 @@ impl SessionInner {
/// A participant has disconnected
/// Cleanup the participant and emit an event
#[instrument(level = Level::DEBUG)]
fn handle_participant_disconnect(self: Arc<Self>, remote_participant: Arc<RemoteParticipant>) {
fn handle_participant_disconnect(self: Arc<Self>, remote_participant: RemoteParticipant) {
tokio::spawn(async move {
for (sid, _) in &*remote_participant.tracks() {
remote_participant.unpublish_track(&sid, true);
remote_participant.unpublish_track(&sid);
}
// Close the participant task
@@ -566,16 +534,12 @@ impl SessionInner {
}
self.participants.write().remove(&remote_participant.sid());
self.dispatcher
.lock()
.dispatch(&RoomEvent::ParticipantDisconnected(
remote_participant.clone(),
));
.dispatch(&RoomEvent::ParticipantDisconnected(remote_participant));
});
}
fn get_participant(&self, sid: &ParticipantSid) -> Option<Arc<RemoteParticipant>> {
fn get_participant(&self, sid: &ParticipantSid) -> Option<RemoteParticipant> {
self.participants.read().get(sid).cloned()
}
}
@@ -590,3 +554,14 @@ fn unpack_stream_id(stream_id: &str) -> Option<(&str, &str)> {
None
}
}
impl From<u8> for ConnectionState {
fn from(value: u8) -> Self {
match value {
0 => ConnectionState::Disconnected,
1 => ConnectionState::Connected,
2 => ConnectionState::Reconnecting,
_ => ConnectionState::Unknown,
}
}
}
livekit/src/room/track/audio_track.rs
-32
View File
@@ -1,32 +0,0 @@
use super::impl_track_trait;
use crate::prelude::*;
use std::sync::Arc;
#[derive(Clone)]
pub enum AudioTrackHandle {
Local(Arc<LocalAudioTrack>),
Remote(Arc<RemoteAudioTrack>),
}
impl From<AudioTrackHandle> for TrackHandle {
fn from(audio_track: AudioTrackHandle) -> Self {
match audio_track {
AudioTrackHandle::Local(local_audio) => Self::LocalAudio(local_audio),
AudioTrackHandle::Remote(remote_audio) => Self::RemoteAudio(remote_audio),
}
}
}
impl TryFrom<TrackHandle> for AudioTrackHandle {
type Error = &'static str;
fn try_from(track: TrackHandle) -> Result<Self, Self::Error> {
match track {
TrackHandle::LocalAudio(local_audio) => Ok(Self::Local(local_audio)),
TrackHandle::RemoteAudio(remote_audio) => Ok(Self::Remote(remote_audio)),
_ => Err("not a audio track"),
}
}
}
impl_track_trait!(AudioTrackHandle, enum_dispatch, [Local, Remote]);
livekit/src/room/track/local_audio_track.rs
+101 -4
View File
@@ -1,8 +1,105 @@
use super::{impl_track_trait, TrackShared};
use super::TrackInner;
use crate::prelude::*;
use crate::proto;
use livekit_webrtc as rtc;
use std::sync::Arc;
use tokio::sync::mpsc;
#[derive(Debug)]
#[derive(Clone, Debug)]
pub struct LocalAudioTrack {
shared: TrackShared,
pub(crate) inner: Arc<TrackInner>,
}
impl_track_trait!(LocalAudioTrack);
impl LocalAudioTrack {
pub(crate) fn new(
sid: TrackSid,
name: String,
rtc_track: rtc::media_stream::RtcAudioTrack,
) -> Self {
Self {
inner: Arc::new(TrackInner::new(
sid,
name,
TrackKind::Audio,
rtc::media_stream::MediaStreamTrack::Audio(rtc_track),
)),
}
}
#[inline]
pub fn sid(&self) -> TrackSid {
self.inner.sid()
}
#[inline]
pub fn name(&self) -> String {
self.inner.name()
}
#[inline]
pub fn kind(&self) -> TrackKind {
self.inner.kind()
}
#[inline]
pub fn source(&self) -> TrackSource {
self.inner.source()
}
#[inline]
pub fn stream_state(&self) -> StreamState {
self.inner.stream_state()
}
#[inline]
pub fn start(&self) {
self.inner.start()
}
#[inline]
pub fn stop(&self) {
self.inner.stop()
}
#[inline]
pub fn muted(&self) -> bool {
self.inner.muted()
}
#[inline]
pub fn set_muted(&self, muted: bool) {
self.inner.set_muted(muted)
}
#[inline]
pub fn rtc_track(&self) -> rtc::media_stream::RtcAudioTrack {
if let rtc::media_stream::MediaStreamTrack::Audio(audio) = self.inner.rtc_track() {
audio
} else {
unreachable!()
}
}
#[inline]
pub fn register_observer(&self) -> mpsc::UnboundedReceiver<TrackEvent> {
self.inner.register_observer()
}
#[inline]
pub(crate) fn transceiver(&self) -> Option<rtc::rtp_transceiver::RtpTransceiver> {
self.inner.transceiver()
}
#[inline]
pub(crate) fn update_transceiver(
&self,
transceiver: Option<rtc::rtp_transceiver::RtpTransceiver>,
) {
self.inner.update_transceiver(transceiver)
}
#[inline]
pub(crate) fn update_info(&self, info: proto::TrackInfo) {
self.inner.update_info(info)
}
}
livekit/src/room/track/local_track.rs
-32
View File
@@ -1,32 +0,0 @@
use super::impl_track_trait;
use crate::prelude::*;
use std::sync::Arc;
#[derive(Clone)]
pub enum LocalTrackHandle {
Audio(Arc<LocalAudioTrack>),
Video(Arc<LocalVideoTrack>),
}
impl From<LocalTrackHandle> for TrackHandle {
fn from(local_track: LocalTrackHandle) -> Self {
match local_track {
LocalTrackHandle::Audio(local_audio) => Self::LocalAudio(local_audio),
LocalTrackHandle::Video(local_video) => Self::LocalVideo(local_video),
}
}
}
impl TryFrom<TrackHandle> for LocalTrackHandle {
type Error = &'static str;
fn try_from(track: TrackHandle) -> Result<Self, Self::Error> {
match track {
TrackHandle::LocalAudio(local_audio) => Ok(Self::Audio(local_audio)),
TrackHandle::LocalVideo(local_video) => Ok(Self::Video(local_video)),
_ => Err("not a local track"),
}
}
}
impl_track_trait!(LocalTrackHandle, enum_dispatch, [Audio, Video]);
livekit/src/room/track/local_video_track.rs
+134 -4
View File
@@ -1,8 +1,138 @@
use super::{impl_track_trait, TrackShared};
use super::TrackInner;
use crate::proto;
use crate::rtc_engine::lk_runtime::LkRuntime;
use crate::{options::VideoCaptureOptions, prelude::*};
use livekit_webrtc as rtc;
use livekit_webrtc::peer_connection_factory::native::PeerConnectionFactoryExt;
use parking_lot::Mutex;
use rtc::video_source::native::NativeVideoSource;
use std::sync::Arc;
use tokio::sync::mpsc;
#[derive(Debug)]
pub struct LocalVideoTrack {
shared: TrackShared,
struct LocalVideoTrackInner {
track_inner: TrackInner,
capture_options: Mutex<VideoCaptureOptions>,
}
impl_track_trait!(LocalVideoTrack);
#[derive(Clone, Debug)]
pub struct LocalVideoTrack {
inner: Arc<LocalVideoTrackInner>,
}
impl LocalVideoTrack {
pub fn new(
name: String,
rtc_track: rtc::media_stream::RtcVideoTrack,
capture_options: VideoCaptureOptions,
) -> Self {
Self {
inner: Arc::new(LocalVideoTrackInner {
track_inner: TrackInner::new(
"unknown".to_string().into(), // sid
name,
TrackKind::Video,
rtc::media_stream::MediaStreamTrack::Video(rtc_track),
),
capture_options: Mutex::new(capture_options),
}),
}
}
pub fn capture_options(&self) -> VideoCaptureOptions {
self.inner.capture_options.lock().clone()
}
#[inline]
pub fn sid(&self) -> TrackSid {
self.inner.track_inner.sid()
}
#[inline]
pub fn name(&self) -> String {
self.inner.track_inner.name()
}
#[inline]
pub fn kind(&self) -> TrackKind {
self.inner.track_inner.kind()
}
#[inline]
pub fn source(&self) -> TrackSource {
self.inner.track_inner.source()
}
#[inline]
pub fn stream_state(&self) -> StreamState {
self.inner.track_inner.stream_state()
}
#[inline]
pub fn start(&self) {
self.inner.track_inner.start()
}
#[inline]
pub fn stop(&self) {
self.inner.track_inner.stop()
}
#[inline]
pub fn muted(&self) -> bool {
self.inner.track_inner.muted()
}
#[inline]
pub fn set_muted(&self, muted: bool) {
self.inner.track_inner.set_muted(muted)
}
#[inline]
pub fn rtc_track(&self) -> rtc::media_stream::RtcVideoTrack {
if let rtc::media_stream::MediaStreamTrack::Video(video) =
self.inner.track_inner.rtc_track()
{
video
} else {
unreachable!()
}
}
#[inline]
pub fn register_observer(&self) -> mpsc::UnboundedReceiver<TrackEvent> {
self.inner.track_inner.register_observer()
}
#[inline]
pub fn transceiver(&self) -> Option<rtc::rtp_transceiver::RtpTransceiver> {
self.inner.track_inner.transceiver()
}
#[inline]
pub(crate) fn update_transceiver(
&self,
transceiver: Option<rtc::rtp_transceiver::RtpTransceiver>,
) {
self.inner.track_inner.update_transceiver(transceiver)
}
#[inline]
pub(crate) fn update_info(&self, info: proto::TrackInfo) {
self.inner.track_inner.update_info(info)
}
}
impl LocalVideoTrack {
pub fn create_video_track(
name: &str,
options: VideoCaptureOptions,
source: NativeVideoSource,
) -> LocalVideoTrack {
let rtc_track = LkRuntime::instance()
.pc_factory
.create_video_track(&rtc::native::create_random_uuid(), source);
Self::new(name.to_string(), rtc_track, options)
}
}
livekit/src/room/track/mod.rs
+367 -237
View File
@@ -2,29 +2,22 @@ use crate::prelude::*;
use crate::proto;
use livekit_utils::enum_dispatch;
use livekit_utils::observer::Dispatcher;
use livekit_webrtc as rtc;
use parking_lot::Mutex;
use rtc::MediaType;
use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
use std::sync::Arc;
use thiserror::Error;
use tokio::sync::mpsc;
pub mod audio_track;
pub mod local_audio_track;
pub mod local_track;
pub mod local_video_track;
pub mod remote_audio_track;
pub mod remote_track;
pub mod remote_video_track;
pub mod video_track;
pub use audio_track::*;
pub use local_audio_track::*;
pub use local_track::*;
pub use local_video_track::*;
pub use remote_audio_track::*;
pub use remote_track::*;
pub use remote_video_track::*;
pub use video_track::*;
#[derive(Error, Debug, Clone)]
pub enum TrackError {
@@ -32,51 +25,19 @@ pub enum TrackError {
TrackNotFound(String),
}
#[derive(Debug)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TrackKind {
Unknown,
Audio,
Video,
}
impl From<u8> for TrackKind {
fn from(val: u8) -> Self {
match val {
1 => Self::Audio,
2 => Self::Video,
_ => Self::Unknown,
}
}
}
impl From<proto::TrackType> for TrackKind {
fn from(r#type: proto::TrackType) -> Self {
match r#type {
proto::TrackType::Audio => Self::Audio,
proto::TrackType::Video => Self::Video,
_ => Self::Unknown,
}
}
}
#[derive(Debug)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum StreamState {
Unknown,
Active,
Paused,
}
impl From<u8> for StreamState {
fn from(val: u8) -> Self {
match val {
1 => Self::Active,
2 => Self::Paused,
_ => Self::Unknown,
}
}
}
#[derive(Debug)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TrackSource {
Unknown,
Camera,
@@ -85,9 +46,364 @@ pub enum TrackSource {
ScreenshareAudio,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TrackEvent {
Mute,
Unmute,
}
#[derive(Clone, Copy, Debug)]
pub struct TrackDimension(pub u32, pub u32);
#[derive(Clone, Debug)]
pub enum Track {
LocalAudio(LocalAudioTrack),
LocalVideo(LocalVideoTrack),
RemoteAudio(RemoteAudioTrack),
RemoteVideo(RemoteVideoTrack),
}
#[derive(Clone, Debug)]
pub enum LocalTrack {
Audio(LocalAudioTrack),
Video(LocalVideoTrack),
}
#[derive(Clone, Debug)]
pub enum RemoteTrack {
Audio(RemoteAudioTrack),
Video(RemoteVideoTrack),
}
#[derive(Clone, Debug)]
pub enum VideoTrack {
Local(LocalVideoTrack),
Remote(RemoteVideoTrack),
}
#[derive(Clone, Debug)]
pub enum AudioTrack {
Local(LocalAudioTrack),
Remote(RemoteAudioTrack),
}
macro_rules! track_dispatch {
([$($variant:ident),+]) => {
enum_dispatch!(
[$($variant),+];
pub fn sid(self: &Self) -> TrackSid;
pub fn name(self: &Self) -> String;
pub fn kind(self: &Self) -> TrackKind;
pub fn source(self: &Self) -> TrackSource;
pub fn stream_state(self: &Self) -> StreamState;
pub fn start(self: &Self) -> ();
pub fn stop(self: &Self) -> ();
pub fn muted(self: &Self) -> bool;
pub fn set_muted(self: &Self, muted: bool) -> ();
pub fn register_observer(self: &Self) -> mpsc::UnboundedReceiver<TrackEvent>;
pub(crate) fn transceiver(self: &Self) -> Option<rtc::rtp_transceiver::RtpTransceiver>;
pub(crate) fn update_transceiver(self: &Self, transceiver: Option<rtc::rtp_transceiver::RtpTransceiver>) -> ();
pub(crate) fn update_info(self: &Self, info: proto::TrackInfo) -> ();
);
};
}
impl Track {
track_dispatch!([LocalAudio, LocalVideo, RemoteAudio, RemoteVideo]);
#[inline]
pub fn rtc_track(&self) -> rtc::media_stream::MediaStreamTrack {
match self {
Self::LocalAudio(track) => track.rtc_track().into(),
Self::LocalVideo(track) => track.rtc_track().into(),
Self::RemoteAudio(track) => track.rtc_track().into(),
Self::RemoteVideo(track) => track.rtc_track().into(),
}
}
}
impl LocalTrack {
track_dispatch!([Audio, Video]);
#[inline]
pub fn rtc_track(&self) -> rtc::media_stream::MediaStreamTrack {
match self {
Self::Audio(track) => track.rtc_track().into(),
Self::Video(track) => track.rtc_track().into(),
}
}
}
impl RemoteTrack {
track_dispatch!([Audio, Video]);
#[inline]
pub fn rtc_track(&self) -> rtc::media_stream::MediaStreamTrack {
match self {
Self::Audio(track) => track.rtc_track().into(),
Self::Video(track) => track.rtc_track().into(),
}
}
}
impl VideoTrack {
track_dispatch!([Local, Remote]);
#[inline]
pub fn rtc_track(&self) -> rtc::media_stream::RtcVideoTrack {
match self {
Self::Local(track) => track.rtc_track(),
Self::Remote(track) => track.rtc_track(),
}
}
}
impl AudioTrack {
track_dispatch!([Local, Remote]);
#[inline]
pub fn rtc_track(&self) -> rtc::media_stream::RtcAudioTrack {
match self {
Self::Local(track) => track.rtc_track().into(),
Self::Remote(track) => track.rtc_track().into(),
}
}
}
#[derive(Debug)]
pub(crate) struct TrackInner {
pub sid: Mutex<TrackSid>,
pub name: Mutex<String>,
pub kind: AtomicU8, // TrackKind
pub source: AtomicU8, // TrackSource
pub stream_state: AtomicU8, // StreamState
pub muted: AtomicBool,
pub rtc_track: rtc::media_stream::MediaStreamTrack,
pub transceiver: Mutex<Option<rtc::rtp_transceiver::RtpTransceiver>>,
pub dispatcher: Dispatcher<TrackEvent>,
}
impl TrackInner {
pub fn new(
sid: TrackSid,
name: String,
kind: TrackKind,
rtc_track: rtc::media_stream::MediaStreamTrack,
) -> Self {
Self {
sid: Mutex::new(sid),
name: Mutex::new(name),
kind: AtomicU8::new(kind as u8),
source: AtomicU8::new(TrackSource::Unknown as u8),
stream_state: AtomicU8::new(StreamState::Active as u8),
muted: AtomicBool::new(false),
rtc_track,
transceiver: Default::default(),
dispatcher: Default::default(),
}
}
pub fn sid(&self) -> TrackSid {
self.sid.lock().clone()
}
pub fn name(&self) -> String {
self.name.lock().clone()
}
pub fn kind(&self) -> TrackKind {
self.kind.load(Ordering::SeqCst).try_into().unwrap()
}
pub fn source(&self) -> TrackSource {
self.source.load(Ordering::SeqCst).into()
}
pub fn stream_state(&self) -> StreamState {
self.stream_state.load(Ordering::SeqCst).try_into().unwrap()
}
pub fn muted(&self) -> bool {
self.muted.load(Ordering::SeqCst)
}
pub fn start(&self) {
self.rtc_track.set_enabled(true);
}
pub fn stop(&self) {
self.rtc_track.set_enabled(false);
}
pub fn set_muted(&self, muted: bool) {
if self
.muted
.compare_exchange(!muted, muted, Ordering::SeqCst, Ordering::SeqCst)
.is_err()
{
return;
}
if !muted {
self.start();
} else {
self.stop();
}
let event = if muted {
TrackEvent::Mute
} else {
TrackEvent::Unmute
};
self.dispatcher.dispatch(&event);
}
pub fn rtc_track(&self) -> rtc::media_stream::MediaStreamTrack {
self.rtc_track.clone()
}
pub fn register_observer(&self) -> mpsc::UnboundedReceiver<TrackEvent> {
self.dispatcher.register()
}
pub fn transceiver(&self) -> Option<rtc::rtp_transceiver::RtpTransceiver> {
self.transceiver.lock().clone()
}
pub fn update_transceiver(&self, transceiver: Option<rtc::rtp_transceiver::RtpTransceiver>) {
*self.transceiver.lock() = transceiver;
}
pub fn update_info(&self, info: proto::TrackInfo) {
*self.name.lock() = info.name;
*self.sid.lock() = info.sid.into();
self.kind.store(
TrackKind::try_from(proto::TrackType::from_i32(info.r#type).unwrap()).unwrap() as u8,
Ordering::SeqCst,
);
self.source.store(
TrackSource::from(proto::TrackSource::from_i32(info.source).unwrap()) as u8,
Ordering::SeqCst,
);
// Muted and StreamState are not handled separately (events)
}
}
impl From<RemoteTrack> for Track {
fn from(track: RemoteTrack) -> Self {
match track {
RemoteTrack::Audio(track) => Self::RemoteAudio(track),
RemoteTrack::Video(track) => Self::RemoteVideo(track),
}
}
}
impl From<LocalTrack> for Track {
fn from(track: LocalTrack) -> Self {
match track {
LocalTrack::Audio(track) => Self::LocalAudio(track),
LocalTrack::Video(track) => Self::LocalVideo(track),
}
}
}
impl From<VideoTrack> for Track {
fn from(track: VideoTrack) -> Self {
match track {
VideoTrack::Local(track) => Self::LocalVideo(track),
VideoTrack::Remote(track) => Self::RemoteVideo(track),
}
}
}
impl From<AudioTrack> for Track {
fn from(track: AudioTrack) -> Self {
match track {
AudioTrack::Local(track) => Self::LocalAudio(track),
AudioTrack::Remote(track) => Self::RemoteAudio(track),
}
}
}
impl TryFrom<Track> for RemoteTrack {
type Error = &'static str;
fn try_from(track: Track) -> Result<Self, Self::Error> {
match track {
Track::RemoteAudio(track) => Ok(Self::Audio(track)),
Track::RemoteVideo(track) => Ok(Self::Video(track)),
_ => Err("not a remote track"),
}
}
}
impl TryFrom<Track> for LocalTrack {
type Error = &'static str;
fn try_from(track: Track) -> Result<Self, Self::Error> {
match track {
Track::LocalAudio(track) => Ok(Self::Audio(track)),
Track::LocalVideo(track) => Ok(Self::Video(track)),
_ => Err("not a local track"),
}
}
}
impl TryFrom<Track> for VideoTrack {
type Error = &'static str;
fn try_from(track: Track) -> Result<Self, Self::Error> {
match track {
Track::LocalVideo(track) => Ok(Self::Local(track)),
Track::RemoteVideo(track) => Ok(Self::Remote(track)),
_ => Err("not a video track"),
}
}
}
impl TryFrom<Track> for AudioTrack {
type Error = &'static str;
fn try_from(track: Track) -> Result<Self, Self::Error> {
match track {
Track::LocalAudio(track) => Ok(Self::Local(track)),
Track::RemoteAudio(track) => Ok(Self::Remote(track)),
_ => Err("not an audio track"),
}
}
}
// Conversions from integers (Useful since we're using atomic values to represent our enums)
impl TryFrom<u8> for TrackKind {
type Error = &'static str;
fn try_from(kind: u8) -> Result<Self, Self::Error> {
match kind {
0 => Ok(Self::Audio),
1 => Ok(Self::Video),
_ => Err("invalid track kind"),
}
}
}
impl TryFrom<u8> for StreamState {
type Error = &'static str;
fn try_from(state: u8) -> Result<Self, Self::Error> {
match state {
0 => Ok(Self::Active),
1 => Ok(Self::Paused),
_ => Err("invalid stream state"),
}
}
}
impl From<u8> for TrackSource {
fn from(val: u8) -> Self {
match val {
fn from(source: u8) -> Self {
match source {
1 => Self::Camera,
2 => Self::Microphone,
3 => Self::Screenshare,
@@ -97,197 +413,11 @@ impl From<u8> for TrackSource {
}
}
impl From<proto::TrackSource> for TrackSource {
fn from(source: proto::TrackSource) -> Self {
match source {
proto::TrackSource::Camera => Self::Camera,
proto::TrackSource::Microphone => Self::Microphone,
proto::TrackSource::ScreenShare => Self::Screenshare,
proto::TrackSource::ScreenShareAudio => Self::ScreenshareAudio,
proto::TrackSource::Unknown => Self::Unknown,
impl From<TrackKind> for MediaType {
fn from(kind: TrackKind) -> Self {
match kind {
TrackKind::Audio => Self::Audio,
TrackKind::Video => Self::Video,
}
}
}
#[derive(Clone, Copy, Debug)]
pub struct TrackDimension(pub u32, pub u32);
pub trait TrackTrait {
fn sid(&self) -> TrackSid;
fn name(&self) -> String;
fn kind(&self) -> TrackKind;
fn stream_state(&self) -> StreamState;
fn muted(&self) -> bool;
fn start(&self);
fn stop(&self);
fn register_observer(&self) -> mpsc::UnboundedReceiver<TrackEvent>;
fn set_muted(&self, muted: bool);
}
#[derive(Debug, Clone)]
pub enum TrackEvent {
Mute,
Unmute,
}
#[derive(Debug)]
pub(super) struct TrackShared {
pub(super) sid: Mutex<TrackSid>,
pub(super) name: Mutex<String>,
pub(super) kind: AtomicU8, // TrackKind
pub(super) stream_state: AtomicU8, // StreamState
pub(super) muted: AtomicBool,
pub(super) rtc_track: MediaStreamTrackHandle,
pub(super) dispatcher: Mutex<Dispatcher<TrackEvent>>,
}
impl TrackShared {
pub(crate) fn new(
sid: TrackSid,
name: String,
kind: TrackKind,
rtc_track: MediaStreamTrackHandle,
) -> Self {
Self {
sid: Mutex::new(sid),
name: Mutex::new(name),
kind: AtomicU8::new(kind as u8),
stream_state: AtomicU8::new(StreamState::Active as u8),
muted: AtomicBool::new(false),
rtc_track,
dispatcher: Default::default(),
}
}
pub(crate) fn start(&self) {
self.rtc_track.set_enabled(true);
}
pub(crate) fn stop(&self) {
self.rtc_track.set_enabled(false);
}
pub(crate) fn set_muted(&self, muted: bool) {
if self.muted.load(Ordering::SeqCst) == muted {
return;
}
self.muted.store(muted, Ordering::SeqCst);
self.rtc_track.set_enabled(!muted);
self.dispatcher.lock().dispatch(if muted {
&TrackEvent::Mute
} else {
&TrackEvent::Unmute
});
}
pub(crate) fn register_observer(&self) -> mpsc::UnboundedReceiver<TrackEvent> {
self.dispatcher.lock().register()
}
}
#[derive(Clone, Debug)]
pub enum TrackHandle {
LocalVideo(Arc<LocalVideoTrack>),
LocalAudio(Arc<LocalAudioTrack>),
RemoteVideo(Arc<RemoteVideoTrack>),
RemoteAudio(Arc<RemoteAudioTrack>),
}
impl TrackTrait for TrackHandle {
enum_dispatch!(
[LocalVideo, LocalAudio, RemoteVideo, RemoteAudio]
fnc!(sid, &Self, [], TrackSid);
fnc!(name, &Self, [], String);
fnc!(kind, &Self, [], TrackKind);
fnc!(stream_state, &Self, [], StreamState);
fnc!(muted, &Self, [], bool);
fnc!(start, &Self, [], ());
fnc!(stop, &Self, [], ());
fnc!(register_observer, &Self, [], mpsc::UnboundedReceiver<TrackEvent>);
fnc!(set_muted, &Self, [muted: bool], ());
);
}
impl TrackHandle {
pub fn rtc_track(&self) -> MediaStreamTrackHandle {
match self {
Self::RemoteVideo(remote_video) => {
MediaStreamTrackHandle::Video(remote_video.rtc_track())
}
Self::RemoteAudio(remote_audio) => {
MediaStreamTrackHandle::Audio(remote_audio.rtc_track())
}
_ => todo!(),
}
}
}
macro_rules! impl_track_trait {
($x:ident) => {
use std::sync::atomic::Ordering;
use tokio::sync::mpsc;
use $crate::room::id::TrackSid;
use $crate::room::track::{StreamState, TrackEvent, TrackKind, TrackTrait};
impl TrackTrait for $x {
fn sid(&self) -> TrackSid {
self.shared.sid.lock().clone()
}
fn name(&self) -> String {
self.shared.name.lock().clone()
}
fn kind(&self) -> TrackKind {
self.shared.kind.load(Ordering::SeqCst).into()
}
fn stream_state(&self) -> StreamState {
self.shared.stream_state.load(Ordering::SeqCst).into()
}
fn muted(&self) -> bool {
self.shared.muted.load(Ordering::SeqCst)
}
fn start(&self) {
self.shared.start();
}
fn stop(&self) {
self.shared.stop();
}
fn register_observer(&self) -> mpsc::UnboundedReceiver<TrackEvent> {
self.shared.register_observer()
}
fn set_muted(&self, muted: bool) {
self.shared.set_muted(muted);
}
}
};
($x:ident, enum_dispatch, [$($variant:ident),+]) => {
use livekit_utils::enum_dispatch;
use tokio::sync::mpsc;
impl TrackTrait for $x {
enum_dispatch!(
[$($variant),+]
fnc!(sid, &Self, [], TrackSid);
fnc!(name, &Self, [], String);
fnc!(kind, &Self, [], TrackKind);
fnc!(stream_state, &Self, [], StreamState);
fnc!(muted, &Self, [], bool);
fnc!(start, &Self, [], ());
fnc!(stop, &Self, [], ());
fnc!(register_observer, &Self, [], mpsc::UnboundedReceiver<TrackEvent>);
fnc!(set_muted, &Self, [muted: bool], ());
);
}
};
}
pub(super) use impl_track_trait;
livekit/src/room/track/remote_audio_track.rs
+86 -12
View File
@@ -1,31 +1,105 @@
use super::{impl_track_trait, TrackShared};
use super::TrackInner;
use crate::prelude::*;
use crate::proto;
use livekit_webrtc as rtc;
use std::sync::Arc;
use tokio::sync::mpsc;
#[derive(Debug)]
#[derive(Clone, Debug)]
pub struct RemoteAudioTrack {
shared: TrackShared,
pub(crate) inner: Arc<TrackInner>,
}
impl RemoteAudioTrack {
pub(crate) fn new(sid: TrackSid, name: String, track: Arc<AudioTrack>) -> Self {
pub(crate) fn new(
sid: TrackSid,
name: String,
rtc_track: rtc::media_stream::RtcAudioTrack,
) -> Self {
Self {
shared: TrackShared::new(
inner: Arc::new(TrackInner::new(
sid,
name,
TrackKind::Audio,
MediaStreamTrackHandle::Audio(track),
),
rtc::media_stream::MediaStreamTrack::Audio(rtc_track),
)),
}
}
pub fn rtc_track(&self) -> Arc<AudioTrack> {
if let MediaStreamTrackHandle::Audio(audio) = &self.shared.rtc_track {
audio.clone()
#[inline]
pub fn sid(&self) -> TrackSid {
self.inner.sid()
}
#[inline]
pub fn name(&self) -> String {
self.inner.name()
}
#[inline]
pub fn kind(&self) -> TrackKind {
self.inner.kind()
}
#[inline]
pub fn source(&self) -> TrackSource {
self.inner.source()
}
#[inline]
pub fn stream_state(&self) -> StreamState {
self.inner.stream_state()
}
#[inline]
pub fn start(&self) {
self.inner.start()
}
#[inline]
pub fn stop(&self) {
self.inner.stop()
}
#[inline]
pub fn muted(&self) -> bool {
self.inner.muted()
}
#[inline]
pub fn set_muted(&self, muted: bool) {
self.inner.set_muted(muted)
}
#[inline]
pub fn rtc_track(&self) -> rtc::media_stream::RtcAudioTrack {
if let rtc::media_stream::MediaStreamTrack::Audio(audio) = self.inner.rtc_track() {
audio
} else {
unreachable!()
}
}
}
impl_track_trait!(RemoteAudioTrack);
#[inline]
pub fn register_observer(&self) -> mpsc::UnboundedReceiver<TrackEvent> {
self.inner.register_observer()
}
#[inline]
pub(crate) fn transceiver(&self) -> Option<rtc::rtp_transceiver::RtpTransceiver> {
self.inner.transceiver()
}
#[inline]
pub(crate) fn update_transceiver(
&self,
transceiver: Option<rtc::rtp_transceiver::RtpTransceiver>,
) {
self.inner.update_transceiver(transceiver)
}
#[inline]
pub(crate) fn update_info(&self, info: proto::TrackInfo) {
self.inner.update_info(info)
}
}
livekit/src/room/track/remote_track.rs
-32
View File
@@ -1,32 +0,0 @@
use super::impl_track_trait;
use crate::prelude::*;
use std::sync::Arc;
#[derive(Clone, Debug)]
pub enum RemoteTrackHandle {
Audio(Arc<RemoteAudioTrack>),
Video(Arc<RemoteVideoTrack>),
}
impl From<RemoteTrackHandle> for TrackHandle {
fn from(remote_track: RemoteTrackHandle) -> Self {
match remote_track {
RemoteTrackHandle::Audio(remote_audio) => Self::RemoteAudio(remote_audio),
RemoteTrackHandle::Video(remote_video) => Self::RemoteVideo(remote_video),
}
}
}
impl TryFrom<TrackHandle> for RemoteTrackHandle {
type Error = &'static str;
fn try_from(track: TrackHandle) -> Result<Self, Self::Error> {
match track {
TrackHandle::RemoteAudio(remote_audio) => Ok(Self::Audio(remote_audio)),
TrackHandle::RemoteVideo(remote_video) => Ok(Self::Video(remote_video)),
_ => Err("not a remote track"),
}
}
}
impl_track_trait!(RemoteTrackHandle, enum_dispatch, [Audio, Video]);
livekit/src/room/track/remote_video_track.rs
+86 -12
View File
@@ -1,31 +1,105 @@
use super::{impl_track_trait, TrackShared};
use super::TrackInner;
use crate::prelude::*;
use crate::proto;
use livekit_webrtc as rtc;
use std::sync::Arc;
use tokio::sync::mpsc;
#[derive(Debug)]
#[derive(Clone, Debug)]
pub struct RemoteVideoTrack {
shared: TrackShared,
pub(crate) inner: Arc<TrackInner>,
}
impl RemoteVideoTrack {
pub(crate) fn new(sid: TrackSid, name: String, track: Arc<VideoTrack>) -> Self {
pub(crate) fn new(
sid: TrackSid,
name: String,
rtc_track: rtc::media_stream::RtcVideoTrack,
) -> Self {
Self {
shared: TrackShared::new(
inner: Arc::new(TrackInner::new(
sid,
name,
TrackKind::Video,
MediaStreamTrackHandle::Video(track),
),
rtc::media_stream::MediaStreamTrack::Video(rtc_track),
)),
}
}
pub fn rtc_track(&self) -> Arc<VideoTrack> {
if let MediaStreamTrackHandle::Video(video) = &self.shared.rtc_track {
video.clone()
#[inline]
pub fn sid(&self) -> TrackSid {
self.inner.sid()
}
#[inline]
pub fn name(&self) -> String {
self.inner.name()
}
#[inline]
pub fn kind(&self) -> TrackKind {
self.inner.kind()
}
#[inline]
pub fn source(&self) -> TrackSource {
self.inner.source()
}
#[inline]
pub fn stream_state(&self) -> StreamState {
self.inner.stream_state()
}
#[inline]
pub fn start(&self) {
self.inner.start()
}
#[inline]
pub fn stop(&self) {
self.inner.stop()
}
#[inline]
pub fn muted(&self) -> bool {
self.inner.muted()
}
#[inline]
pub fn set_muted(&self, muted: bool) {
self.inner.set_muted(muted)
}
#[inline]
pub fn rtc_track(&self) -> rtc::media_stream::RtcVideoTrack {
if let rtc::media_stream::MediaStreamTrack::Video(video) = self.inner.rtc_track() {
video
} else {
unreachable!()
}
}
}
impl_track_trait!(RemoteVideoTrack);
#[inline]
pub fn register_observer(&self) -> mpsc::UnboundedReceiver<TrackEvent> {
self.inner.register_observer()
}
#[inline]
pub(crate) fn transceiver(&self) -> Option<rtc::rtp_transceiver::RtpTransceiver> {
self.inner.transceiver()
}
#[inline]
pub(crate) fn update_transceiver(
&self,
transceiver: Option<rtc::rtp_transceiver::RtpTransceiver>,
) {
self.inner.update_transceiver(transceiver)
}
#[inline]
pub(crate) fn update_info(&self, info: proto::TrackInfo) {
self.inner.update_info(info);
}
}
livekit/src/room/track/video_track.rs
-32
View File
@@ -1,32 +0,0 @@
use super::impl_track_trait;
use crate::prelude::*;
use std::sync::Arc;
#[derive(Clone)]
pub enum VideoTrackHandle {
Local(Arc<LocalVideoTrack>),
Remote(Arc<RemoteVideoTrack>),
}
impl From<VideoTrackHandle> for TrackHandle {
fn from(video_track: VideoTrackHandle) -> Self {
match video_track {
VideoTrackHandle::Local(local_video) => Self::LocalVideo(local_video),
VideoTrackHandle::Remote(remote_video) => Self::RemoteVideo(remote_video),
}
}
}
impl TryFrom<TrackHandle> for VideoTrackHandle {
type Error = &'static str;
fn try_from(track: TrackHandle) -> Result<Self, Self::Error> {
match track {
TrackHandle::LocalVideo(local_video) => Ok(Self::Local(local_video)),
TrackHandle::RemoteVideo(remote_video) => Ok(Self::Remote(remote_video)),
_ => Err("not a video track"),
}
}
}
impl_track_trait!(VideoTrackHandle, enum_dispatch, [Local, Remote]);
livekit/src/rtc_engine/lk_runtime.rs
+32 -19
View File
@@ -1,35 +1,48 @@
use livekit_webrtc::peer_connection_factory::PeerConnectionFactory;
use livekit_webrtc::webrtc::RTCRuntime;
use lazy_static::lazy_static;
use livekit_webrtc::prelude::*;
use parking_lot::Mutex;
use std::fmt::{Debug, Formatter};
use std::sync::{Arc, Weak};
use tracing::trace;
/// SAFETY: The order of initialization and deletion is important for LKRuntime.
/// See the C++ constructors & destructors of these fields
pub struct LKRuntime {
pub pc_factory: PeerConnectionFactory,
pub rtc_runtime: RTCRuntime,
lazy_static! {
static ref LK_RUNTIME: Mutex<Weak<LkRuntime>> = Mutex::new(Weak::new());
}
impl Debug for LKRuntime {
pub struct LkRuntime {
pub pc_factory: PeerConnectionFactory,
}
impl Debug for LkRuntime {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
write!(f, "LKRuntime")
f.debug_struct("LkRuntime").finish()
}
}
impl Default for LKRuntime {
fn default() -> Self {
trace!("LKRuntime::default()");
let rtc_runtime = RTCRuntime::new();
Self {
pc_factory: PeerConnectionFactory::new(rtc_runtime.clone()),
rtc_runtime,
impl LkRuntime {
pub fn instance() -> Arc<LkRuntime> {
let mut lk_runtime_ref = LK_RUNTIME.lock();
if let Some(lk_runtime) = lk_runtime_ref.upgrade() {
lk_runtime
} else {
let new_runtime = Arc::new(LkRuntime::default());
*lk_runtime_ref = Arc::downgrade(&new_runtime);
new_runtime
}
}
}
impl Drop for LKRuntime {
impl Default for LkRuntime {
fn default() -> Self {
trace!("LkRuntime::default()");
Self {
pc_factory: PeerConnectionFactory::default(),
}
}
}
impl Drop for LkRuntime {
fn drop(&mut self) {
trace!("LKRuntime::drop()");
trace!("LkRuntime::drop()");
}
}
livekit/src/rtc_engine/mod.rs
+102 -62
View File
@@ -1,14 +1,11 @@
use crate::prelude::*;
use crate::options::TrackPublishOptions;
use crate::prelude::LocalTrack;
use crate::proto;
use crate::rtc_engine::lk_runtime::LKRuntime;
use crate::rtc_engine::rtc_session::{RTCSession, SessionEvent, SessionEvents, SessionInfo};
use crate::rtc_engine::lk_runtime::LkRuntime;
use crate::rtc_engine::rtc_session::{RtcSession, SessionEvent, SessionEvents, SessionInfo};
use crate::signal_client::{SignalError, SignalOptions};
use futures::future::BoxFuture;
use futures::FutureExt;
use lazy_static::lazy_static;
use livekit_webrtc::data_channel::DataSendError;
use livekit_webrtc::jsep::SdpParseError;
use livekit_webrtc::prelude::*;
use livekit_webrtc::session_description::SdpParseError;
use parking_lot::Mutex;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Weak};
@@ -20,8 +17,8 @@ use tokio::task::JoinHandle;
use tokio::time::{interval, Interval};
use tracing::{error, info, warn};
mod lk_runtime;
mod pc_transport;
pub mod lk_runtime;
mod peer_transport;
mod rtc_events;
mod rtc_session;
@@ -46,13 +43,13 @@ pub enum EngineError {
#[error("signal failure: {0}")]
Signal(#[from] SignalError),
#[error("internal webrtc failure")]
Rtc(#[from] RTCError),
Rtc(#[from] RtcError),
#[error("failed to parse sdp")]
Parse(#[from] SdpParseError),
#[error("serde error")]
Serde(#[from] serde_json::Error),
#[error("failed to send data to the datachannel")]
Data(#[from] DataSendError),
Data(#[from] DataChannelError),
#[error("connection error: {0}")]
Connection(String),
#[error("decode error")]
@@ -67,7 +64,7 @@ pub enum EngineEvent {
updates: Vec<proto::ParticipantInfo>,
},
MediaTrack {
track: MediaStreamTrackHandle,
track: MediaStreamTrack,
stream: MediaStream,
receiver: RtpReceiver,
},
@@ -92,23 +89,19 @@ pub enum EngineEvent {
pub const RECONNECT_ATTEMPTS: u32 = 10;
pub const RECONNECT_INTERVAL: Duration = Duration::from_secs(5);
lazy_static! {
// Share one LKRuntime across all RTCEngine instances
static ref LK_RUNTIME: Mutex<Weak<LKRuntime>> = Mutex::new(Weak::new());
}
///
/// Represents a running RTCSession with the ability to close the session
/// and the engine_task
#[derive(Debug)]
struct EngineHandle {
session: RTCSession,
session: RtcSession,
engine_task: JoinHandle<()>,
close_sender: oneshot::Sender<()>,
}
#[derive(Debug)]
struct EngineInner {
lk_runtime: Arc<LKRuntime>,
lk_runtime: Arc<LkRuntime>,
session_info: Mutex<Option<SessionInfo>>, // Last/Current Sessioninfo
running_handle: AsyncRwLock<Option<EngineHandle>>,
opened: AtomicBool,
@@ -121,26 +114,15 @@ struct EngineInner {
}
#[derive(Debug)]
pub struct RTCEngine {
pub struct RtcEngine {
inner: Arc<EngineInner>,
}
impl RTCEngine {
impl RtcEngine {
pub fn new() -> (Self, EngineEvents) {
let lk_runtime = {
let mut lk_runtime_ref = LK_RUNTIME.lock();
if let Some(lk_runtime) = lk_runtime_ref.upgrade() {
lk_runtime
} else {
let new_runtime = Arc::new(LKRuntime::default());
*lk_runtime_ref = Arc::downgrade(&new_runtime);
new_runtime
}
};
let (engine_emitter, engine_events) = mpsc::channel(8);
let inner = Arc::new(EngineInner {
lk_runtime,
lk_runtime: LkRuntime::instance(),
session_info: Default::default(),
running_handle: Default::default(),
opened: Default::default(),
@@ -153,6 +135,10 @@ impl RTCEngine {
(Self { inner }, engine_events)
}
pub(crate) fn lk_runtime(&self) -> Arc<LkRuntime> {
self.inner.lk_runtime.clone()
}
#[tracing::instrument]
pub async fn connect(
&self,
@@ -200,6 +186,64 @@ impl RTCEngine {
Ok(())
}
pub async fn add_track(&self, req: proto::AddTrackRequest) -> EngineResult<proto::TrackInfo> {
self.inner.wait_reconnection().await?;
self.inner
.running_handle
.read()
.await
.as_ref()
.unwrap()
.session
.add_track(req)
.await
}
pub async fn remove_track(&self, sender: RtpSender) -> EngineResult<()> {
self.inner.wait_reconnection().await?;
self.inner
.running_handle
.read()
.await
.as_ref()
.unwrap()
.session
.remove_track(sender)
.await
}
pub async fn create_sender(
&self,
track: LocalTrack,
options: TrackPublishOptions,
encodings: Vec<RtpEncodingParameters>,
) -> EngineResult<RtpTransceiver> {
self.inner.wait_reconnection().await?;
self.inner
.running_handle
.read()
.await
.as_ref()
.unwrap()
.session
.create_sender(track, options, encodings)
.await
}
pub async fn negotiate_publisher(&self) -> EngineResult<()> {
// TODO(theomonnom): guard for reconnection
self.inner.wait_reconnection().await?;
self.inner
.running_handle
.read()
.await
.as_ref()
.unwrap()
.session
.negotiate_publisher()
.await
}
pub fn join_response(&self) -> Option<proto::JoinResponse> {
if let Some(info) = self.inner.session_info.lock().as_ref() {
Some(info.join_response.clone())
@@ -307,37 +351,33 @@ impl EngineInner {
Ok(())
}
fn connect<'a>(
self: &'a Arc<Self>,
url: &'a str,
token: &'a str,
async fn connect(
self: &Arc<Self>,
url: &str,
token: &str,
options: SignalOptions,
) -> BoxFuture<'a, EngineResult<()>> {
async {
let (session_emitter, session_events) = mpsc::unbounded_channel();
let session = RTCSession::connect(
url,
token,
options,
self.lk_runtime.clone(),
session_emitter,
)
.await?;
) -> EngineResult<()> {
let (session_emitter, session_events) = mpsc::unbounded_channel();
let session = RtcSession::connect(
url,
token,
options,
self.lk_runtime.clone(),
session_emitter,
)
.await?;
let (close_sender, close_receiver) = oneshot::channel();
let engine_task =
tokio::spawn(self.clone().engine_task(session_events, close_receiver));
*self.session_info.lock() = Some(session.info().clone());
*self.running_handle.write().await = Some(EngineHandle {
session,
engine_task,
close_sender,
});
let (close_sender, close_receiver) = oneshot::channel();
let engine_task = tokio::spawn(self.clone().engine_task(session_events, close_receiver));
*self.session_info.lock() = Some(session.info().clone());
*self.running_handle.write().await = Some(EngineHandle {
session,
engine_task,
close_sender,
});
self.opened.store(true, Ordering::SeqCst);
Ok(())
}
.boxed()
self.opened.store(true, Ordering::SeqCst);
Ok(())
}
async fn terminate_session(&self) {
@@ -362,7 +402,7 @@ impl EngineInner {
}
while self.reconnecting.load(Ordering::Acquire) {
tokio::task::yield_now().await;
tokio::task::yield_now().await; // TODO(theomonnom): Remove yield
}
if self.running_handle.read().await.is_none() {
livekit/src/rtc_engine/pc_transport.rs → livekit/src/rtc_engine/peer_transport.rs
+28 -38
View File
@@ -1,35 +1,31 @@
use crate::proto;
use livekit_webrtc::prelude::*;
use std::fmt::{Debug, Formatter};
use std::future::Future;
use std::pin::Pin;
use std::time::Duration;
use tracing::{event, Level};
use tracing::{debug, event, Level};
const NEGOTIATION_FREQUENCY: Duration = Duration::from_millis(150);
pub type OnOfferHandler = Box<
dyn (FnMut(SessionDescription) -> Pin<Box<dyn Future<Output = ()> + Send + 'static>>)
+ Send
+ Sync,
>;
pub type OnOfferCreated = Box<dyn FnMut(SessionDescription) + Send + Sync>;
pub struct PCTransport {
pub struct PeerTransport {
signal_target: proto::SignalTarget,
peer_connection: PeerConnection,
pending_candidates: Vec<IceCandidate>,
on_offer_handler: Option<OnOfferHandler>,
on_offer_handler: Option<OnOfferCreated>,
renegotiate: bool,
restarting_ice: bool,
}
impl Debug for PCTransport {
impl Debug for PeerTransport {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
f.write_str("PCTransport")
f.debug_struct("PeerTransport")
.field("target", &self.signal_target)
.finish()
}
}
impl PCTransport {
impl PeerTransport {
pub fn new(peer_connection: PeerConnection, signal_target: proto::SignalTarget) -> Self {
Self {
signal_target,
@@ -42,9 +38,10 @@ impl PCTransport {
}
pub fn is_connected(&self) -> bool {
self.peer_connection.ice_connection_state() == IceConnectionState::IceConnectionConnected
|| self.peer_connection.ice_connection_state()
== IceConnectionState::IceConnectionCompleted
matches!(
self.peer_connection.ice_connection_state(),
IceConnectionState::Connected | IceConnectionState::Completed
)
}
pub fn peer_connection(&mut self) -> &mut PeerConnection {
@@ -55,8 +52,8 @@ impl PCTransport {
self.signal_target.clone()
}
pub fn on_offer(&mut self, handler: OnOfferHandler) {
self.on_offer_handler = Some(handler);
pub fn on_offer(&mut self, handler: Option<OnOfferCreated>) {
self.on_offer_handler = handler;
}
pub fn prepare_ice_restart(&mut self) {
@@ -68,8 +65,8 @@ impl PCTransport {
}
#[tracing::instrument(level = Level::DEBUG)]
pub async fn add_ice_candidate(&mut self, ice_candidate: IceCandidate) -> Result<(), RTCError> {
if self.peer_connection.remote_description().is_some() && !self.restarting_ice {
pub async fn add_ice_candidate(&mut self, ice_candidate: IceCandidate) -> Result<(), RtcError> {
if self.peer_connection.current_remote_description().is_some() && !self.restarting_ice {
self.peer_connection
.add_ice_candidate(ice_candidate)
.await?;
@@ -85,7 +82,7 @@ impl PCTransport {
pub async fn set_remote_description(
&mut self,
remote_description: SessionDescription,
) -> Result<(), RTCError> {
) -> Result<(), RtcError> {
self.peer_connection
.set_remote_description(remote_description)
.await?;
@@ -97,31 +94,26 @@ impl PCTransport {
if self.renegotiate {
self.renegotiate = false;
self.create_and_send_offer(RTCOfferAnswerOptions::default())
.await?;
self.create_and_send_offer(OfferOptions::default()).await?;
}
Ok(())
}
#[tracing::instrument(level = Level::DEBUG)]
pub async fn negotiate(&mut self) -> Result<(), RTCError> {
pub async fn negotiate(&mut self) -> Result<(), RtcError> {
// TODO(theomonnom) Debounce here with NEGOTIATION_FREQUENCY
self.create_and_send_offer(RTCOfferAnswerOptions::default())
.await
self.create_and_send_offer(OfferOptions::default()).await
}
#[tracing::instrument(level = Level::DEBUG)]
pub async fn create_anwser(
&mut self,
offer: SessionDescription,
options: RTCOfferAnswerOptions,
) -> Result<SessionDescription, RTCError> {
options: AnswerOptions,
) -> Result<SessionDescription, RtcError> {
self.set_remote_description(offer).await?;
let answer = self
.peer_connection()
.create_answer(RTCOfferAnswerOptions::default())
.await?;
let answer = self.peer_connection().create_answer(options).await?;
self.peer_connection()
.set_local_description(answer.clone())
.await?;
@@ -130,10 +122,7 @@ impl PCTransport {
}
#[tracing::instrument(level = Level::DEBUG)]
pub async fn create_and_send_offer(
&mut self,
options: RTCOfferAnswerOptions,
) -> Result<(), RTCError> {
pub async fn create_and_send_offer(&mut self, options: OfferOptions) -> Result<(), RtcError> {
if self.on_offer_handler.is_none() {
return Ok(());
}
@@ -145,7 +134,8 @@ impl PCTransport {
if self.peer_connection.signaling_state() == SignalingState::HaveLocalOffer {
if options.ice_restart {
if let Some(remote_description) = self.peer_connection.remote_description() {
if let Some(remote_description) = self.peer_connection.current_remote_description()
{
self.peer_connection
.set_remote_description(remote_description)
.await?;
@@ -165,7 +155,7 @@ impl PCTransport {
self.peer_connection
.set_local_description(offer.clone())
.await?;
self.on_offer_handler.as_mut().unwrap()(offer).await;
self.on_offer_handler.as_mut().unwrap()(offer);
Ok(())
}
}
livekit/src/rtc_engine/rtc_events.rs
+62 -56
View File
@@ -1,20 +1,15 @@
use super::pc_transport::PCTransport;
use super::peer_transport::PeerTransport;
use crate::proto;
use crate::rtc_engine::pc_transport::OnOfferHandler;
use livekit_webrtc::data_channel::OnMessageHandler;
use livekit_webrtc::peer_connection::{
OnAddTrackHandler, OnConnectionChangeHandler, OnDataChannelHandler, OnIceCandidateErrorHandler,
OnIceCandidateHandler, PeerConnectionState,
};
use livekit_webrtc::prelude::*;
use crate::rtc_engine::peer_transport::OnOfferCreated;
use livekit_webrtc::{self as rtc, prelude::*};
use tokio::sync::mpsc;
use tracing::error;
use tracing::{debug, error};
pub type RTCEmitter = mpsc::UnboundedSender<RTCEvent>;
pub type RTCEvents = mpsc::UnboundedReceiver<RTCEvent>;
pub type RtcEmitter = mpsc::UnboundedSender<RtcEvent>;
pub type RtcEvents = mpsc::UnboundedReceiver<RtcEvent>;
#[derive(Debug)]
pub enum RTCEvent {
pub enum RtcEvent {
IceCandidate {
ice_candidate: IceCandidate,
target: proto::SignalTarget,
@@ -27,14 +22,16 @@ pub enum RTCEvent {
data_channel: DataChannel,
target: proto::SignalTarget,
},
// TODO (theomonnom): Move Offer to PCTransport
// TODO (theomonnom): Move Offer to PeerTransport
Offer {
offer: SessionDescription,
target: proto::SignalTarget,
},
AddTrack {
rtp_receiver: RtpReceiver,
Track {
receiver: RtpReceiver,
streams: Vec<MediaStream>,
track: MediaStreamTrack,
transceiver: RtpTransceiver,
target: proto::SignalTarget,
},
Data {
@@ -46,99 +43,108 @@ pub enum RTCEvent {
/// Handlers used to forward events to a channel
/// Every callback here is called on the signaling thread
fn on_connection_change(
fn on_connection_state_change(
target: proto::SignalTarget,
emitter: RTCEmitter,
) -> OnConnectionChangeHandler {
emitter: RtcEmitter,
) -> rtc::peer_connection::OnConnectionChange {
Box::new(move |state| {
let _ = emitter.send(RTCEvent::ConnectionChange { state, target });
let _ = emitter.send(RtcEvent::ConnectionChange { state, target });
})
}
fn on_ice_candidate(target: proto::SignalTarget, emitter: RTCEmitter) -> OnIceCandidateHandler {
fn on_ice_candidate(
target: proto::SignalTarget,
emitter: RtcEmitter,
) -> rtc::peer_connection::OnIceCandidate {
Box::new(move |ice_candidate| {
let _ = emitter.send(RTCEvent::IceCandidate {
let _ = emitter.send(RtcEvent::IceCandidate {
ice_candidate,
target,
});
})
}
fn on_offer(target: proto::SignalTarget, emitter: RTCEmitter) -> OnOfferHandler {
fn on_offer(target: proto::SignalTarget, emitter: RtcEmitter) -> OnOfferCreated {
Box::new(move |offer| {
let _ = emitter.send(RTCEvent::Offer { offer, target });
Box::pin(async {})
let _ = emitter.send(RtcEvent::Offer { offer, target });
})
}
fn on_data_channel(target: proto::SignalTarget, emitter: RTCEmitter) -> OnDataChannelHandler {
Box::new(move |mut data_channel| {
data_channel.on_message(on_message(emitter.clone()));
fn on_data_channel(
target: proto::SignalTarget,
emitter: RtcEmitter,
) -> rtc::peer_connection::OnDataChannel {
Box::new(move |data_channel| {
data_channel.on_message(Some(on_message(emitter.clone())));
let _ = emitter.send(RTCEvent::DataChannel {
let _ = emitter.send(RtcEvent::DataChannel {
data_channel,
target,
});
})
}
fn on_add_track(target: proto::SignalTarget, emitter: RTCEmitter) -> OnAddTrackHandler {
Box::new(move |rtp_receiver, streams| {
let _ = emitter.send(RTCEvent::AddTrack {
rtp_receiver,
streams,
fn on_track(target: proto::SignalTarget, emitter: RtcEmitter) -> rtc::peer_connection::OnTrack {
Box::new(move |event| {
let _ = emitter.send(RtcEvent::Track {
receiver: event.receiver,
streams: event.streams,
track: event.track,
transceiver: event.transceiver,
target,
});
})
}
fn on_ice_candidate_error(
target: proto::SignalTarget,
_emitter: RTCEmitter,
) -> OnIceCandidateErrorHandler {
Box::new(move |address, port, url, error_code, error_text| {
error!(
"ICE candidate error ({:?}): address: {} - port: {} - url: {} - error_code: {} - error_text: {}",
target, address, port, url, error_code, error_text
);
_target: proto::SignalTarget,
_emitter: RtcEmitter,
) -> rtc::peer_connection::OnIceCandidateError {
Box::new(move |ice_error| {
error!("{:?}", ice_error);
})
}
pub fn forward_pc_events(transport: &mut PCTransport, rtc_emitter: RTCEmitter) {
pub fn forward_pc_events(transport: &mut PeerTransport, rtc_emitter: RtcEmitter) {
let signal_target = transport.signal_target();
transport
.peer_connection()
.on_ice_candidate(on_ice_candidate(signal_target, rtc_emitter.clone()));
.on_ice_candidate(Some(on_ice_candidate(signal_target, rtc_emitter.clone())));
transport
.peer_connection()
.on_data_channel(on_data_channel(signal_target, rtc_emitter.clone()));
.on_data_channel(Some(on_data_channel(signal_target, rtc_emitter.clone())));
transport
.peer_connection()
.on_add_track(on_add_track(signal_target, rtc_emitter.clone()));
.on_track(Some(on_track(signal_target, rtc_emitter.clone())));
transport
.peer_connection()
.on_connection_change(on_connection_change(signal_target, rtc_emitter.clone()));
.on_connection_state_change(Some(on_connection_state_change(
signal_target,
rtc_emitter.clone(),
)));
transport
.peer_connection()
.on_ice_candidate_error(on_ice_candidate_error(signal_target, rtc_emitter.clone()));
.on_ice_candidate_error(Some(on_ice_candidate_error(
signal_target,
rtc_emitter.clone(),
)));
transport.on_offer(on_offer(transport.signal_target(), rtc_emitter.clone()));
transport.on_offer(Some(on_offer(signal_target, rtc_emitter.clone())));
}
fn on_message(emitter: RTCEmitter) -> OnMessageHandler {
Box::new(move |data, binary| {
let _ = emitter.send(RTCEvent::Data {
data: data.to_vec(),
binary,
fn on_message(emitter: RtcEmitter) -> rtc::data_channel::OnMessage {
Box::new(move |buffer| {
let _ = emitter.send(RtcEvent::Data {
data: buffer.data.to_vec(),
binary: buffer.binary,
});
})
}
pub fn forward_dc_events(dc: &mut DataChannel, rtc_emitter: RTCEmitter) {
dc.on_message(on_message(rtc_emitter.clone()));
pub fn forward_dc_events(dc: &mut DataChannel, rtc_emitter: RtcEmitter) {
dc.on_message(Some(on_message(rtc_emitter.clone())));
}
livekit/src/rtc_engine/rtc_session.rs
+221 -68
View File
@@ -1,23 +1,28 @@
use super::{rtc_events, EngineError, EngineResult, SimulateScenario};
use crate::rtc_engine::lk_runtime::LKRuntime;
use crate::rtc_engine::pc_transport::PCTransport;
use crate::rtc_engine::rtc_events::{RTCEvent, RTCEvents};
use crate::options::TrackPublishOptions;
use crate::rtc_engine::lk_runtime::LkRuntime;
use crate::rtc_engine::peer_transport::PeerTransport;
use crate::rtc_engine::rtc_events::{RtcEvent, RtcEvents};
use crate::signal_client::{SignalClient, SignalEvent, SignalEvents, SignalOptions};
use crate::track::LocalTrack;
use crate::{proto, signal_client};
use livekit_webrtc::prelude::*;
use parking_lot::Mutex;
use prost::Message;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::convert::TryInto;
use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::{mpsc, watch, Mutex as AsyncMutex};
use tokio::sync::Mutex as AsyncMutex;
use tokio::sync::{mpsc, oneshot, watch};
use tokio::task::JoinHandle;
use tokio::time::sleep;
use tracing::{debug, error, trace, warn};
pub const MAX_ICE_CONNECT_TIMEOUT: Duration = Duration::from_secs(15);
pub const ICE_CONNECT_TIMEOUT: Duration = Duration::from_secs(15);
pub const TRACK_PUBLISH_TIMEOUT: Duration = Duration::from_secs(10);
pub const LOSSY_DC_LABEL: &str = "_lossy";
pub const RELIABLE_DC_LABEL: &str = "_reliable";
@@ -35,7 +40,7 @@ pub enum SessionEvent {
kind: proto::data_packet::Kind,
},
MediaTrack {
track: MediaStreamTrackHandle,
track: MediaStreamTrack,
stream: MediaStream,
receiver: RtpReceiver,
},
@@ -56,8 +61,8 @@ pub enum SessionEvent {
Connected,
}
#[repr(u8)]
pub enum PCState {
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PeerState {
New,
Connected,
Disconnected,
@@ -65,27 +70,27 @@ pub enum PCState {
Closed,
}
impl TryInto<PCState> for u8 {
impl TryFrom<u8> for PeerState {
type Error = &'static str;
fn try_into(self) -> Result<PCState, Self::Error> {
match self {
0 => Ok(PCState::New),
1 => Ok(PCState::Connected),
2 => Ok(PCState::Disconnected),
3 => Ok(PCState::Reconnecting),
4 => Ok(PCState::Closed),
_ => Err("invalid PCState"),
fn try_from(v: u8) -> Result<Self, Self::Error> {
match v {
0 => Ok(Self::New),
1 => Ok(Self::Connected),
2 => Ok(Self::Disconnected),
3 => Ok(Self::Reconnecting),
4 => Ok(Self::Closed),
_ => Err("invalid PeerState"),
}
}
}
#[derive(Serialize, Deserialize)]
#[allow(non_snake_case)]
struct IceCandidateJSON {
sdpMid: String,
sdpMLineIndex: i32,
candidate: String,
#[serde(rename_all = "camelCase")]
struct IceCandidateJson {
pub sdp_mid: String,
pub sdp_m_line_index: i32,
pub candidate: String,
}
#[derive(Debug, Clone, Default)]
@@ -104,8 +109,10 @@ struct SessionInner {
pc_state: AtomicU8, // PCState
has_published: AtomicBool,
publisher_pc: AsyncMutex<PCTransport>,
subscriber_pc: AsyncMutex<PCTransport>,
publisher_pc: AsyncMutex<PeerTransport>,
subscriber_pc: AsyncMutex<PeerTransport>,
pending_tracks: Mutex<HashMap<String, oneshot::Sender<proto::TrackInfo>>>,
// Publisher data channels
// used to send data to other participants ( The SFU forwards the messages )
@@ -124,20 +131,21 @@ struct SessionInner {
///
/// RTCSession is also responsable for the signaling and the negotation
#[derive(Debug)]
pub struct RTCSession {
lk_runtime: Arc<LKRuntime>,
pub struct RtcSession {
#[allow(dead_code)]
lk_runtime: Arc<LkRuntime>,
inner: Arc<SessionInner>,
close_tx: watch::Sender<bool>, // false = is_running
signal_task: JoinHandle<()>,
rtc_task: JoinHandle<()>,
}
impl RTCSession {
impl RtcSession {
pub async fn connect(
url: &str,
token: &str,
options: SignalOptions,
lk_runtime: Arc<LKRuntime>,
lk_runtime: Arc<LkRuntime>,
session_emitter: SessionEmitter,
) -> EngineResult<Self> {
// Connect to the SignalClient
@@ -148,16 +156,16 @@ impl RTCSession {
debug!("received JoinResponse: {:?}", join_response);
let (rtc_emitter, rtc_events) = mpsc::unbounded_channel();
let rtc_config = RTCConfiguration::from(join_response.clone());
let rtc_config = RtcConfiguration::from(join_response.clone());
let mut publisher_pc = PCTransport::new(
let mut publisher_pc = PeerTransport::new(
lk_runtime
.pc_factory
.create_peer_connection(rtc_config.clone())?,
proto::SignalTarget::Publisher,
);
let mut subscriber_pc = PCTransport::new(
let mut subscriber_pc = PeerTransport::new(
lk_runtime
.pc_factory
.create_peer_connection(rtc_config.clone())?,
@@ -181,7 +189,7 @@ impl RTCSession {
},
)?;
// Forward events received in the Signaling Thread to our rtc channel
// Forward events received inside the signaling thread to our rtc channel
rtc_events::forward_pc_events(&mut publisher_pc, rtc_emitter.clone());
rtc_events::forward_pc_events(&mut subscriber_pc, rtc_emitter.clone());
rtc_events::forward_dc_events(&mut lossy_dc, rtc_emitter.clone());
@@ -197,11 +205,12 @@ impl RTCSession {
let (close_tx, close_rx) = watch::channel(false);
let inner = Arc::new(SessionInner {
info: session_info,
pc_state: AtomicU8::new(PCState::New as u8),
pc_state: AtomicU8::new(PeerState::New as u8),
has_published: Default::default(),
signal_client,
publisher_pc: AsyncMutex::new(publisher_pc),
subscriber_pc: AsyncMutex::new(subscriber_pc),
pending_tracks: Default::default(),
lossy_dc,
reliable_dc,
subscriber_dc: Default::default(),
@@ -228,6 +237,31 @@ impl RTCSession {
Ok(session)
}
#[inline]
pub async fn add_track(&self, req: proto::AddTrackRequest) -> EngineResult<proto::TrackInfo> {
self.inner.add_track(req).await
}
#[inline]
pub async fn remove_track(&self, sender: RtpSender) -> EngineResult<()> {
self.inner.remove_track(sender).await
}
#[inline]
pub async fn create_sender(
&self,
track: LocalTrack,
options: TrackPublishOptions,
encodings: Vec<RtpEncodingParameters>,
) -> EngineResult<RtpTransceiver> {
self.inner.create_sender(track, options, encodings).await
}
#[inline]
pub async fn negotiate_publisher(&self) -> EngineResult<()> {
self.inner.negotiate_publisher().await
}
/// Close the PeerConnections and the SignalClient
#[tracing::instrument]
pub async fn close(self) {
@@ -238,6 +272,7 @@ impl RTCSession {
let _ = self.signal_task.await;
}
#[inline]
pub async fn publish_data(
&self,
data: &proto::DataPacket,
@@ -246,25 +281,28 @@ impl RTCSession {
self.inner.publish_data(data, kind).await
}
#[inline]
pub async fn restart(&self) -> EngineResult<()> {
self.inner.restart_session().await
}
#[inline]
pub async fn wait_pc_connection(&self) -> EngineResult<()> {
self.inner.wait_pc_connection().await
}
#[inline]
pub async fn simulate_scenario(&self, scenario: SimulateScenario) {
self.inner.simulate_scenario(scenario).await
}
}
impl RTCSession {
#[inline]
pub fn info(&self) -> &SessionInfo {
&self.inner.info
}
pub fn state(&self) -> PCState {
#[inline]
pub fn state(&self) -> PeerState {
self.inner
.pc_state
.load(Ordering::SeqCst)
@@ -272,18 +310,22 @@ impl RTCSession {
.unwrap()
}
pub fn publisher(&self) -> &AsyncMutex<PCTransport> {
#[inline]
pub fn publisher(&self) -> &AsyncMutex<PeerTransport> {
&self.inner.publisher_pc
}
pub fn subscriber(&self) -> &AsyncMutex<PCTransport> {
#[inline]
pub fn subscriber(&self) -> &AsyncMutex<PeerTransport> {
&self.inner.subscriber_pc
}
#[inline]
pub fn signal_client(&self) -> &Arc<SignalClient> {
&self.inner.signal_client
}
#[inline]
pub fn data_channel(&self, kind: proto::data_packet::Kind) -> &DataChannel {
&self.inner.data_channel(kind)
}
@@ -292,7 +334,7 @@ impl RTCSession {
impl SessionInner {
async fn rtc_task(
self: Arc<Self>,
mut rtc_events: RTCEvents,
mut rtc_events: RtcEvents,
mut close_receiver: watch::Receiver<bool>,
) {
loop {
@@ -355,7 +397,8 @@ impl SessionInner {
match event {
proto::signal_response::Message::Answer(answer) => {
trace!("received publisher answer: {:?}", answer);
let answer = SessionDescription::from(answer.r#type.parse().unwrap(), &answer.sdp)?;
let answer =
SessionDescription::parse(&answer.sdp, answer.r#type.parse().unwrap())?;
self.publisher_pc
.lock()
.await
@@ -364,12 +407,12 @@ impl SessionInner {
}
proto::signal_response::Message::Offer(offer) => {
trace!("received subscriber offer: {:?}", offer);
let offer = SessionDescription::from(offer.r#type.parse().unwrap(), &offer.sdp)?;
let offer = SessionDescription::parse(&offer.sdp, offer.r#type.parse().unwrap())?;
let answer = self
.subscriber_pc
.lock()
.await
.create_anwser(offer, RTCOfferAnswerOptions::default())
.create_anwser(offer, AnswerOptions::default())
.await?;
self.signal_client
@@ -384,11 +427,11 @@ impl SessionInner {
proto::signal_response::Message::Trickle(trickle) => {
let target = proto::SignalTarget::from_i32(trickle.target).unwrap();
let ice_candidate = {
let json = serde_json::from_str::<IceCandidateJSON>(&trickle.candidate_init)?;
IceCandidate::from(&json.sdpMid, json.sdpMLineIndex, &json.candidate)?
let json = serde_json::from_str::<IceCandidateJson>(&trickle.candidate_init)?;
IceCandidate::parse(&json.sdp_mid, json.sdp_m_line_index, &json.candidate)?
};
trace!("received ice_candidate {:?} {:?}", target, ice_candidate);
debug!("received ice_candidate {:?} {:?}", target, ice_candidate);
if target == proto::SignalTarget::Publisher {
self.publisher_pc
@@ -428,24 +471,31 @@ impl SessionInner {
updates: quality.updates,
});
}
proto::signal_response::Message::TrackPublished(publish_res) => {
let mut pending_tracks = self.pending_tracks.lock();
if let Some(tx) = pending_tracks.remove(&publish_res.cid) {
let _ = tx.send(publish_res.track.unwrap());
}
}
_ => {}
}
Ok(())
}
async fn on_rtc_event(&self, event: RTCEvent) -> EngineResult<()> {
async fn on_rtc_event(&self, event: RtcEvent) -> EngineResult<()> {
match event {
RTCEvent::IceCandidate {
RtcEvent::IceCandidate {
ice_candidate,
target,
} => {
self.signal_client
.send(proto::signal_request::Message::Trickle(
proto::TrickleRequest {
candidate_init: serde_json::to_string(&IceCandidateJSON {
sdpMid: ice_candidate.sdp_mid(),
sdpMLineIndex: ice_candidate.sdp_mline_index(),
candidate_init: serde_json::to_string(&IceCandidateJson {
sdp_mid: ice_candidate.sdp_mid(),
sdp_m_line_index: ice_candidate.sdp_mline_index(),
candidate: ice_candidate.candidate(),
})?,
target: target as i32,
@@ -453,21 +503,21 @@ impl SessionInner {
))
.await;
}
RTCEvent::ConnectionChange { state, target } => {
trace!("connection change, {:?} {:?}", state, target);
RtcEvent::ConnectionChange { state, target } => {
debug!("connection change, {:?} {:?}", state, target);
let is_primary = self.info.join_response.subscriber_primary
&& target == proto::SignalTarget::Subscriber;
if is_primary && state == PeerConnectionState::Connected {
let old_state = self
.pc_state
.swap(PCState::Connected as u8, Ordering::SeqCst);
if old_state == PCState::New as u8 {
.swap(PeerState::Connected as u8, Ordering::SeqCst);
if old_state == PeerState::New as u8 {
let _ = self.emitter.send(SessionEvent::Connected);
}
} else if state == PeerConnectionState::Failed {
self.pc_state
.store(PCState::Disconnected as u8, Ordering::SeqCst);
.store(PeerState::Disconnected as u8, Ordering::SeqCst);
self.on_session_disconnected(
"pc_state failed",
@@ -478,14 +528,15 @@ impl SessionInner {
);
}
}
RTCEvent::DataChannel {
RtcEvent::DataChannel {
data_channel,
target: _,
} => {
self.subscriber_dc.lock().push(data_channel);
}
RTCEvent::Offer { offer, target: _ } => {
RtcEvent::Offer { offer, target: _ } => {
// Send the publisher offer to the server
debug!("sending publisher offer: {:?}", offer);
self.signal_client
.send(proto::signal_request::Message::Offer(
proto::SessionDescription {
@@ -495,22 +546,24 @@ impl SessionInner {
))
.await;
}
RTCEvent::AddTrack {
rtp_receiver,
RtcEvent::Track {
receiver,
mut streams,
track,
transceiver: _,
target: _,
} => {
if !streams.is_empty() {
let _ = self.emitter.send(SessionEvent::MediaTrack {
track: rtp_receiver.track(),
stream: streams.remove(0),
receiver: rtp_receiver,
track,
receiver,
});
} else {
warn!("AddTrack event with no streams");
warn!("Track event with no streams");
}
}
RTCEvent::Data { data, binary } => {
RtcEvent::Data { data, binary } => {
if !binary {
Err(EngineError::Internal(
"text messages aren't supported".to_string(),
@@ -534,6 +587,106 @@ impl SessionInner {
Ok(())
}
async fn add_track(&self, req: proto::AddTrackRequest) -> EngineResult<proto::TrackInfo> {
let (tx, rx) = oneshot::channel();
let cid = req.cid.clone();
{
let mut pendings_tracks = self.pending_tracks.lock();
if pendings_tracks.contains_key(&req.cid) {
Err(EngineError::Internal("track already published".to_string()))?;
}
pendings_tracks.insert(cid.clone(), tx);
}
self.signal_client
.send(proto::signal_request::Message::AddTrack(req))
.await;
// Wait the result from the server (TrackInfo)
tokio::select! {
Ok(info) = rx => Ok(info),
_ = sleep(TRACK_PUBLISH_TIMEOUT) => {
self.pending_tracks.lock().remove(&cid);
Err(EngineError::Internal("track publication timed out, no response received from the server".to_string()))
},
else => {
Err(EngineError::Internal(
"track publication cancelled".to_string(),
))
}
}
}
async fn remove_track(&self, sender: RtpSender) -> EngineResult<()> {
if let Some(track) = sender.track() {
let mut pending_tracks = self.pending_tracks.lock();
pending_tracks.remove(&track.id());
}
self.publisher_pc
.lock()
.await
.peer_connection()
.remove_track(sender)?;
Ok(())
}
async fn create_sender(
&self,
track: LocalTrack,
options: TrackPublishOptions,
encodings: Vec<RtpEncodingParameters>,
) -> EngineResult<RtpTransceiver> {
let init = RtpTransceiverInit {
direction: RtpTransceiverDirection::SendOnly,
stream_ids: Default::default(),
send_encodings: encodings,
};
let transceiver = self
.publisher_pc
.lock()
.await
.peer_connection()
.add_transceiver(track.rtc_track(), init)?;
let capabilities = LkRuntime::instance()
.pc_factory
.get_rtp_sender_capabilities(track.kind().into());
let mut matched = Vec::new();
let mut partial_matched = Vec::new();
let mut unmatched = Vec::new();
for codec in capabilities.codecs {
let mime_type = codec.mime_type.to_lowercase();
if mime_type == "audio/opus" {
matched.push(codec);
} else if mime_type == format!("video/{}", options.video_codec.as_str()) {
if let Some(sdp_fmtp_line) = codec.sdp_fmtp_line.as_ref() {
// for h264 codecs that have sdpFmtpLine available, use only if the
// profile-level-id is 42e01f for cross-browser compatibility
if sdp_fmtp_line.contains("profile-level-id=42e01f") {
matched.push(codec);
continue;
}
}
partial_matched.push(codec);
} else {
unmatched.push(codec);
}
}
matched.append(&mut partial_matched);
matched.append(&mut unmatched);
transceiver.set_codec_preferences(matched)?;
Ok(transceiver)
}
/// Called when the SignalClient or one of the PeerConnection has lost the connection
/// The RTCEngine may try a reconnect.
fn on_session_disconnected(
@@ -663,7 +816,7 @@ impl SessionInner {
self.publisher_pc
.lock()
.await
.create_and_send_offer(RTCOfferAnswerOptions {
.create_and_send_offer(OfferOptions {
ice_restart: true,
..Default::default()
})
@@ -680,7 +833,7 @@ impl SessionInner {
// Timeout after ['MAX_ICE_CONNECT_TIMEOUT']
async fn wait_pc_connection(&self) -> EngineResult<()> {
let wait_connected = async move {
while self.pc_state.load(Ordering::Acquire) != PCState::Connected as u8 {
while self.pc_state.load(Ordering::Acquire) != PeerState::Connected as u8 {
if self.closed.load(Ordering::Acquire) {
return Err(EngineError::Connection("closed".to_string()));
}
@@ -693,7 +846,7 @@ impl SessionInner {
tokio::select! {
res = wait_connected => res,
_ = sleep(MAX_ICE_CONNECT_TIMEOUT) => {
_ = sleep(ICE_CONNECT_TIMEOUT) => {
let err = EngineError::Connection("wait_pc_connection timed out".to_string());
Err(err)
}
@@ -724,7 +877,7 @@ impl SessionInner {
.await
.peer_connection()
.ice_connection_state()
!= IceConnectionState::IceConnectionChecking
!= IceConnectionState::Checking
{
let _ = self.negotiate_publisher().await;
}
@@ -749,7 +902,7 @@ impl SessionInner {
tokio::select! {
res = wait_connected => res,
_ = sleep(MAX_ICE_CONNECT_TIMEOUT) => {
_ = sleep(ICE_CONNECT_TIMEOUT) => {
let err = EngineError::Connection("could not establish publisher connection: timeout".to_string());
error!(error = ?err);
Err(err)
livekit/src/signal_client/mod.rs
+2 -2
View File
@@ -114,13 +114,13 @@ impl SignalClient {
}
}
impl From<proto::JoinResponse> for RTCConfiguration {
impl From<proto::JoinResponse> for RtcConfiguration {
fn from(join_response: proto::JoinResponse) -> Self {
Self {
ice_servers: {
let mut servers = vec![];
for ice_server in join_response.ice_servers.clone() {
servers.push(ICEServer {
servers.push(IceServer {
urls: ice_server.urls,
username: ice_server.username,
password: ice_server.credential,
livekit/src/signal_client/signal_stream.rs
+2 -2
View File
@@ -34,7 +34,7 @@ enum InternalMessage {
///
/// It is replaced by [SignalClient] at each reconnection.
#[derive(Debug)]
pub(crate) struct SignalStream {
pub(super) struct SignalStream {
internal_tx: mpsc::Sender<InternalMessage>,
read_handle: JoinHandle<()>,
write_handle: JoinHandle<()>,
@@ -46,7 +46,7 @@ impl SignalStream {
///
/// SignalStream will never try to reconnect if the connection has been
/// closed.
pub(super) async fn connect(
pub async fn connect(
url: &str,
token: &str,
options: SignalOptions,
webrtc-sys/build.rs
+1 -1
View File
@@ -7,7 +7,7 @@ use std::path;
use std::process::Command;
use tar::Archive;
const WEBRTC_TAG: &str = "m104.5112.08";
const WEBRTC_TAG: &str = "m104.5112.09";
fn download_prebuilt(
target_os: &str,
webrtc-sys/include/livekit/data_channel.h
+10 -5
View File
@@ -37,7 +37,7 @@ class DataChannel {
std::shared_ptr<RTCRuntime> rtc_runtime,
rtc::scoped_refptr<webrtc::DataChannelInterface> data_channel);
void register_observer(NativeDataChannelObserver& observer) const;
void register_observer(NativeDataChannelObserver* observer) const;
void unregister_observer() const;
bool send(const DataBuffer& buffer) const;
rust::String label() const;
@@ -52,14 +52,17 @@ class DataChannel {
std::unique_ptr<NativeDataChannelInit> create_data_channel_init(
DataChannelInit init);
static std::unique_ptr<DataChannel> _unique_data_channel() {
static std::shared_ptr<DataChannel> _shared_data_channel() {
return nullptr; // Ignore
}
class NativeDataChannelObserver : public webrtc::DataChannelObserver {
public:
explicit NativeDataChannelObserver(
rust::Box<DataChannelObserverWrapper> observer);
rust::Box<DataChannelObserverWrapper> observer,
DataChannel* dc);
~NativeDataChannelObserver();
void OnStateChange() override;
void OnMessage(const webrtc::DataBuffer& buffer) override;
@@ -67,8 +70,10 @@ class NativeDataChannelObserver : public webrtc::DataChannelObserver {
private:
rust::Box<DataChannelObserverWrapper> observer_;
DataChannel* dc_;
};
std::unique_ptr<NativeDataChannelObserver> create_native_data_channel_observer(
rust::Box<DataChannelObserverWrapper> observer);
std::shared_ptr<NativeDataChannelObserver> create_native_data_channel_observer(
rust::Box<DataChannelObserverWrapper> observer,
DataChannel* dc);
} // namespace livekit
webrtc-sys/include/livekit/jsep.h
+1
View File
@@ -65,6 +65,7 @@ class SessionDescription {
explicit SessionDescription(
std::unique_ptr<webrtc::SessionDescriptionInterface> session_description);
SdpType sdp_type() const;
rust::String stringify() const;
std::unique_ptr<SessionDescription> clone() const;
std::unique_ptr<webrtc::SessionDescriptionInterface> release();
webrtc-sys/include/livekit/media_stream.h
+22 -11
View File
@@ -26,6 +26,7 @@
#include "rtc_base/synchronization/mutex.h"
#include "rtc_base/timestamp_aligner.h"
#include "rust/cxx.h"
#include "system_wrappers/include/clock.h"
namespace livekit {
class MediaStream;
@@ -50,10 +51,8 @@ class MediaStream {
std::shared_ptr<AudioTrack> find_audio_track(rust::String track_id) const;
std::shared_ptr<VideoTrack> find_video_track(rust::String track_id) const;
bool add_audio_track(std::shared_ptr<AudioTrack> audio_track) const;
bool add_video_track(std::shared_ptr<VideoTrack> video_track) const;
bool remove_audio_track(std::shared_ptr<AudioTrack> audio_track) const;
bool remove_video_track(std::shared_ptr<VideoTrack> video_track) const;
bool add_track(std::shared_ptr<MediaStreamTrack> track) const;
bool remove_track(std::shared_ptr<MediaStreamTrack> track) const;
private:
rtc::scoped_refptr<webrtc::MediaStreamInterface> media_stream_;
@@ -121,7 +120,7 @@ class NativeVideoFrameSink
rust::Box<VideoFrameSinkWrapper> observer_;
};
std::unique_ptr<NativeVideoFrameSink> create_native_video_frame_sink(
std::unique_ptr<NativeVideoFrameSink> new_native_video_frame_sink(
rust::Box<VideoFrameSinkWrapper> observer);
// Native impl of the WebRTC interface
@@ -146,7 +145,7 @@ class AdaptedVideoTrackSource {
public:
AdaptedVideoTrackSource(rtc::scoped_refptr<NativeVideoTrackSource> source);
bool on_captured_frame(std::unique_ptr<VideoFrame> frame)
bool on_captured_frame(const std::unique_ptr<VideoFrame>& frame)
const; // frames pushed from Rust (+interior mutability)
rtc::scoped_refptr<NativeVideoTrackSource> get() const;
@@ -155,14 +154,26 @@ class AdaptedVideoTrackSource {
rtc::scoped_refptr<NativeVideoTrackSource> source_;
};
std::unique_ptr<AdaptedVideoTrackSource> create_adapted_video_track_source();
std::shared_ptr<AdaptedVideoTrackSource> new_adapted_video_track_source();
static const VideoTrack* media_to_video(const MediaStreamTrack* track) {
return static_cast<const VideoTrack*>(track);
static std::shared_ptr<MediaStreamTrack> video_to_media(
std::shared_ptr<VideoTrack> track) {
return track;
}
static const AudioTrack* media_to_audio(const MediaStreamTrack* track) {
return static_cast<const AudioTrack*>(track);
static std::shared_ptr<MediaStreamTrack> audio_to_media(
std::shared_ptr<AudioTrack> track) {
return track;
}
static std::shared_ptr<VideoTrack> media_to_video(
std::shared_ptr<MediaStreamTrack> track) {
return std::static_pointer_cast<VideoTrack>(track);
}
static std::shared_ptr<AudioTrack> media_to_audio(
std::shared_ptr<MediaStreamTrack> track) {
return std::static_pointer_cast<AudioTrack>(track);
}
static std::shared_ptr<MediaStreamTrack> _shared_media_stream_track() {
webrtc-sys/include/livekit/peer_connection.h
+16 -4
View File
@@ -57,7 +57,7 @@ class PeerConnection {
void set_remote_description(std::unique_ptr<SessionDescription> desc,
NativeSetRemoteSdpObserverHandle& observer) const;
std::unique_ptr<DataChannel> create_data_channel(
std::shared_ptr<DataChannel> create_data_channel(
rust::String label,
std::unique_ptr<NativeDataChannelInit> init) const;
@@ -84,24 +84,34 @@ class PeerConnection {
rust::Vec<RtpTransceiverPtr> get_transceivers() const;
std::unique_ptr<SessionDescription> current_local_description() const;
std::unique_ptr<SessionDescription> current_remote_description() const;
std::unique_ptr<SessionDescription> pending_local_description() const;
std::unique_ptr<SessionDescription> pending_remote_description() const;
std::unique_ptr<SessionDescription> local_description() const;
std::unique_ptr<SessionDescription> remote_description() const;
PeerConnectionState connection_state() const;
SignalingState signaling_state() const;
IceGatheringState ice_gathering_state() const;
IceConnectionState ice_connection_state() const;
void close();
void close() const;
private:
std::shared_ptr<RTCRuntime> rtc_runtime_;
rtc::scoped_refptr<webrtc::PeerConnectionInterface> peer_connection_;
};
static std::unique_ptr<PeerConnection> _unique_peer_connection() {
static std::shared_ptr<PeerConnection> _shared_peer_connection() {
return nullptr; // Ignore
}
@@ -126,6 +136,8 @@ class NativePeerConnectionObserver : public webrtc::PeerConnectionObserver {
std::shared_ptr<RTCRuntime> rtc_runtime,
rust::Box<PeerConnectionObserverWrapper> observer);
~NativePeerConnectionObserver();
void OnSignalingChange(
webrtc::PeerConnectionInterface::SignalingState new_state) override;
@@ -188,7 +200,7 @@ class NativePeerConnectionObserver : public webrtc::PeerConnectionObserver {
rust::Box<PeerConnectionObserverWrapper> observer_;
};
std::unique_ptr<NativePeerConnectionObserver>
std::shared_ptr<NativePeerConnectionObserver>
create_native_peer_connection_observer(
std::shared_ptr<RTCRuntime> rtc_runtime,
rust::Box<PeerConnectionObserverWrapper> observer);
webrtc-sys/include/livekit/peer_connection_factory.h
+14 -3
View File
@@ -17,7 +17,10 @@
#pragma once
#include "api/peer_connection_interface.h"
#include "media_stream.h"
#include "peer_connection.h"
#include "rtp_parameters.h"
#include "rust/cxx.h"
#include "webrtc.h"
namespace livekit {
@@ -35,16 +38,24 @@ class PeerConnectionFactory {
explicit PeerConnectionFactory(std::shared_ptr<RTCRuntime> rtc_runtime);
~PeerConnectionFactory();
std::unique_ptr<PeerConnection> create_peer_connection(
std::shared_ptr<PeerConnection> create_peer_connection(
std::unique_ptr<NativeRTCConfiguration> config,
NativePeerConnectionObserver& observer) const;
NativePeerConnectionObserver* observer) const;
std::shared_ptr<VideoTrack> create_video_track(
rust::String label,
std::shared_ptr<AdaptedVideoTrackSource> source) const;
RtpCapabilities get_rtp_sender_capabilities(MediaType type) const;
RtpCapabilities get_rtp_receiver_capabilities(MediaType type) const;
private:
std::shared_ptr<RTCRuntime> rtc_runtime_;
rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface> peer_factory_;
};
std::unique_ptr<PeerConnectionFactory> create_peer_connection_factory(
std::shared_ptr<PeerConnectionFactory> create_peer_connection_factory(
std::shared_ptr<RTCRuntime> rtc_runtime);
std::unique_ptr<NativeRTCConfiguration> create_rtc_configuration(
RTCConfiguration conf);
webrtc-sys/include/livekit/video_frame.h
+2 -2
View File
@@ -57,7 +57,7 @@ class VideoFrameBuilder {
VideoFrameBuilder() = default;
// TODO(theomonnom): other setters?
void set_video_frame_buffer(std::unique_ptr<VideoFrameBuffer> buffer);
void set_video_frame_buffer(const VideoFrameBuffer& buffer);
void set_timestamp_us(int64_t timestamp_us);
void set_rotation(VideoRotation rotation);
void set_id(uint16_t id);
@@ -67,6 +67,6 @@ class VideoFrameBuilder {
webrtc::VideoFrame::Builder builder_;
};
std::unique_ptr<VideoFrameBuilder> create_video_frame_builder();
std::unique_ptr<VideoFrameBuilder> new_video_frame_builder();
} // namespace livekit
webrtc-sys/include/livekit/video_frame_buffer.h
+12 -3
View File
@@ -49,8 +49,9 @@ class VideoFrameBuffer {
int width() const;
int height() const;
// Takes ownership
std::unique_ptr<I420Buffer> to_i420();
std::unique_ptr<I420Buffer> to_i420() const;
// Requires ownership
std::unique_ptr<I420Buffer> get_i420();
std::unique_ptr<I420ABuffer> get_i420a();
std::unique_ptr<I422Buffer> get_i422();
@@ -131,7 +132,9 @@ class BiplanarYuv8Buffer : public BiplanarYuvBuffer {
webrtc::BiplanarYuv8Buffer* buffer() const;
};
std::unique_ptr<I420Buffer> create_i420_buffer(int width, int height);
std::unique_ptr<I420Buffer> new_i420_buffer(int width, int height);
std::unique_ptr<I420Buffer> copy_i420_buffer(
const std::unique_ptr<I420Buffer>& i420);
class I420Buffer : public PlanarYuv8Buffer {
public:
@@ -141,6 +144,12 @@ class I420Buffer : public PlanarYuv8Buffer {
class I420ABuffer : public I420Buffer {
public:
explicit I420ABuffer(rtc::scoped_refptr<webrtc::I420ABufferInterface> buffer);
int stride_a() const;
const uint8_t* data_a() const;
private:
webrtc::I420ABufferInterface* buffer() const;
};
class I422Buffer : public PlanarYuv8Buffer {
webrtc-sys/include/livekit/webrtc.h
+3
View File
@@ -18,6 +18,7 @@
#include "rtc_base/physical_socket_server.h"
#include "rtc_base/ssl_adapter.h"
#include "rust/cxx.h"
#ifdef WEBRTC_WIN
#include "rtc_base/win32_socket_init.h"
@@ -54,6 +55,8 @@ class RTCRuntime {
#endif
};
rust::String create_random_uuid();
std::shared_ptr<RTCRuntime> create_rtc_runtime();
} // namespace livekit
webrtc-sys/include/livekit/yuv_helper.h
+59 -8
View File
@@ -17,12 +17,19 @@
#pragma once
#include <memory>
#include <stdexcept>
#include <string>
#include "api/video/yuv_helper.h"
#include "webrtc-sys/src/yuv_helper.rs.h"
namespace livekit {
#define THROW_ON_ERROR(ret) \
if (ret != 0) { \
throw std::runtime_error("libyuv error: " + std::to_string(ret)); \
}
static void i420_to_argb(const uint8_t* src_y,
int src_stride_y,
const uint8_t* src_u,
@@ -33,8 +40,9 @@ static void i420_to_argb(const uint8_t* src_y,
int dst_stride_argb,
int width,
int height) {
webrtc::I420ToARGB(src_y, src_stride_y, src_u, src_stride_u, src_v,
src_stride_v, dst_argb, dst_stride_argb, width, height);
THROW_ON_ERROR(webrtc::I420ToARGB(src_y, src_stride_y, src_u, src_stride_u,
src_v, src_stride_v, dst_argb,
dst_stride_argb, width, height));
}
static void i420_to_bgra(const uint8_t* src_y,
@@ -47,8 +55,9 @@ static void i420_to_bgra(const uint8_t* src_y,
int dst_stride_bgra,
int width,
int height) {
webrtc::I420ToBGRA(src_y, src_stride_y, src_u, src_stride_u, src_v,
src_stride_v, dst_bgra, dst_stride_bgra, width, height);
THROW_ON_ERROR(webrtc::I420ToBGRA(src_y, src_stride_y, src_u, src_stride_u,
src_v, src_stride_v, dst_bgra,
dst_stride_bgra, width, height));
}
static void i420_to_abgr(const uint8_t* src_y,
@@ -61,8 +70,9 @@ static void i420_to_abgr(const uint8_t* src_y,
int dst_stride_abgr,
int width,
int height) {
webrtc::I420ToABGR(src_y, src_stride_y, src_u, src_stride_u, src_v,
src_stride_v, dst_abgr, dst_stride_abgr, width, height);
THROW_ON_ERROR(webrtc::I420ToABGR(src_y, src_stride_y, src_u, src_stride_u,
src_v, src_stride_v, dst_abgr,
dst_stride_abgr, width, height));
}
static void i420_to_rgba(const uint8_t* src_y,
@@ -75,8 +85,49 @@ static void i420_to_rgba(const uint8_t* src_y,
int dst_stride_rgba,
int width,
int height) {
webrtc::I420ToRGBA(src_y, src_stride_y, src_u, src_stride_u, src_v,
src_stride_v, dst_rgba, dst_stride_rgba, width, height);
THROW_ON_ERROR(webrtc::I420ToRGBA(src_y, src_stride_y, src_u, src_stride_u,
src_v, src_stride_v, dst_rgba,
dst_stride_rgba, width, height));
}
static void argb_to_i420(const uint8_t* src_argb,
int src_stride_argb,
uint8_t* dst_y,
int dst_stride_y,
uint8_t* dst_u,
int dst_stride_u,
uint8_t* dst_v,
int dst_stride_v,
int width,
int height) {
THROW_ON_ERROR(webrtc::ARGBToI420(src_argb, src_stride_argb, dst_y,
dst_stride_y, dst_u, dst_stride_u, dst_v,
dst_stride_v, width, height));
}
static void abgr_to_i420(const uint8_t* src_abgr,
int src_stride_abgr,
uint8_t* dst_y,
int dst_stride_y,
uint8_t* dst_u,
int dst_stride_u,
uint8_t* dst_v,
int dst_stride_v,
int width,
int height) {
THROW_ON_ERROR(webrtc::ABGRToI420(src_abgr, src_stride_abgr, dst_y,
dst_stride_y, dst_u, dst_stride_u, dst_v,
dst_stride_v, width, height));
}
static void argb_to_rgb24(const uint8_t* src_argb,
int src_stride_argb,
uint8_t* dst_rgb24,
int dst_stride_rgb24,
int width,
int height) {
THROW_ON_ERROR(webrtc::ARGBToRGB24(src_argb, src_stride_argb, dst_rgb24,
dst_stride_rgb24, width, height));
}
} // namespace livekit

Some files were not shown because too many files have changed in this diff Show More