talksik/client-sdk-rust
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: add ffi datachannel & mute events (#88)

Browse Source
This commit is contained in:
Théo Monnom
2023-06-18 22:45:05 +02:00
committed by GitHub
parent 33dfcb27b0
commit 25f4c9a075
69 changed files with 1934 additions and 1546 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+1 -25
View File
@@ -785,15 +785,14 @@ dependencies = [
"lazy_static",
"livekit-protocol",
"livekit-webrtc",
"log",
"parking_lot",
"prost",
"serde",
"serde_json",
"thiserror",
"tokio",
"tokio-stream",
"tokio-tungstenite",
"tracing",
"url",
]
@@ -1676,17 +1675,6 @@ dependencies = [
"tokio",
]
[[package]]
name = "tokio-stream"
version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842"
dependencies = [
"futures-core",
"pin-project-lite",
"tokio",
]
[[package]]
name = "tokio-tungstenite"
version = "0.19.0"
@@ -1733,21 +1721,9 @@ checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8"
dependencies = [
"cfg-if",
"pin-project-lite",
"tracing-attributes",
"tracing-core",
]
[[package]]
name = "tracing-attributes"
version = "0.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.18",
]
[[package]]
name = "tracing-core"
version = "0.1.31"
examples/Cargo.lock
Generated
+2 -119
View File
@@ -841,7 +841,7 @@ dependencies = [
"futures-sink",
"nanorand",
"pin-project",
"spin 0.9.8",
"spin",
]
[[package]]
@@ -1481,15 +1481,14 @@ dependencies = [
"lazy_static",
"livekit-protocol",
"livekit-webrtc",
"log",
"parking_lot",
"prost",
"serde",
"serde_json",
"thiserror",
"tokio",
"tokio-stream",
"tokio-tungstenite",
"tracing",
"url",
]
@@ -2352,21 +2351,6 @@ dependencies = [
"winreg",
]
[[package]]
name = "ring"
version = "0.16.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc"
dependencies = [
"cc",
"libc",
"once_cell",
"spin 0.5.2",
"untrusted",
"web-sys",
"winapi",
]
[[package]]
name = "rustc-demangle"
version = "0.1.23"
@@ -2393,49 +2377,6 @@ dependencies = [
"windows-sys 0.48.0",
]
[[package]]
name = "rustls"
version = "0.21.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c911ba11bc8433e811ce56fde130ccf32f5127cab0e0194e9c68c5a5b671791e"
dependencies = [
"log",
"ring",
"rustls-webpki",
"sct",
]
[[package]]
name = "rustls-native-certs"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50"
dependencies = [
"openssl-probe",
"rustls-pemfile",
"schannel",
"security-framework",
]
[[package]]
name = "rustls-pemfile"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b"
dependencies = [
"base64",
]
[[package]]
name = "rustls-webpki"
version = "0.100.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b"
dependencies = [
"ring",
"untrusted",
]
[[package]]
name = "ryu"
version = "1.0.13"
@@ -2489,16 +2430,6 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1"
[[package]]
name = "sct"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4"
dependencies = [
"ring",
"untrusted",
]
[[package]]
name = "sctk-adwaita"
version = "0.5.4"
@@ -2687,12 +2618,6 @@ dependencies = [
"winapi",
]
[[package]]
name = "spin"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
[[package]]
name = "spin"
version = "0.9.8"
@@ -2928,27 +2853,6 @@ dependencies = [
"tokio",
]
[[package]]
name = "tokio-rustls"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0d409377ff5b1e3ca6437aa86c1eb7d40c134bfec254e44c830defa92669db5"
dependencies = [
"rustls",
"tokio",
]
[[package]]
name = "tokio-stream"
version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842"
dependencies = [
"futures-core",
"pin-project-lite",
"tokio",
]
[[package]]
name = "tokio-tungstenite"
version = "0.19.0"
@@ -2957,10 +2861,7 @@ checksum = "ec509ac96e9a0c43427c74f003127d953a265737636129424288d27cb5c4b12c"
dependencies = [
"futures-util",
"log",
"rustls",
"rustls-native-certs",
"tokio",
"tokio-rustls",
"tungstenite",
]
@@ -3084,12 +2985,10 @@ dependencies = [
"httparse",
"log",
"rand",
"rustls",
"sha1",
"thiserror",
"url",
"utf-8",
"webpki",
]
[[package]]
@@ -3140,12 +3039,6 @@ version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c"
[[package]]
name = "untrusted"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
[[package]]
name = "url"
version = "2.3.1"
@@ -3379,16 +3272,6 @@ dependencies = [
"web-sys",
]
[[package]]
name = "webpki"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd"
dependencies = [
"ring",
"untrusted",
]
[[package]]
name = "webrtc-sys"
version = "0.1.1"
examples/basic_room/src/main.rs
+1 -1
View File
@@ -11,7 +11,7 @@ async fn main() {
let url = env::var("LIVEKIT_URL").expect("LIVEKIT_URL is not set");
let token = env::var("LIVEKIT_TOKEN").expect("LIVEKIT_TOKEN is not set");
let (room, mut rx) = Room::connect(&url, &token).await.unwrap();
let (room, mut rx) = Room::connect(&url, &token, RoomOptions::default()).await.unwrap();
log::info!("Connected to room: {} - {}", room.name(), room.sid());
while let Some(msg) = rx.recv().await {
examples/save_to_disk/src/main.rs
+3 -1
View File
@@ -89,7 +89,9 @@ async fn main() {
let url = env::var("LIVEKIT_URL").expect("LIVEKIT_URL is not set");
let token = env::var("LIVEKIT_TOKEN").expect("LIVEKIT_TOKEN is not set");
let (room, mut rx) = Room::connect(&url, &token).await.unwrap();
let (room, mut rx) = Room::connect(&url, &token, RoomOptions::default())
.await
.unwrap();
println!("Connected to room: {} - {}", room.name(), room.sid());
while let Some(msg) = rx.recv().await {
examples/wgpu_room/Cargo.toml
+1 -1
View File
@@ -11,7 +11,7 @@ livekit = { path = "../../livekit", version = "0.1.1" }
futures = "0.3"
wgpu = "0.16"
winit = "0.28"
parking_lot = "0.12.1"
parking_lot = { version = "0.12.1", features=["send_guard"] }
image = "0.24"
egui = { git = "https://github.com/emilk/egui" }
egui-wgpu = { git = "https://github.com/emilk/egui", features = ["winit"] }
examples/wgpu_room/src/app.rs
+1 -1
View File
@@ -106,7 +106,7 @@ pub fn run(rt: tokio::runtime::Runtime) {
AsyncCmd::RoomConnect { url, token } => {
state.connecting.store(true, Ordering::SeqCst);
let res = Room::connect(&url, &token).await;
let res = Room::connect(&url, &token, RoomOptions::default()).await;
if let Ok((room, room_events)) = res {
let room = Arc::new(room);
let (close_tx, close_rx) = oneshot::channel();
examples/wgpu_room/src/logo_track.rs
+4 -4
View File
@@ -1,7 +1,8 @@
use image::ImageFormat;
use image::RgbaImage;
use livekit::options::{TrackPublishOptions, VideoCaptureOptions};
use livekit::options::TrackPublishOptions;
use livekit::prelude::*;
use livekit::webrtc::video_source::RtcVideoSource;
use livekit::webrtc::{
native::yuv_helper,
video_frame::native::I420BufferExt,
@@ -61,8 +62,7 @@ impl LogoTrack {
let (close_tx, close_rx) = oneshot::channel();
let track = LocalVideoTrack::create_video_track(
"livekit_logo",
VideoCaptureOptions::default(),
self.rtc_source.clone(),
RtcVideoSource::Native(self.rtc_source.clone()),
);
let task = tokio::spawn(Self::track_task(close_rx, self.rtc_source.clone()));
@@ -118,7 +118,7 @@ impl LogoTrack {
video_frame: Arc::new(Mutex::new(VideoFrame {
rotation: VideoRotation::VideoRotation0,
buffer: I420Buffer::new(FB_WIDTH as u32, FB_HEIGHT as u32),
timestamp: 0,
timestamp_us: 0,
})),
pos: (0, 0),
direction: (1, 1),
examples/wgpu_room/src/sine_track.rs
+3 -7
View File
@@ -1,5 +1,6 @@
use livekit::options::{AudioCaptureOptions, TrackPublishOptions};
use livekit::options::TrackPublishOptions;
use livekit::webrtc::audio_frame::AudioFrame;
use livekit::webrtc::audio_source::RtcAudioSource;
use livekit::{prelude::*, webrtc::audio_source::native::NativeAudioSource};
use std::sync::Arc;
use std::time::Duration;
@@ -54,12 +55,7 @@ impl SineTrack {
let (close_tx, close_rx) = oneshot::channel();
let track = LocalAudioTrack::create_audio_track(
"sine_wave",
AudioCaptureOptions {
auto_gain_control: false,
echo_cancellation: false,
noise_suppression: false,
},
self.rtc_source.clone(),
RtcAudioSource::Native(self.rtc_source.clone()),
);
let task = tokio::spawn(Self::track_task(close_rx, self.rtc_source.clone()));
livekit-api/src/services/egress.rs
+2 -1
View File
@@ -128,7 +128,8 @@ impl EgressClient {
file_outputs,
stream_outputs,
segment_outputs,
output: None, // Deprecated
output: None, // Deprecated
await_start_signal: false, // TODO Expose
},
self.base.auth_header(VideoGrants {
room_record: true,
livekit-api/src/services/ingress.rs
+3 -1
View File
@@ -57,6 +57,7 @@ impl IngressClient {
participant_name: options.participant_name,
audio: Some(options.audio),
video: Some(options.video),
bypass_transcoding: false, // TODO Expose
},
self.base.auth_header(VideoGrants {
ingress_admin: true,
@@ -84,6 +85,7 @@ impl IngressClient {
participant_name: options.participant_name,
audio: Some(options.audio),
video: Some(options.video),
bypass_transcoding: None, // TODO Expose
},
self.base.auth_header(VideoGrants {
ingress_admin: true,
@@ -104,7 +106,7 @@ impl IngressClient {
SVC,
"ListIngress",
proto::ListIngressRequest {
room_name: match filter{
room_name: match filter {
IngressListFilter::All => Default::default(),
IngressListFilter::Room(room) => room,
},
livekit-ffi/Cargo.toml
+2 -2
View File
@@ -18,12 +18,12 @@ __rustls-tls = ["livekit/__rustls-tls"]
livekit = { path = "../livekit", version = "0.1.1" }
livekit-protocol = { path = "../livekit-protocol", version = "0.1.0" }
tokio = { version = "1.0", features = ["full"] }
parking_lot = { version = "0.12.1", features = ["send_guard"] }
futures-util = { version = "0.3", default-features = false, features = ["sink"] }
parking_lot = { version = "0.12.1", features=["send_guard"] }
prost = "0.11.0"
prost-types = "0.11.1"
lazy_static = "1.4.0"
thiserror = "1.0.38"
futures-util = "0.3.23"
log = "0.4.17"
dashmap = "5.4.0"
env_logger = "0.10.0"
livekit-ffi/protocol/audio_frame.proto
+12 -6
View File
@@ -18,15 +18,16 @@ message AllocAudioBufferResponse { AudioFrameBufferInfo buffer = 1; }
// Create a new AudioStream
// AudioStream is used to receive audio frames from a track
message NewAudioStreamRequest {
FfiHandleId room_handle = 1;
string participant_sid = 2;
string track_sid = 3;
AudioStreamType type = 4;
FfiHandleId track_handle = 1;
AudioStreamType type = 2;
}
message NewAudioStreamResponse { AudioStreamInfo stream = 1; }
// Create a new AudioSource
message NewAudioSourceRequest { AudioSourceType type = 1; }
message NewAudioSourceRequest {
AudioSourceType type = 1;
optional AudioSourceOptions options = 2;
}
message NewAudioSourceResponse { AudioSourceInfo source = 1; }
// Push a frame to an AudioSource
@@ -78,7 +79,6 @@ enum AudioStreamType {
message AudioStreamInfo {
FfiHandleId handle = 1;
AudioStreamType type = 2;
string track_sid = 3;
}
message AudioStreamEvent {
@@ -94,6 +94,12 @@ message AudioFrameReceived {
/// AudioSource ///
///
message AudioSourceOptions {
bool echo_cancellation = 1;
bool noise_suppression = 2;
bool auto_gain_control = 3;
}
enum AudioSourceType {
AUDIO_SOURCE_NATIVE = 0;
}
livekit-ffi/protocol/ffi.proto
+34 -30
View File
@@ -22,26 +22,27 @@ message FfiRequest {
DisconnectRequest disconnect = 4;
PublishTrackRequest publish_track = 5;
UnpublishTrackRequest unpublish_track = 6;
PublishDataRequest publish_data = 7;
// Track
CreateVideoTrackRequest create_video_track = 7;
CreateAudioTrackRequest create_audio_track = 8;
CreateVideoTrackRequest create_video_track = 8;
CreateAudioTrackRequest create_audio_track = 9;
// Video
AllocVideoBufferRequest alloc_video_buffer = 9;
NewVideoStreamRequest new_video_stream = 10;
NewVideoSourceRequest new_video_source = 11;
CaptureVideoFrameRequest capture_video_frame = 12;
ToI420Request to_i420 = 13;
ToArgbRequest to_argb = 14;
AllocVideoBufferRequest alloc_video_buffer = 10;
NewVideoStreamRequest new_video_stream = 11;
NewVideoSourceRequest new_video_source = 12;
CaptureVideoFrameRequest capture_video_frame = 13;
ToI420Request to_i420 = 14;
ToArgbRequest to_argb = 15;
// Audio
AllocAudioBufferRequest alloc_audio_buffer = 15;
NewAudioStreamRequest new_audio_stream = 16;
NewAudioSourceRequest new_audio_source = 17;
CaptureAudioFrameRequest capture_audio_frame = 18;
NewAudioResamplerRequest new_audio_resampler = 19;
RemixAndResampleRequest remix_and_resample = 20;
AllocAudioBufferRequest alloc_audio_buffer = 16;
NewAudioStreamRequest new_audio_stream = 17;
NewAudioSourceRequest new_audio_source = 18;
CaptureAudioFrameRequest capture_audio_frame = 19;
NewAudioResamplerRequest new_audio_resampler = 20;
RemixAndResampleRequest remix_and_resample = 21;
}
}
@@ -56,26 +57,27 @@ message FfiResponse {
DisconnectResponse disconnect = 4;
PublishTrackResponse publish_track = 5;
UnpublishTrackResponse unpublish_track = 6;
PublishDataResponse publish_data = 7;
// Track
CreateVideoTrackResponse create_video_track = 7;
CreateAudioTrackResponse create_audio_track = 8;
CreateVideoTrackResponse create_video_track = 8;
CreateAudioTrackResponse create_audio_track = 9;
// Video
AllocVideoBufferResponse alloc_video_buffer = 9;
NewVideoStreamResponse new_video_stream = 10;
NewVideoSourceResponse new_video_source = 11;
CaptureVideoFrameResponse capture_video_frame = 12;
ToI420Response to_i420 = 13;
ToArgbResponse to_argb = 14;
AllocVideoBufferResponse alloc_video_buffer = 10;
NewVideoStreamResponse new_video_stream = 11;
NewVideoSourceResponse new_video_source = 12;
CaptureVideoFrameResponse capture_video_frame = 13;
ToI420Response to_i420 = 14;
ToArgbResponse to_argb = 15;
// Audio
AllocAudioBufferResponse alloc_audio_buffer = 15;
NewAudioStreamResponse new_audio_stream = 16;
NewAudioSourceResponse new_audio_source = 17;
CaptureAudioFrameResponse capture_audio_frame = 18;
NewAudioResamplerResponse new_audio_resampler = 19;
RemixAndResampleResponse remix_and_resample = 20;
AllocAudioBufferResponse alloc_audio_buffer = 16;
NewAudioStreamResponse new_audio_stream = 17;
NewAudioSourceResponse new_audio_source = 18;
CaptureAudioFrameResponse capture_audio_frame = 19;
NewAudioResamplerResponse new_audio_resampler = 20;
RemixAndResampleResponse remix_and_resample = 21;
}
}
@@ -87,8 +89,10 @@ message FfiEvent {
VideoStreamEvent video_stream_event = 4;
AudioStreamEvent audio_stream_event = 5;
ConnectCallback connect = 6;
DisposeCallback dispose = 7;
PublishTrackCallback publish_track = 8;
DisconnectCallback disconnect = 7;
DisposeCallback dispose = 8;
PublishTrackCallback publish_track = 9;
PublishDataCallback publish_data = 10;
}
}
livekit-ffi/protocol/handle.proto
+4 -1
View File
@@ -6,7 +6,10 @@ option csharp_namespace = "LiveKit.Proto";
/// # Safety
/// The foreign language is responsable for disposing handles
/// Forgetting to dispose the handle may lead to memory leaks
/// Messages in this file can contain an FFIHandle
///
/// A handle means that the foreign language may still use the corresponding object
/// Dropping a handle doesn't necessarily mean that the object is destroyed if it is still used
/// on the FfiServer (Rust)
message FfiHandleId {
uint64 id = 1;
}
livekit-ffi/protocol/participant.proto
+1
View File
@@ -6,6 +6,7 @@ option csharp_namespace = "LiveKit.Proto";
//import "handle.proto";
import "track.proto";
// Seems like we don't need a FfiHandle for participants (atm at least)
message ParticipantInfo {
string sid = 1;
string name = 2;
livekit-ffi/protocol/room.proto
+25 -7
View File
@@ -26,7 +26,7 @@ message ConnectCallback {
// Disconnect from the a room
message DisconnectRequest { FfiHandleId room_handle = 1; }
message DisconnectResponse { FfiAsyncId async_id = 1; }
message DisconnectCallback { }
message DisconnectCallback { FfiAsyncId async_id = 1; }
// Publish a track to the room
message PublishTrackRequest {
@@ -53,9 +53,27 @@ message UnpublishTrackResponse {
FfiAsyncId async_id = 1;
}
message UnpublishTrackCallback {
optional string error = 1;
FfiAsyncId async_id = 1;
optional string error = 2;
}
// Publish data to other participants
message PublishDataRequest {
FfiHandleId room_handle = 1;
uint64 data_ptr = 2;
uint64 data_size = 3;
DataPacketKind kind = 4;
repeated string destination_sids = 5; // destination
}
message PublishDataResponse {
FfiAsyncId async_id = 1;
}
message PublishDataCallback {
FfiAsyncId async_id = 1;
optional string error = 2;
}
///
/// Options
///
@@ -77,13 +95,13 @@ message TrackPublishOptions {
bool dtx = 4;
bool red = 5;
bool simulcast = 6;
string name = 7;
TrackSource source = 8;
TrackSource source = 7;
}
message RoomOptions {
bool auto_subscribe = 1;
bool adaptive_stream = 2;
bool dynacast = 3;
}
///
@@ -104,7 +122,7 @@ enum ConnectionState {
}
enum DataPacketKind {
KIND_UNRELIABLE = 0;
KIND_LOSSY = 0;
KIND_RELIABLE = 1;
}
@@ -141,7 +159,7 @@ message RoomInfo {
message DataReceived {
FfiHandleId handle = 1;
optional string participant_sid = 2;
optional string participant_sid = 2; // Can be empty if the data is sent a server SDK
uint64 data_ptr = 3;
uint64 data_size = 4;
DataPacketKind kind = 5;
@@ -196,4 +214,4 @@ message ConnectionStateChanged { ConnectionState state = 1; }
message Connected {}
message Disconnected {}
message Reconnecting {}
message Reconnected {}
message Reconnected {}
livekit-ffi/protocol/track.proto
+3 -16
View File
@@ -7,21 +7,10 @@ import "handle.proto";
import "video_frame.proto";
import "audio_frame.proto";
message VideoCaptureOptions {
VideoResolution resolution = 1;
}
message AudioCaptureOptions {
bool echo_cancellation = 1;
bool noise_suppression = 2;
bool auto_gain_control = 3;
}
// Create a new VideoTrack from a VideoSource
message CreateVideoTrackRequest {
string name = 1;
VideoCaptureOptions options = 2;
FfiHandleId source_handle = 3;
FfiHandleId source_handle = 2;
}
message CreateVideoTrackResponse {
TrackInfo track = 1;
@@ -30,8 +19,7 @@ message CreateVideoTrackResponse {
// Create a new AudioTrack from a AudioSource
message CreateAudioTrackRequest {
string name = 1;
AudioCaptureOptions options = 2;
FfiHandleId source_handle = 3;
FfiHandleId source_handle = 2;
}
message CreateAudioTrackResponse {
TrackInfo track = 1;
@@ -79,8 +67,7 @@ message TrackPublicationInfo {
}
message TrackInfo {
// Tracks created/owned by the client will have a handle
FfiHandleId opt_handle = 1;
FfiHandleId handle = 1;
string sid = 2;
string name = 3;
TrackKind kind = 4;
livekit-ffi/protocol/video_frame.proto
+14 -7
View File
@@ -16,16 +16,19 @@ message AllocVideoBufferResponse { VideoFrameBufferInfo buffer = 1; }
// Create a new VideoStream
// VideoStream is used to receive video frames from a track
message NewVideoStreamRequest {
FfiHandleId room_handle = 1;
string participant_sid = 2;
string track_sid = 3;
VideoStreamType type = 4;
FfiHandleId track_handle = 1;
VideoStreamType type = 2;
}
message NewVideoStreamResponse { VideoStreamInfo stream = 1; }
// Create a new VideoSource
// VideoSource is used to send video frame to a track
message NewVideoSourceRequest { VideoSourceType type = 1; }
message NewVideoSourceRequest {
VideoSourceType type = 1;
// Used to determine which encodings to use + simulcast layers
// Most of the time it corresponds to the source resolution
optional VideoSourceResolution resolution = 2;
}
message NewVideoSourceResponse { VideoSourceInfo source = 1; }
// Push a frame to a VideoSource
@@ -110,7 +113,7 @@ message ARGBBufferInfo {
}
message VideoFrameInfo {
int64 timestamp = 1;
int64 timestamp_us = 1; // In microseconds
VideoRotation rotation = 2;
}
@@ -168,7 +171,6 @@ enum VideoStreamType {
message VideoStreamInfo {
FfiHandleId handle = 1;
VideoStreamType type = 2;
string track_sid = 3;
}
message VideoStreamEvent {
@@ -185,6 +187,11 @@ message VideoFrameReceived {
/// VideoSource ///
///
message VideoSourceResolution {
uint32 width = 1;
uint32 height = 2;
}
enum VideoSourceType {
VIDEO_SOURCE_NATIVE = 0;
}
livekit-ffi/src/conversion/audio_frame.rs
+11 -1
View File
@@ -1,7 +1,18 @@
use crate::server::audio_frame::{FfiAudioSource, FfiAudioSream};
use crate::{proto, FfiHandleId};
use livekit::webrtc::audio_source::AudioSourceOptions;
use livekit::webrtc::prelude::*;
impl From<proto::AudioSourceOptions> for AudioSourceOptions {
fn from(opts: proto::AudioSourceOptions) -> Self {
Self {
echo_cancellation: opts.echo_cancellation,
auto_gain_control: opts.auto_gain_control,
noise_suppression: opts.noise_suppression,
}
}
}
impl proto::AudioFrameBufferInfo {
pub fn from(handle_id: FfiHandleId, buffer: &AudioFrame) -> Self {
Self {
@@ -20,7 +31,6 @@ impl From<&FfiAudioSream> for proto::AudioStreamInfo {
handle: Some(proto::FfiHandleId {
id: stream.handle_id() as u64,
}),
track_sid: stream.track_sid().clone().into(),
r#type: stream.stream_type() as i32,
}
}
livekit-ffi/src/conversion/room.rs
+16 -61
View File
@@ -1,67 +1,23 @@
use crate::{proto, FfiHandleId, INVALID_HANDLE};
use crate::{proto, FfiHandleId};
use livekit::options::{AudioEncoding, TrackPublishOptions, VideoEncoding};
use livekit::prelude::*;
impl proto::RoomEvent {
pub fn from(room_handle: FfiHandleId, event: RoomEvent) -> Option<Self> {
let message = match event {
RoomEvent::ParticipantConnected(participant) => Some(
proto::room_event::Message::ParticipantConnected(proto::ParticipantConnected {
info: Some((&participant).into()),
}),
),
RoomEvent::ParticipantDisconnected(participant) => {
Some(proto::room_event::Message::ParticipantDisconnected(
proto::ParticipantDisconnected {
info: Some((&participant).into()),
},
))
}
RoomEvent::TrackPublished {
publication,
participant,
} => Some(proto::room_event::Message::TrackPublished(
proto::TrackPublished {
participant_sid: participant.sid().to_string(),
publication: Some((&publication).into()),
},
)),
RoomEvent::TrackUnpublished {
publication,
participant,
} => Some(proto::room_event::Message::TrackUnpublished(
proto::TrackUnpublished {
participant_sid: participant.sid().to_string(),
publication_sid: publication.sid().into(),
},
)),
RoomEvent::TrackSubscribed {
track,
publication: _,
participant,
} => Some(proto::room_event::Message::TrackSubscribed(
proto::TrackSubscribed {
participant_sid: participant.sid().to_string(),
track: Some(proto::TrackInfo::from_remote_track(INVALID_HANDLE, &track)),
},
)),
RoomEvent::TrackUnsubscribed {
track,
publication: _,
participant,
} => Some(proto::room_event::Message::TrackUnsubscribed(
proto::TrackUnsubscribed {
participant_sid: participant.sid().to_string(),
track_sid: track.sid().to_string(),
},
)),
_ => None,
};
impl From<proto::RoomOptions> for RoomOptions {
fn from(value: proto::RoomOptions) -> Self {
Self {
adaptive_stream: value.adaptive_stream,
auto_subscribe: value.auto_subscribe,
dynacast: value.dynacast,
}
}
}
message.map(|message| proto::RoomEvent {
room_handle: Some(room_handle.into()),
message: Some(message),
})
impl From<proto::DataPacketKind> for DataPacketKind {
fn from(value: proto::DataPacketKind) -> Self {
match value {
proto::DataPacketKind::KindReliable => Self::Reliable,
proto::DataPacketKind::KindLossy => Self::Lossy,
}
}
}
@@ -93,7 +49,6 @@ impl From<proto::TrackPublishOptions> for TrackPublishOptions {
dtx: opts.dtx,
red: opts.red,
simulcast: opts.simulcast,
name: opts.name,
source: proto::TrackSource::from_i32(opts.source).unwrap().into(),
}
}
livekit-ffi/src/conversion/track.rs
+1 -20
View File
@@ -1,25 +1,6 @@
use crate::{proto, FfiHandleId};
use livekit::options::{AudioCaptureOptions, VideoCaptureOptions};
use livekit::prelude::*;
impl From<proto::VideoCaptureOptions> for VideoCaptureOptions {
fn from(opts: proto::VideoCaptureOptions) -> Self {
Self {
resolution: opts.resolution.unwrap_or_default().into(),
}
}
}
impl From<proto::AudioCaptureOptions> for AudioCaptureOptions {
fn from(opts: proto::AudioCaptureOptions) -> Self {
Self {
echo_cancellation: opts.echo_cancellation,
auto_gain_control: opts.auto_gain_control,
noise_suppression: opts.noise_suppression,
}
}
}
impl From<TrackSource> for proto::TrackSource {
fn from(source: TrackSource) -> proto::TrackSource {
match source {
@@ -63,7 +44,7 @@ macro_rules! impl_track_into {
#[allow(dead_code)]
pub fn $fnc(handle_id: FfiHandleId, track: $t) -> Self {
Self {
opt_handle: Some(handle_id.into()),
handle: Some(handle_id.into()),
name: track.name(),
stream_state: proto::StreamState::from(track.stream_state()).into(),
sid: track.sid().to_string(),
livekit-ffi/src/conversion/video_frame.rs
+11 -2
View File
@@ -4,6 +4,16 @@ use crate::FfiHandleId;
use livekit::options::{VideoCodec, VideoResolution};
use livekit::webrtc::prelude::*;
use livekit::webrtc::video_frame;
use livekit::webrtc::video_source::VideoResolution as VideoSourceResolution;
impl From<proto::VideoSourceResolution> for VideoSourceResolution {
fn from(res: proto::VideoSourceResolution) -> Self {
Self {
width: res.width,
height: res.height,
}
}
}
macro_rules! impl_yuv_into {
(@fields, $buffer:ident, $data_y:ident, $data_u:ident, $data_v: ident) => {
@@ -72,7 +82,7 @@ impl proto::VideoFrameInfo {
T: AsRef<dyn VideoFrameBuffer>,
{
Self {
timestamp: frame.timestamp,
timestamp_us: frame.timestamp_us,
rotation: proto::VideoRotation::from(frame.rotation).into(),
}
}
@@ -236,7 +246,6 @@ impl From<&FfiVideoStream> for proto::VideoStreamInfo {
handle: Some(proto::FfiHandleId {
id: stream.handle_id() as u64,
}),
track_sid: stream.track_sid().clone().into(),
r#type: stream.stream_type() as i32,
}
}
livekit-ffi/src/server/audio_frame.rs
+32 -34
View File
@@ -2,11 +2,9 @@ use crate::{proto, server, FfiError, FfiHandleId, FfiResult};
use futures_util::StreamExt;
use livekit::prelude::*;
use livekit::webrtc::audio_frame::AudioFrame;
use livekit::webrtc::audio_source::native::NativeAudioSource;
use livekit::webrtc::audio_stream::native::NativeAudioStream;
use livekit::webrtc::prelude::*;
use log::warn;
use server::utils;
use tokio::sync::oneshot;
// ===== FFIAudioStream =====
@@ -14,7 +12,6 @@ use tokio::sync::oneshot;
pub struct FfiAudioSream {
handle_id: FfiHandleId,
stream_type: proto::AudioStreamType,
track_sid: TrackSid,
#[allow(dead_code)]
close_tx: oneshot::Sender<()>, // Close the stream on drop
@@ -35,42 +32,45 @@ impl FfiAudioSream {
) -> FfiResult<proto::AudioStreamInfo> {
let (close_tx, close_rx) = oneshot::channel();
let stream_type = proto::AudioStreamType::from_i32(new_stream.r#type).unwrap();
let track_sid: TrackSid = new_stream.track_sid.into();
let room_handle = new_stream
.room_handle
.ok_or(FfiError::InvalidRequest("room_handle is empty"))?
let handle_id = new_stream
.track_handle
.ok_or(FfiError::InvalidRequest("track_handle is empty"))?
.id as FfiHandleId;
let track = utils::find_remote_track(
server,
&track_sid,
&new_stream.participant_sid.into(),
room_handle,
)?
.rtc_track();
let track = server
.ffi_handles()
.get(&handle_id)
.ok_or(FfiError::InvalidRequest("track not found"))?;
let MediaStreamTrack::Audio(track) = track else {
let track = track
.downcast_ref::<Track>()
.ok_or(FfiError::InvalidRequest("handle is not a Track"))?;
let rtc_track = track.rtc_track();
let MediaStreamTrack::Audio(rtc_track) = rtc_track else {
return Err(FfiError::InvalidRequest("not an audio track"));
};
let audio_stream = match stream_type {
#[cfg(not(target_arch = "wasm32"))]
proto::AudioStreamType::AudioStreamNative => {
let audio_stream = Self {
handle_id: server.next_id(),
stream_type,
close_tx,
track_sid,
};
let native_stream = NativeAudioStream::new(rtc_track);
server.async_runtime.spawn(Self::native_audio_stream_task(
server,
audio_stream.handle_id,
NativeAudioStream::new(track),
native_stream,
close_rx,
));
Ok::<FfiAudioSream, FfiError>(audio_stream)
}
// TODO(theomonnom): Support other stream types
_ => return Err(FfiError::InvalidRequest("unsupported audio stream type")),
}?;
@@ -91,10 +91,6 @@ impl FfiAudioSream {
self.stream_type
}
pub fn track_sid(&self) -> &TrackSid {
&self.track_sid
}
async fn native_audio_stream_task(
server: &'static server::FfiServer,
stream_handle_id: FfiHandleId,
@@ -139,12 +135,7 @@ impl FfiAudioSream {
pub struct FfiAudioSource {
handle_id: FfiHandleId,
source_type: proto::AudioSourceType,
source: AudioSource,
}
#[derive(Clone)]
pub enum AudioSource {
Native(NativeAudioSource),
source: RtcAudioSource,
}
impl FfiAudioSource {
@@ -153,12 +144,17 @@ impl FfiAudioSource {
new_source: proto::NewAudioSourceRequest,
) -> FfiResult<proto::AudioSourceInfo> {
let source_type = proto::AudioSourceType::from_i32(new_source.r#type).unwrap();
#[allow(unreachable_patterns)]
let source_inner = match source_type {
#[cfg(not(target_arch = "wasm32"))]
proto::AudioSourceType::AudioSourceNative => {
let audio_source = NativeAudioSource::default();
Ok::<AudioSource, FfiError>(AudioSource::Native(audio_source))
} //_ => return Err(FfiError::InvalidRequest("unsupported audio source type")),
}?;
use livekit::webrtc::audio_source::native::NativeAudioSource;
let audio_source =
NativeAudioSource::new(new_source.options.map(Into::into).unwrap_or_default());
RtcAudioSource::Native(audio_source)
}
_ => return Err(FfiError::InvalidRequest("unsupported audio source type")),
};
let audio_source = Self {
handle_id: server.next_id(),
@@ -180,7 +176,8 @@ impl FfiAudioSource {
capture: proto::CaptureAudioFrameRequest,
) -> FfiResult<()> {
match self.source {
AudioSource::Native(ref source) => {
#[cfg(not(target_arch = "wasm32"))]
RtcAudioSource::Native(ref source) => {
let buffer_handle = capture
.buffer_handle
.ok_or(FfiError::InvalidRequest("buffer_handle is empty"))?
@@ -197,6 +194,7 @@ impl FfiAudioSource {
source.capture_frame(frame);
}
_ => {}
}
Ok(())
@@ -210,7 +208,7 @@ impl FfiAudioSource {
self.source_type
}
pub fn inner_source(&self) -> &AudioSource {
pub fn inner_source(&self) -> &RtcAudioSource {
&self.source
}
}
livekit-ffi/src/server/mod.rs
+70 -30
View File
@@ -15,7 +15,6 @@ use std::sync::Arc;
pub mod audio_frame;
pub mod room;
pub mod utils;
pub mod video_frame;
#[cfg(test)]
@@ -31,7 +30,7 @@ pub struct FfiConfig {
pub struct FfiServer {
rooms: Mutex<HashMap<RoomSid, FfiHandleId>>,
/// Store all FFI handles inside an HashMap, if this isn't efficient enough
/// Store all Ffi handles inside an HashMap, if this isn't efficient enough
/// We can still use Box::into_raw & Box::from_raw in the future (but keep it safe for now)
ffi_handles: DashMap<FfiHandleId, FfiHandle>,
next_id: AtomicUsize,
@@ -116,11 +115,9 @@ impl FfiServer {
}
// # SAFETY: The foreign language is responsible for ensuring that the callback function is valid
unsafe {
*self.config.lock() = Some(FfiConfig {
callback_fn: std::mem::transmute(init.event_callback_ptr),
});
}
*self.config.lock() = Some(FfiConfig {
callback_fn: unsafe { std::mem::transmute(init.event_callback_ptr) },
});
Ok(proto::InitializeResponse::default())
}
@@ -191,9 +188,37 @@ impl FfiServer {
fn on_disconnect(
&'static self,
_disconnect: proto::DisconnectRequest,
disconnect: proto::DisconnectRequest,
) -> FfiResult<proto::DisconnectResponse> {
Ok(proto::DisconnectResponse::default())
let async_id = self.next_id() as FfiAsyncId;
let room_handle = disconnect
.room_handle
.as_ref()
.ok_or(FfiError::InvalidRequest("room_handle is empty"))?
.id as FfiHandleId;
let ffi_room = self
.ffi_handles
.remove(&room_handle)
.ok_or(FfiError::InvalidRequest("room not found"))?
.1;
let ffi_room = ffi_room
.downcast::<room::FfiRoom>()
.map_err(|_| FfiError::InvalidRequest("room is not a FfiRoom"))?;
self.async_runtime.spawn(async move {
ffi_room.close().await;
let _ = self.send_event(proto::ffi_event::Message::Disconnect(
proto::DisconnectCallback {
async_id: Some(async_id.into()),
},
));
});
Ok(proto::DisconnectResponse {
async_id: Some(async_id.into()),
})
}
fn on_publish_track(
@@ -230,14 +255,17 @@ impl FfiServer {
.ok_or(FfiError::InvalidRequest("track not found"))?;
let track = track
.downcast_ref::<LocalTrack>()
.ok_or(FfiError::InvalidRequest("track is not a LocalTrack"))?;
.downcast_ref::<Track>()
.ok_or(FfiError::InvalidRequest("track is not a Track"))?;
let local_track = LocalTrack::try_from(track.clone())
.map_err(|_| FfiError::InvalidRequest("track is not a LocalTrack"))?;
let publication = ffi_room
.room()
.local_participant()
.publish_track(
track.clone(),
local_track,
publish.options.map(Into::into).unwrap_or_default(),
)
.await?;
@@ -269,6 +297,29 @@ impl FfiServer {
Ok(proto::UnpublishTrackResponse::default())
}
fn on_publish_data(
&'static self,
publish: proto::PublishDataRequest,
) -> FfiResult<proto::PublishDataResponse> {
let room_handle = publish
.room_handle
.as_ref()
.ok_or(FfiError::InvalidRequest("room_handle is empty"))?
.id as FfiHandleId;
let ffi_room = self
.ffi_handles
.get(&room_handle)
.ok_or(FfiError::InvalidRequest("room not found"))?;
let ffi_room = ffi_room
.downcast_ref::<room::FfiRoom>()
.ok_or(FfiError::InvalidRequest("room is not a FfiRoom"))?;
// Push the data to an async queue (avoid blocking and keep the order)
ffi_room.publish_data(self, publish)
}
// Track
fn on_create_video_track(
&'static self,
@@ -276,7 +327,6 @@ impl FfiServer {
) -> FfiResult<proto::CreateVideoTrackResponse> {
let handle_id = create
.source_handle
.as_ref()
.ok_or(FfiError::InvalidRequest("source_handle is empty"))?
.id as FfiHandleId;
@@ -290,19 +340,13 @@ impl FfiServer {
.ok_or(FfiError::InvalidRequest("handle is not a video source"))?;
let source = source.inner_source().clone();
let video_track = match source {
video_frame::VideoSource::Native(native_source) => LocalVideoTrack::create_video_track(
&create.name,
create.options.unwrap_or_default().into(),
native_source,
),
};
let video_track = LocalVideoTrack::create_video_track(&create.name, source);
let handle_id = self.next_id() as FfiHandleId;
let track_info = proto::TrackInfo::from_local_video_track(handle_id, &video_track);
self.ffi_handles
.insert(handle_id, Box::new(LocalTrack::Video(video_track)));
.insert(handle_id, Box::new(Track::LocalVideo(video_track)));
Ok(proto::CreateVideoTrackResponse {
track: Some(track_info),
@@ -315,7 +359,6 @@ impl FfiServer {
) -> FfiResult<proto::CreateAudioTrackResponse> {
let handle_id = create
.source_handle
.as_ref()
.ok_or(FfiError::InvalidRequest("source_handle is empty"))?
.id as FfiHandleId;
@@ -329,19 +372,13 @@ impl FfiServer {
.ok_or(FfiError::InvalidRequest("handle is not an audio source"))?;
let source = source.inner_source().clone();
let audio_track = match source {
audio_frame::AudioSource::Native(native_source) => LocalAudioTrack::create_audio_track(
&create.name,
create.options.unwrap_or_default().into(),
native_source,
),
};
let audio_track = LocalAudioTrack::create_audio_track(&create.name, source);
let handle_id = self.next_id() as FfiHandleId;
let track_info = proto::TrackInfo::from_local_audio_track(handle_id, &audio_track);
self.ffi_handles
.insert(handle_id, Box::new(LocalTrack::Audio(audio_track)));
.insert(handle_id, Box::new(Track::LocalAudio(audio_track)));
Ok(proto::CreateAudioTrackResponse {
track: Some(track_info),
@@ -714,6 +751,9 @@ impl FfiServer {
proto::ffi_request::Message::UnpublishTrack(unpublish) => {
proto::ffi_response::Message::UnpublishTrack(self.on_unpublish_track(unpublish)?)
}
proto::ffi_request::Message::PublishData(publish) => {
proto::ffi_response::Message::PublishData(self.on_publish_data(publish)?)
}
proto::ffi_request::Message::CreateVideoTrack(create) => {
proto::ffi_response::Message::CreateVideoTrack(self.on_create_video_track(create)?)
}
livekit-ffi/src/server/room.rs
+165 -22
View File
@@ -1,14 +1,24 @@
use crate::server::FfiServer;
use crate::{proto, FfiHandleId, FfiResult};
use crate::{proto, FfiAsyncId, FfiError, FfiHandleId, FfiResult};
use livekit::prelude::*;
use std::slice;
use std::sync::Arc;
use tokio::sync::{mpsc, oneshot};
use tokio::sync::{broadcast, mpsc};
use tokio::task::JoinHandle;
struct DataPacket {
data: Vec<u8>,
kind: DataPacketKind,
destination_sids: Vec<String>,
async_id: FfiAsyncId,
}
pub struct FfiRoom {
room: Arc<Room>,
handle: JoinHandle<()>,
close_tx: oneshot::Sender<()>,
event_handle: JoinHandle<()>,
data_handle: JoinHandle<()>,
close_tx: broadcast::Sender<()>,
data_tx: mpsc::UnboundedSender<DataPacket>,
}
impl FfiRoom {
@@ -16,33 +26,77 @@ impl FfiRoom {
server: &'static FfiServer,
connect: proto::ConnectRequest,
) -> FfiResult<proto::RoomInfo> {
let (room, events) = Room::connect(&connect.url, &connect.token).await?;
let (room, events) = Room::connect(
&connect.url,
&connect.token,
connect.options.map(Into::into).unwrap_or_default(),
)
.await?;
let room = Arc::new(room);
let (close_tx, close_rx) = oneshot::channel();
let next_id = server.next_id() as FfiHandleId;
let (close_tx, close_rx) = broadcast::channel(1);
let (data_tx, data_rx) = mpsc::unbounded_channel();
let handle =
let next_id = server.next_id() as FfiHandleId;
let event_handle = server.async_runtime.spawn(room_task(
server,
room.clone(),
next_id,
events,
close_rx.resubscribe(),
));
let data_handle =
server
.async_runtime
.spawn(room_task(server, room.clone(), next_id, events, close_rx));
let room_info = proto::RoomInfo::from_room(next_id, &room);
.spawn(data_task(server, room.clone(), data_rx, close_rx));
let ffi_room = Self {
room: room.clone(),
handle,
event_handle,
data_handle,
close_tx,
data_tx,
};
server.ffi_handles().insert(next_id, Box::new(ffi_room));
server.rooms().lock().insert(room.sid(), next_id);
let room_info = proto::RoomInfo::from_room(next_id, &room);
Ok(room_info)
}
pub fn publish_data(
&self,
server: &'static FfiServer,
publish: proto::PublishDataRequest,
) -> FfiResult<proto::PublishDataResponse> {
let data = unsafe {
slice::from_raw_parts(publish.data_ptr as *const u8, publish.data_size as usize)
};
let kind = proto::DataPacketKind::from_i32(publish.kind).unwrap();
let destination_sids: Vec<String> = publish.destination_sids;
let async_id = server.next_id() as FfiAsyncId;
let packet = DataPacket {
data: data.to_vec(), // Avoid copy?
kind: kind.into(),
destination_sids,
async_id,
};
self.data_tx
.send(packet)
.map_err(|_| FfiError::InvalidRequest("failed to send data packet"))?;
Ok(proto::PublishDataResponse {
async_id: Some(async_id.into()),
})
}
pub async fn close(self) {
let _ = self.room.close().await;
let _ = self.close_tx.send(());
let _ = self.handle.await;
let _ = self.event_handle.await;
let _ = self.data_handle.await;
}
pub fn room(&self) -> &Arc<Room> {
@@ -50,12 +104,41 @@ impl FfiRoom {
}
}
async fn data_task(
server: &'static FfiServer,
room: Arc<Room>,
mut data_rx: mpsc::UnboundedReceiver<DataPacket>,
mut close_rx: broadcast::Receiver<()>,
) {
loop {
tokio::select! {
Some(event) = data_rx.recv() => {
let res = room.local_participant().publish_data(
event.data,
event.kind,
event.destination_sids,
).await;
let cb = proto::PublishDataCallback {
async_id: Some(event.async_id.into()),
error: res.err().map(|e| e.to_string()),
};
let _ = server.send_event(proto::ffi_event::Message::PublishData(cb));
},
_ = close_rx.recv() => {
break;
}
}
}
}
async fn room_task(
server: &'static FfiServer,
room: Arc<Room>,
room_handle: FfiHandleId,
mut events: mpsc::UnboundedReceiver<livekit::RoomEvent>,
mut close_rx: oneshot::Receiver<()>,
mut close_rx: broadcast::Receiver<()>,
) {
server
.async_runtime
@@ -66,18 +149,78 @@ async fn room_task(
loop {
tokio::select! {
Some(event) = events.recv() => {
if let Some(event) = proto::RoomEvent::from(room_handle, event.clone()) {
let _ = server.send_event(proto::ffi_event::Message::RoomEvent(event));
let message = match event {
RoomEvent::ParticipantConnected(participant) => {
server.async_runtime.spawn(participant_task(Participant::Remote(participant.clone())));
Some(proto::room_event::Message::ParticipantConnected(
proto::ParticipantConnected {
info: Some(proto::ParticipantInfo::from(&participant)),
}
))
},
RoomEvent::ParticipantDisconnected(participant) => {
Some(proto::room_event::Message::ParticipantDisconnected(
proto::ParticipantDisconnected {
info: Some(proto::ParticipantInfo::from(&participant)),
},
))
}
RoomEvent::TrackPublished {
publication,
participant,
} => Some(proto::room_event::Message::TrackPublished(
proto::TrackPublished {
participant_sid: participant.sid().to_string(),
publication: Some(proto::TrackPublicationInfo::from(&publication))
},
)),
RoomEvent::TrackUnpublished {
publication,
participant,
} => Some(proto::room_event::Message::TrackUnpublished(
proto::TrackUnpublished {
participant_sid: participant.sid().to_string(),
publication_sid: publication.sid().into(),
},
)),
RoomEvent::TrackSubscribed {
track,
publication: _,
participant,
} => {
let handle_id = server.next_id() as FfiHandleId;
let track_info = proto::TrackInfo::from_remote_track(handle_id, &track);
server.ffi_handles().insert(handle_id, Box::new(Track::from(track)));
Some(proto::room_event::Message::TrackSubscribed(
proto::TrackSubscribed {
participant_sid: participant.sid().to_string(),
track: Some(track_info),
},
))
},
RoomEvent::TrackUnsubscribed {
track,
publication: _,
participant,
} => Some(proto::room_event::Message::TrackUnsubscribed(
proto::TrackUnsubscribed {
participant_sid: participant.sid().to_string(),
track_sid: track.sid().to_string(),
},
)),
_ => None
};
if message.is_some() {
let _ = server.send_event(proto::ffi_event::Message::RoomEvent(proto::RoomEvent{
room_handle: Some(room_handle.into()),
message
}));
}
match event {
RoomEvent::ParticipantConnected(p) => {
server.async_runtime.spawn(participant_task(Participant::Remote(p)));
}
_ => {}
}
},
_ = &mut close_rx => {
_ = close_rx.recv() => {
break;
}
};
livekit-ffi/src/server/tests.rs
+10 -14
View File
@@ -212,18 +212,22 @@ fn publish_video_track() {
client::FfiHandle(connect.room.unwrap().handle.unwrap().id as FfiHandleId);
// Create a new VideoSource
const VIDEO_WIDTH: u32 = 640;
const VIDEO_HEIGHT: u32 = 480;
const VIDEO_FPS: f64 = 8.0;
let res = client.send_request(proto::FfiRequest {
message: Some(proto::ffi_request::Message::NewVideoSource(
proto::NewVideoSourceRequest {
r#type: proto::VideoSourceType::VideoSourceNative as i32,
resolution: Some(proto::VideoSourceResolution {
width: VIDEO_WIDTH,
height: VIDEO_HEIGHT,
}),
},
)),
});
const VIDEO_WIDTH: u32 = 640;
const VIDEO_HEIGHT: u32 = 480;
const VIDEO_FPS: f64 = 8.0;
let proto::ffi_response::Message::NewVideoSource(new_video_source) =
res.message.unwrap() else {
panic!("unexpected response");
@@ -241,13 +245,6 @@ fn publish_video_track() {
source_handle: Some(proto::FfiHandleId {
id: source_handle.0 as u64,
}),
options: Some(proto::VideoCaptureOptions {
resolution: Some(proto::VideoResolution {
width: VIDEO_WIDTH,
height: VIDEO_HEIGHT,
frame_rate: VIDEO_FPS,
}),
}),
},
)),
});
@@ -258,11 +255,10 @@ fn publish_video_track() {
};
let track_handle = client::FfiHandle(
create_video_track.track.unwrap().opt_handle.unwrap().id as FfiHandleId,
create_video_track.track.unwrap().handle.unwrap().id as FfiHandleId,
);
let publish_options = proto::TrackPublishOptions {
name: "video_test".to_string(),
video_codec: proto::VideoCodec::H264 as i32,
source: proto::TrackSource::SourceCamera as i32,
..Default::default()
@@ -320,7 +316,7 @@ fn publish_video_track() {
id: buffer_handle.0 as u64,
}),
frame: Some(proto::VideoFrameInfo {
timestamp: 0, // TODO
timestamp_us: 0,
rotation: proto::VideoRotation::VideoRotation0 as i32,
}),
},
livekit-ffi/src/server/utils.rs
-32
View File
@@ -1,32 +0,0 @@
use crate::{server, FfiError, FfiHandleId, FfiResult};
use livekit::prelude::*;
pub fn find_remote_track(
server: &'static server::FfiServer,
track_sid: &TrackSid,
participant_sid: &ParticipantSid,
room_handle: FfiHandleId,
) -> FfiResult<RemoteTrack> {
let ffi_room = server
.ffi_handles()
.get(&room_handle)
.ok_or(FfiError::InvalidRequest("room not found"))?;
let ffi_room = ffi_room
.downcast_ref::<server::room::FfiRoom>()
.ok_or(FfiError::InvalidRequest("room is not ffi room"))?;
let room = ffi_room.room();
let participants = room.participants();
let participant = participants
.get(participant_sid)
.ok_or(FfiError::InvalidRequest("participant not found"))?;
let track = participant
.get_track_publication(track_sid)
.ok_or(FfiError::InvalidRequest("publication not found"))?
.track()
.ok_or(FfiError::InvalidRequest("track not found/subscribed"))?;
Ok(track)
}
livekit-ffi/src/server/video_frame.rs
+30 -33
View File
@@ -3,10 +3,8 @@ use futures_util::StreamExt;
use livekit::prelude::*;
use livekit::webrtc::prelude::*;
use livekit::webrtc::video_frame::{BoxVideoFrameBuffer, VideoFrame};
use livekit::webrtc::video_source::native::NativeVideoSource;
use livekit::webrtc::video_stream::native::NativeVideoStream;
use log::warn;
use server::utils;
use tokio::sync::oneshot;
// ===== FFIVideoStream =====
@@ -14,7 +12,6 @@ use tokio::sync::oneshot;
pub struct FfiVideoStream {
handle_id: FfiHandleId,
stream_type: proto::VideoStreamType,
track_sid: TrackSid,
#[allow(dead_code)]
close_tx: oneshot::Sender<()>, // Close the stream on drop
@@ -35,42 +32,43 @@ impl FfiVideoStream {
) -> FfiResult<proto::VideoStreamInfo> {
let (close_tx, close_rx) = oneshot::channel();
let stream_type = proto::VideoStreamType::from_i32(new_stream.r#type).unwrap();
let track_sid: TrackSid = new_stream.track_sid.into();
let room_handle = new_stream
.room_handle
.ok_or(FfiError::InvalidRequest("room_handle is empty"))?
let handle_id = new_stream
.track_handle
.ok_or(FfiError::InvalidRequest("track_handle is empty"))?
.id as FfiHandleId;
let track = utils::find_remote_track(
server,
&track_sid,
&new_stream.participant_sid.into(),
room_handle,
)?
.rtc_track();
let track = server
.ffi_handles()
.get(&handle_id)
.ok_or(FfiError::InvalidRequest("track not found"))?;
let MediaStreamTrack::Video(track) = track else {
let track = track
.downcast_ref::<Track>()
.ok_or(FfiError::InvalidRequest("handle is not a Track"))?;
let rtc_track = track.rtc_track();
let MediaStreamTrack::Video(rtc_track) = rtc_track else {
return Err(FfiError::InvalidRequest("not a video track"));
};
let stream = match stream_type {
#[cfg(not(target_arch = "wasm32"))]
proto::VideoStreamType::VideoStreamNative => {
let video_stream = Self {
handle_id: server.next_id(),
close_tx,
stream_type,
track_sid,
};
server.async_runtime.spawn(Self::native_video_stream_task(
server,
video_stream.handle_id,
NativeVideoStream::new(track),
NativeVideoStream::new(rtc_track),
close_rx,
));
Ok::<FfiVideoStream, FfiError>(video_stream)
}
// TODO(theomonnom): Support other stream types
_ => return Err(FfiError::InvalidRequest("unsupported video stream type")),
}?;
@@ -91,10 +89,6 @@ impl FfiVideoStream {
self.stream_type
}
pub fn track_sid(&self) -> &TrackSid {
&self.track_sid
}
async fn native_video_stream_task(
server: &'static server::FfiServer,
stream_handle_id: FfiHandleId,
@@ -143,12 +137,7 @@ impl FfiVideoStream {
pub struct FfiVideoSource {
handle_id: FfiHandleId,
source_type: proto::VideoSourceType,
source: VideoSource,
}
#[derive(Clone)]
pub enum VideoSource {
Native(NativeVideoSource),
source: RtcVideoSource,
}
impl FfiVideoSource {
@@ -157,11 +146,17 @@ impl FfiVideoSource {
new_source: proto::NewVideoSourceRequest,
) -> FfiResult<proto::VideoSourceInfo> {
let source_type = proto::VideoSourceType::from_i32(new_source.r#type).unwrap();
#[allow(unreachable_patterns)]
let source_inner = match source_type {
#[cfg(not(target_arch = "wasm32"))]
proto::VideoSourceType::VideoSourceNative => {
let video_source = NativeVideoSource::default();
VideoSource::Native(video_source)
use livekit::webrtc::video_source::native::NativeVideoSource;
let video_source = NativeVideoSource::new(
new_source.resolution.map(Into::into).unwrap_or_default(),
);
RtcVideoSource::Native(video_source)
}
_ => return Err(FfiError::InvalidRequest("unsupported video source type")),
};
let video_source = Self {
@@ -184,7 +179,8 @@ impl FfiVideoSource {
capture: proto::CaptureVideoFrameRequest,
) -> FfiResult<()> {
match self.source {
VideoSource::Native(ref source) => {
#[cfg(not(target_arch = "wasm32"))]
RtcVideoSource::Native(ref source) => {
let frame_info = capture
.frame
.ok_or(FfiError::InvalidRequest("frame is empty"))?;
@@ -206,12 +202,13 @@ impl FfiVideoSource {
let rotation = proto::VideoRotation::from_i32(frame_info.rotation).unwrap();
let frame = VideoFrame {
rotation: rotation.into(),
timestamp: frame_info.timestamp,
timestamp_us: frame_info.timestamp_us,
buffer,
};
source.capture_frame(&frame);
}
_ => {}
}
Ok(())
}
@@ -224,7 +221,7 @@ impl FfiVideoSource {
self.source_type
}
pub fn inner_source(&self) -> &VideoSource {
pub fn inner_source(&self) -> &RtcVideoSource {
&self.source
}
}
livekit-protocol/protocol
+1 -1
Submodule livekit-protocol/protocol updated: 1e793a8870...89359963fc
livekit-webrtc/Cargo.toml
+1 -1
View File
@@ -15,7 +15,7 @@ thiserror = "1.0"
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
webrtc-sys = { path = "../webrtc-sys", version = "0.1.1" }
lazy_static = "1.4"
parking_lot = { version = "0.12.1", features = ["send_guard"] }
parking_lot = { version = "0.12.1" }
futures = { version = "0.3" }
tokio = { version = "1", features = ["full"] }
cxx = "1.0"
livekit-webrtc/src/audio_source.rs
+45 -2
View File
@@ -1,12 +1,35 @@
use crate::imp::audio_source as imp_as;
use livekit_protocol::enum_dispatch;
#[derive(Default, Debug)]
pub struct AudioSourceOptions {
pub echo_cancellation: bool,
pub noise_suppression: bool,
pub auto_gain_control: bool,
}
#[non_exhaustive]
#[derive(Debug, Clone)]
pub enum RtcAudioSource {
#[cfg(not(target_arch = "wasm32"))]
Native(native::NativeAudioSource),
}
impl RtcAudioSource {
enum_dispatch!(
[Native];
fn set_audio_options(self: &Self, options: AudioSourceOptions) -> ();
fn audio_options(self: &Self) -> AudioSourceOptions;
);
}
#[cfg(not(target_arch = "wasm32"))]
pub mod native {
use super::imp_as;
use super::*;
use crate::audio_frame::AudioFrame;
use std::fmt::{Debug, Formatter};
#[derive(Default, Clone)]
#[derive(Clone)]
pub struct NativeAudioSource {
pub(crate) handle: imp_as::NativeAudioSource,
}
@@ -17,9 +40,29 @@ pub mod native {
}
}
impl Default for NativeAudioSource {
fn default() -> Self {
Self::new(AudioSourceOptions::default())
}
}
impl NativeAudioSource {
pub fn new(options: AudioSourceOptions) -> NativeAudioSource {
Self {
handle: imp_as::NativeAudioSource::new(options),
}
}
pub fn capture_frame(&self, frame: &AudioFrame) {
self.handle.capture_frame(frame)
}
pub fn set_audio_options(&self, options: AudioSourceOptions) {
self.handle.set_audio_options(options)
}
pub fn audio_options(&self) -> AudioSourceOptions {
self.handle.audio_options()
}
}
}
livekit-webrtc/src/native/audio_source.rs
+33 -6
View File
@@ -1,9 +1,29 @@
use crate::audio_frame::AudioFrame;
use crate::{audio_frame::AudioFrame, audio_source::AudioSourceOptions};
use cxx::SharedPtr;
use parking_lot::Mutex;
use std::sync::Arc;
use webrtc_sys::audio_track as sys_at;
impl From<sys_at::ffi::AudioSourceOptions> for AudioSourceOptions {
fn from(options: sys_at::ffi::AudioSourceOptions) -> Self {
Self {
echo_cancellation: options.echo_cancellation,
noise_suppression: options.noise_suppression,
auto_gain_control: options.auto_gain_control,
}
}
}
impl From<AudioSourceOptions> for sys_at::ffi::AudioSourceOptions {
fn from(options: AudioSourceOptions) -> Self {
Self {
echo_cancellation: options.echo_cancellation,
noise_suppression: options.noise_suppression,
auto_gain_control: options.auto_gain_control,
}
}
}
#[derive(Clone)]
pub struct NativeAudioSource {
sys_handle: SharedPtr<sys_at::ffi::AudioTrackSource>,
@@ -18,20 +38,27 @@ struct AudioSourceInner {
num_channels: u32,
}
impl Default for NativeAudioSource {
fn default() -> Self {
impl NativeAudioSource {
pub fn new(options: AudioSourceOptions) -> NativeAudioSource {
Self {
sys_handle: sys_at::ffi::new_audio_track_source(),
sys_handle: sys_at::ffi::new_audio_track_source(options.into()),
inner: Default::default(),
}
}
}
impl NativeAudioSource {
pub fn sys_handle(&self) -> SharedPtr<sys_at::ffi::AudioTrackSource> {
self.sys_handle.clone()
}
pub fn set_audio_options(&self, options: AudioSourceOptions) {
self.sys_handle
.set_audio_options(&sys_at::ffi::AudioSourceOptions::from(options))
}
pub fn audio_options(&self) -> AudioSourceOptions {
self.sys_handle.audio_options().into()
}
pub fn capture_frame(&self, frame: &AudioFrame) {
let mut inner = self.inner.lock();
let samples_10ms = (frame.sample_rate / 100 * frame.num_channels) as usize;
livekit-webrtc/src/native/video_source.rs
+38 -5
View File
@@ -1,22 +1,42 @@
use crate::video_frame::{VideoFrame, VideoFrameBuffer};
use crate::video_source::VideoResolution;
use cxx::SharedPtr;
use std::time::{SystemTime, UNIX_EPOCH};
use webrtc_sys::video_frame as vf_sys;
use webrtc_sys::video_track as vt_sys;
impl From<vt_sys::ffi::VideoResolution> for VideoResolution {
fn from(res: vt_sys::ffi::VideoResolution) -> Self {
Self {
width: res.width,
height: res.height,
}
}
}
impl From<VideoResolution> for vt_sys::ffi::VideoResolution {
fn from(res: VideoResolution) -> Self {
Self {
width: res.width,
height: res.height,
}
}
}
#[derive(Clone)]
pub struct NativeVideoSource {
sys_handle: SharedPtr<vt_sys::ffi::VideoTrackSource>,
}
impl Default for NativeVideoSource {
fn default() -> Self {
impl NativeVideoSource {
pub fn new(resolution: VideoResolution) -> NativeVideoSource {
Self {
sys_handle: vt_sys::ffi::new_video_track_source(),
sys_handle: vt_sys::ffi::new_video_track_source(&vt_sys::ffi::VideoResolution::from(
resolution,
)),
}
}
}
impl NativeVideoSource {
pub fn sys_handle(&self) -> SharedPtr<vt_sys::ffi::VideoTrackSource> {
self.sys_handle.clone()
}
@@ -27,7 +47,20 @@ impl NativeVideoSource {
builder
.pin_mut()
.set_video_frame_buffer(frame.buffer.as_ref().sys_handle());
if frame.timestamp_us == 0 {
// If the timestamp is set to 0, default to now
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
builder.pin_mut().set_timestamp_us(now.as_micros() as i64);
} else {
builder.pin_mut().set_timestamp_us(frame.timestamp_us);
}
self.sys_handle
.on_captured_frame(&builder.pin_mut().build());
}
pub fn video_resolution(&self) -> VideoResolution {
self.sys_handle.video_resolution().into()
}
}
livekit-webrtc/src/native/video_stream.rs
+1 -1
View File
@@ -67,7 +67,7 @@ impl sys_vt::VideoSink for VideoTrackObserver {
fn on_frame(&self, frame: UniquePtr<webrtc_sys::video_frame::ffi::VideoFrame>) {
let _ = self.frame_tx.send(VideoFrame {
rotation: frame.rotation().into(),
timestamp: frame.timestamp_us(),
timestamp_us: frame.timestamp_us(),
buffer: new_video_frame_buffer(unsafe { frame.video_frame_buffer() }),
});
}
livekit-webrtc/src/prelude.rs
+2
View File
@@ -1,4 +1,5 @@
pub use crate::audio_frame::AudioFrame;
pub use crate::audio_source::{AudioSourceOptions, RtcAudioSource};
pub use crate::audio_track::RtcAudioTrack;
pub use crate::data_channel::{
DataBuffer, DataChannel, DataChannelError, DataChannelInit, DataState,
@@ -22,5 +23,6 @@ pub use crate::video_frame::{
BoxVideoFrame, I010Buffer, I420ABuffer, I420Buffer, I422Buffer, I444Buffer, NV12Buffer,
VideoFormatType, VideoFrame, VideoFrameBuffer, VideoFrameBufferType, VideoRotation,
};
pub use crate::video_source::{RtcVideoSource, VideoResolution};
pub use crate::video_track::RtcVideoTrack;
pub use crate::{MediaType, RtcError, RtcErrorType};
livekit-webrtc/src/video_frame.rs
+1 -1
View File
@@ -43,7 +43,7 @@ where
T: AsRef<dyn VideoFrameBuffer>,
{
pub rotation: VideoRotation,
pub timestamp: i64, // When the frame was captured
pub timestamp_us: i64, // When the frame was captured in microseconds
pub buffer: T,
}
livekit-webrtc/src/video_source.rs
+42 -2
View File
@@ -1,12 +1,36 @@
use livekit_protocol::enum_dispatch;
use crate::imp::video_source as vs_imp;
#[derive(Default, Debug, Clone)]
pub struct VideoResolution {
pub width: u32,
pub height: u32,
}
#[non_exhaustive]
#[derive(Debug, Clone)]
pub enum RtcVideoSource {
// TODO(theomonnom): Web video sources (eq. to tracks on browsers?)
#[cfg(not(target_arch = "wasm32"))]
Native(native::NativeVideoSource),
}
// TODO(theomonnom): Support enum dispatch with conditional compilation?
impl RtcVideoSource {
enum_dispatch!(
[Native];
pub fn video_resolution(self: &Self) -> VideoResolution;
);
}
#[cfg(not(target_arch = "wasm32"))]
pub mod native {
use super::vs_imp;
use super::*;
use crate::video_frame::{VideoFrame, VideoFrameBuffer};
use std::fmt::{Debug, Formatter};
#[derive(Default, Clone)]
#[derive(Clone)]
pub struct NativeVideoSource {
pub(crate) handle: vs_imp::NativeVideoSource,
}
@@ -17,10 +41,26 @@ pub mod native {
}
}
impl Default for NativeVideoSource {
fn default() -> Self {
Self::new(VideoResolution::default())
}
}
impl NativeVideoSource {
pub fn new(resolution: VideoResolution) -> Self {
Self {
handle: vs_imp::NativeVideoSource::new(resolution),
}
}
pub fn capture_frame<T: AsRef<dyn VideoFrameBuffer>>(&self, frame: &VideoFrame<T>) {
self.handle.capture_frame(frame)
}
pub fn video_resolution(&self) -> VideoResolution {
self.handle.video_resolution()
}
}
}
livekit/Cargo.toml
+3 -4
View File
@@ -26,10 +26,9 @@ serde = { version = "1", features = ["derive"] }
serde_json = "1.0"
tokio-tungstenite = { version = "0.19" }
tokio = { version = "1", features = ["full"] }
tokio-stream = "0.1"
parking_lot = { version = "0.12.1", features = ["send_guard"] }
parking_lot = { version = "0.12.1" }
url = "2.3"
futures-util = "0.3"
futures-util = { version = "0.3", default-features = false, features = ["sink"] }
thiserror = "1.0"
lazy_static = "1.4"
tracing = "0.1"
log = "0.4"
livekit/src/prelude.rs
+3 -1
View File
@@ -1,6 +1,8 @@
pub use crate::participant::{LocalParticipant, Participant, ParticipantEvent, RemoteParticipant};
pub use crate::{ConnectionState, Room, RoomError, RoomEvent, RoomResult};
pub use crate::{
ConnectionState, DataPacketKind, Room, RoomError, RoomEvent, RoomOptions, RoomResult,
};
pub use crate::publication::{LocalTrackPublication, RemoteTrackPublication, TrackPublication};
livekit/src/proto.rs
+19 -1
View File
@@ -1,4 +1,4 @@
use crate::track;
use crate::{track, DataPacketKind};
use livekit_protocol::*;
// Conversions
@@ -46,3 +46,21 @@ impl From<track::TrackSource> for TrackSource {
}
}
}
impl From<DataPacketKind> for data_packet::Kind {
fn from(kind: DataPacketKind) -> Self {
match kind {
DataPacketKind::Lossy => Self::Lossy,
DataPacketKind::Reliable => Self::Reliable,
}
}
}
impl From<data_packet::Kind> for DataPacketKind {
fn from(kind: data_packet::Kind) -> Self {
match kind {
data_packet::Kind::Lossy => Self::Lossy,
data_packet::Kind::Reliable => Self::Reliable,
}
}
}
livekit/src/room/id.rs
+8 -14
View File
@@ -3,16 +3,6 @@ use std::fmt;
macro_rules! id_str {
($($name:ident;)*) => {
$(
impl $name {
pub fn new(str: String) -> Self {
Self(str)
}
pub fn as_str(&self) -> &str {
&self.0
}
}
impl From<String> for $name {
fn from(str: String) -> $name {
$name(str)
@@ -41,16 +31,20 @@ macro_rules! id_str {
}
#[derive(Clone, Default, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)]
pub struct ParticipantSid(String);
#[repr(transparent)]
pub struct ParticipantSid(pub String);
#[derive(Clone, Default, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)]
pub struct ParticipantIdentity(String);
#[repr(transparent)]
pub struct ParticipantIdentity(pub String);
#[derive(Clone, Default, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)]
pub struct TrackSid(String);
#[repr(transparent)]
pub struct TrackSid(pub String);
#[derive(Clone, Default, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)]
pub struct RoomSid(String);
#[repr(transparent)]
pub struct RoomSid(pub String);
id_str! {
ParticipantSid;
livekit/src/room/mod.rs
+93 -77
View File
@@ -9,12 +9,10 @@ use livekit_protocol::observer::Dispatcher;
use parking_lot::{Mutex, RwLock, RwLockReadGuard};
use std::collections::HashMap;
use std::fmt::Debug;
use std::sync::atomic::{AtomicU8, Ordering};
use std::sync::Arc;
use thiserror::Error;
use tokio::sync::{mpsc, oneshot};
use tokio::task::JoinHandle;
use tracing::{error, info, instrument, trace, Level};
pub use crate::rtc_engine::SimulateScenario;
@@ -82,7 +80,7 @@ pub enum RoomEvent {
},
DataReceived {
payload: Arc<Vec<u8>>,
kind: proto::data_packet::Kind,
kind: DataPacketKind,
participant: RemoteParticipant,
},
ConnectionStateChanged(ConnectionState),
@@ -100,18 +98,41 @@ pub enum ConnectionState {
Unknown,
}
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
pub enum DataPacketKind {
Lossy,
Reliable,
}
#[derive(Debug, Clone)]
pub struct RoomOptions {
pub auto_subscribe: bool,
pub adaptive_stream: bool,
pub dynacast: bool,
}
impl Default for RoomOptions {
fn default() -> Self {
Self {
auto_subscribe: true,
adaptive_stream: false,
dynacast: false,
}
}
}
struct RoomHandle {
session_task: JoinHandle<()>,
close_emitter: oneshot::Sender<()>,
}
pub struct Room {
inner: Arc<SessionInner>,
inner: Arc<RoomSession>,
handle: Mutex<Option<RoomHandle>>,
}
impl Debug for Room {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
f.debug_struct("Room")
.field("sid", &self.sid())
.field("name", &self.name())
@@ -124,14 +145,21 @@ impl Room {
pub async fn connect(
url: &str,
token: &str,
options: RoomOptions,
) -> RoomResult<(Self, mpsc::UnboundedReceiver<RoomEvent>)> {
let (rtc_engine, engine_events) = RtcEngine::new();
let (rtc_engine, engine_events) = RtcEngine::connect(
url,
token,
SignalOptions {
auto_subscribe: options.auto_subscribe,
adaptive_stream: options.adaptive_stream,
..Default::default()
},
)
.await?;
let rtc_engine = Arc::new(rtc_engine);
rtc_engine
.connect(url, token, SignalOptions::default())
.await?;
let join_response = rtc_engine.join_response().unwrap();
let join_response = rtc_engine.join_response();
let pi = join_response.participant.unwrap().clone();
let local_participant = LocalParticipant::new(
rtc_engine.clone(),
@@ -142,11 +170,13 @@ impl Room {
);
let room_info = join_response.room.unwrap();
let inner = Arc::new(SessionInner {
state: AtomicU8::new(ConnectionState::Disconnected as u8),
sid: Mutex::new(room_info.sid.into()),
name: Mutex::new(room_info.name),
metadata: Mutex::new(room_info.metadata),
let inner = Arc::new(RoomSession {
sid: room_info.sid.into(),
name: room_info.name,
info: RwLock::new(RoomInfo {
state: ConnectionState::Disconnected,
metadata: room_info.metadata,
}),
participants: Default::default(),
participants_tasks: Default::default(),
active_speakers: Default::default(),
@@ -196,15 +226,15 @@ impl Room {
}
pub fn sid(&self) -> RoomSid {
self.inner.sid.lock().clone()
self.inner.sid.clone()
}
pub fn name(&self) -> String {
self.inner.name.lock().clone()
self.inner.name.clone()
}
pub fn metadata(&self) -> String {
self.inner.metadata.lock().clone()
self.inner.info.read().metadata.clone()
}
pub fn local_participant(&self) -> LocalParticipant {
@@ -212,7 +242,7 @@ impl Room {
}
pub fn connection_state(&self) -> ConnectionState {
self.inner.state.load(Ordering::Acquire).try_into().unwrap()
self.inner.info.read().state
}
pub fn participants(&self) -> RwLockReadGuard<HashMap<ParticipantSid, RemoteParticipant>> {
@@ -224,20 +254,24 @@ impl Room {
}
}
struct SessionInner {
state: AtomicU8, // ConnectionState
sid: Mutex<RoomSid>,
name: Mutex<String>,
metadata: Mutex<String>,
participants: RwLock<HashMap<ParticipantSid, RemoteParticipant>>,
participants_tasks: RwLock<HashMap<ParticipantSid, (JoinHandle<()>, oneshot::Sender<()>)>>,
active_speakers: RwLock<Vec<Participant>>,
rtc_engine: Arc<RtcEngine>,
local_participant: LocalParticipant,
dispatcher: Dispatcher<RoomEvent>,
struct RoomInfo {
metadata: String,
state: ConnectionState,
}
impl Debug for SessionInner {
pub(crate) struct RoomSession {
rtc_engine: Arc<RtcEngine>,
sid: RoomSid,
name: String,
info: RwLock<RoomInfo>,
dispatcher: Dispatcher<RoomEvent>,
active_speakers: RwLock<Vec<Participant>>,
local_participant: LocalParticipant,
participants: RwLock<HashMap<ParticipantSid, RemoteParticipant>>,
participants_tasks: RwLock<HashMap<ParticipantSid, (JoinHandle<()>, oneshot::Sender<()>)>>,
}
impl Debug for RoomSession {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("SessionInner")
.field("sid", &self.sid)
@@ -247,8 +281,7 @@ impl Debug for SessionInner {
}
}
impl SessionInner {
#[instrument(level = Level::DEBUG)]
impl RoomSession {
async fn room_task(
self: Arc<Self>,
mut engine_events: EngineEvents,
@@ -259,20 +292,19 @@ impl SessionInner {
res = engine_events.recv() => {
if let Some(event) = res {
if let Err(err) = self.on_engine_event(event).await {
error!("failed to handle engine event: {:?}", err);
log::error!("failed to handle engine event: {:?}", err);
}
}
},
_ = &mut close_receiver => {
trace!("closing room_task");
log::trace!("closing room_task");
break;
}
}
}
}
/// Listen to the Participant events and forward them to the Room Dispatcher
#[instrument(level = Level::DEBUG)]
/// Forward participant events to the room dispatcher
async fn participant_task(
self: Arc<Self>,
participant: Participant,
@@ -284,19 +316,18 @@ impl SessionInner {
res = participant_events.recv() => {
if let Some(event) = res {
if let Err(err) = self.on_participant_event(&participant, event).await {
error!("failed to handle participant event for {:?}: {:?}", participant.sid(), err);
log::error!("failed to handle participant event for {:?}: {:?}", participant.sid(), err);
}
}
},
_ = &mut close_rx => {
trace!("closing participant_task for {:?}", participant.sid());
log::trace!("closing participant_task for {:?}", participant.sid());
break;
},
}
}
}
#[instrument(level = Level::DEBUG)]
async fn on_participant_event(
self: &Arc<Self>,
participant: &Participant,
@@ -337,7 +368,6 @@ impl SessionInner {
Ok(())
}
#[instrument(level = Level::DEBUG)]
async fn on_engine_event(self: &Arc<Self>, event: EngineEvent) -> RoomResult<()> {
match event {
EngineEvent::ParticipantUpdate { updates } => self.handle_participant_update(updates),
@@ -414,21 +444,20 @@ impl SessionInner {
Ok(())
}
#[instrument(level = Level::DEBUG)]
async fn close(&self) {
self.rtc_engine.close().await;
}
/// Change the connection state and emit an event
/// Does nothing if the state is already the same
#[instrument(level = Level::DEBUG)]
/// Returns true if the state changed
fn update_connection_state(&self, state: ConnectionState) -> bool {
let old_state = self.state.load(Ordering::Acquire);
if old_state == state as u8 {
let mut info = self.info.write();
if info.state == state {
return false;
}
self.state.store(state as u8, Ordering::Release);
info.state = state;
self.dispatcher
.dispatch(&RoomEvent::ConnectionStateChanged(state));
return true;
@@ -437,7 +466,6 @@ impl SessionInner {
/// Update the participants inside a Room.
/// It'll create, update or remove a participant
/// It also update the participant tracks.
#[instrument(level = Level::DEBUG)]
fn handle_participant_update(self: &Arc<Self>, updates: Vec<proto::ParticipantInfo>) {
for pi in updates {
if pi.sid == self.local_participant.sid()
@@ -452,7 +480,7 @@ impl SessionInner {
if let Some(remote_participant) = remote_participant {
if pi.state == proto::participant_info::State::Disconnected as i32 {
// Participant disconnected
info!("Participant disconnected: {}", pi.sid);
log::info!("Participant disconnected: {}", pi.sid);
self.clone()
.handle_participant_disconnect(remote_participant)
} else {
@@ -461,7 +489,7 @@ impl SessionInner {
}
} else {
// Create a new participant
info!("Participant connected: {}", pi.sid);
log::info!("Participant connected: {}", pi.sid);
let remote_participant = {
let pi = pi.clone();
self.create_participant(pi.sid.into(), pi.identity.into(), pi.name, pi.metadata)
@@ -478,7 +506,6 @@ impl SessionInner {
/// Active speakers changed
/// Update the participants & sort the active_speakers by audio_level
#[instrument(level = Level::DEBUG)]
fn handle_speakers_changed(&self, speakers_info: Vec<proto::SpeakerInfo>) {
let mut speakers = Vec::new();
@@ -513,7 +540,6 @@ impl SessionInner {
/// Handle a connection quality update
/// Emit ConnectionQualityChanged event for the concerned participants
#[instrument(level = Level::DEBUG)]
fn handle_connection_quality_update(&self, updates: Vec<proto::ConnectionQualityInfo>) {
for update in updates {
let participant = {
@@ -542,7 +568,6 @@ impl SessionInner {
}
}
#[instrument(level = Level::DEBUG)]
fn handle_restarting(self: &Arc<Self>) {
// Remove existing participants/subscriptions on full reconnect
for (_, participant) in self.participants.read().iter() {
@@ -555,10 +580,9 @@ impl SessionInner {
}
}
#[instrument(level = Level::DEBUG)]
fn handle_restarted(self: &Arc<Self>) {
// Full reconnect succeeded!
let join_response = self.rtc_engine.join_response().unwrap();
let join_response = self.rtc_engine.join_response();
self.update_connection_state(ConnectionState::Connected);
self.dispatcher.dispatch(&RoomEvent::Reconnected);
@@ -569,22 +593,19 @@ impl SessionInner {
self.handle_participant_update(join_response.other_participants);
// TODO(theomonnom): Synchronize states
// TODO(theomonnom): Room info changed?
// TODO(theomonnom): unpublish & republish tracks
}
#[instrument(level = Level::DEBUG)]
fn handle_disconnected(&self) {
if self.state.load(Ordering::Acquire) == ConnectionState::Disconnected as u8 {
return;
if self.update_connection_state(ConnectionState::Disconnected) {
self.dispatcher.dispatch(&RoomEvent::Disconnected);
}
self.update_connection_state(ConnectionState::Disconnected);
self.dispatcher.dispatch(&RoomEvent::Disconnected);
}
/// Create a new participant
/// Also add it to the participants list
#[instrument(level = Level::DEBUG)]
fn create_participant(
self: &Arc<Self>,
sid: ParticipantSid,
@@ -592,7 +613,13 @@ impl SessionInner {
name: String,
metadata: String,
) -> RemoteParticipant {
let participant = RemoteParticipant::new(sid.clone(), identity, name, metadata);
let participant = RemoteParticipant::new(
self.rtc_engine.clone(),
sid.clone(),
identity,
name,
metadata,
);
// Create the participant task
let (close_tx, close_rx) = oneshot::channel();
@@ -611,7 +638,6 @@ impl SessionInner {
/// A participant has disconnected
/// Cleanup the participant and emit an event
#[instrument(level = Level::DEBUG)]
fn handle_participant_disconnect(self: Arc<Self>, remote_participant: RemoteParticipant) {
tokio::spawn(async move {
for (sid, _) in &*remote_participant.tracks() {
@@ -619,11 +645,12 @@ impl SessionInner {
}
// Close the participant task
if let Some((task, close_tx)) = self
let ptask = self
.participants_tasks
.write()
.remove(&remote_participant.sid())
{
.remove(&remote_participant.sid());
if let Some((task, close_tx)) = ptask {
let _ = close_tx.send(());
let _ = task.await;
}
@@ -649,14 +676,3 @@ fn unpack_stream_id(stream_id: &str) -> Option<(&str, &str)> {
None
}
}
impl From<u8> for ConnectionState {
fn from(value: u8) -> Self {
match value {
0 => ConnectionState::Disconnected,
1 => ConnectionState::Connected,
2 => ConnectionState::Reconnecting,
_ => ConnectionState::Unknown,
}
}
}
livekit/src/room/options.rs
+1 -32
View File
@@ -58,36 +58,6 @@ impl AudioPreset {
}
}
#[derive(Debug, Clone)]
pub struct AudioCaptureOptions {
pub echo_cancellation: bool,
pub noise_suppression: bool,
pub auto_gain_control: bool,
}
impl Default for AudioCaptureOptions {
fn default() -> Self {
Self {
echo_cancellation: true,
noise_suppression: true,
auto_gain_control: true,
}
}
}
#[derive(Clone, Debug)]
pub struct VideoCaptureOptions {
pub resolution: VideoResolution,
}
impl Default for VideoCaptureOptions {
fn default() -> Self {
Self {
resolution: video::H720.resolution(),
}
}
}
#[derive(Clone, Debug)]
pub struct TrackPublishOptions {
// If the encodings aren't set, LiveKit will compute the most appropriate ones
@@ -97,7 +67,7 @@ pub struct TrackPublishOptions {
pub dtx: bool,
pub red: bool,
pub simulcast: bool,
pub name: String,
// pub name: String,
pub source: TrackSource,
}
@@ -110,7 +80,6 @@ impl Default for TrackPublishOptions {
dtx: true,
red: true,
simulcast: true,
name: "unnamed track".to_owned(),
source: TrackSource::Unknown,
}
}
livekit/src/room/participant/local_participant.rs
+37 -29
View File
@@ -1,10 +1,12 @@
use super::{ConnectionQuality, ParticipantInner};
use super::ConnectionQuality;
use super::ParticipantInternal;
use crate::options;
use crate::options::compute_video_encodings;
use crate::options::video_layers_from_encodings;
use crate::options::TrackPublishOptions;
use crate::prelude::*;
use crate::rtc_engine::RtcEngine;
use crate::DataPacketKind;
use livekit_protocol as proto;
use livekit_webrtc::rtp_parameters::RtpEncodingParameters;
use parking_lot::RwLockReadGuard;
@@ -12,12 +14,10 @@ use std::collections::HashMap;
use std::fmt::Debug;
use std::sync::Arc;
use tokio::sync::mpsc;
use tracing::debug;
#[derive(Clone)]
pub struct LocalParticipant {
inner: Arc<ParticipantInner>,
rtc_engine: Arc<RtcEngine>,
inner: Arc<ParticipantInternal>,
}
impl Debug for LocalParticipant {
@@ -39,8 +39,9 @@ impl LocalParticipant {
metadata: String,
) -> Self {
Self {
inner: Arc::new(ParticipantInner::new(sid, identity, name, metadata)),
rtc_engine,
inner: Arc::new(ParticipantInternal::new(
rtc_engine, sid, identity, name, metadata,
)),
}
}
@@ -51,7 +52,7 @@ impl LocalParticipant {
) -> RoomResult<LocalTrackPublication> {
let mut req = proto::AddTrackRequest {
cid: track.rtc_track().id(),
name: options.name.clone(),
name: track.name().clone(),
r#type: proto::TrackType::from(track.kind()) as i32,
muted: track.is_muted(),
source: proto::TrackSource::from(options.source) as i32,
@@ -65,9 +66,9 @@ impl LocalParticipant {
LocalTrack::Video(video_track) => {
// Get the video dimension
// TODO(theomonnom): Use MediaStreamTrack::getSettings() on web
let capture_options = video_track.capture_options();
req.width = capture_options.resolution.width;
req.height = capture_options.resolution.height;
let resolution = video_track.rtc_source().video_resolution();
req.width = resolution.width;
req.height = resolution.height;
encodings = compute_video_encodings(req.width, req.height, &options);
req.layers = video_layers_from_encodings(req.width, req.height, &encodings);
@@ -85,29 +86,34 @@ impl LocalParticipant {
});
}
}
let track_info = self.inner.rtc_engine.add_track(req).await?;
let publication = LocalTrackPublication::new(
track_info.clone(),
Arc::downgrade(&self.inner),
track.clone(),
);
track.update_info(track_info); // Update sid + source
let track_info = self.rtc_engine.add_track(req).await?;
let publication =
LocalTrackPublication::new(track_info.clone(), track.clone(), options.clone());
track.update_info(track_info); // Update SID + Source
debug!("publishing track with cid {:?}", track.rtc_track().id());
log::debug!("publishing track with cid {:?}", track.rtc_track().id());
let transceiver = self
.inner
.rtc_engine
.create_sender(track.clone(), options, encodings)
.await?;
track.update_transceiver(Some(transceiver));
track.start();
//track.start();
track.enable();
tokio::spawn({
let rtc_engine = self.rtc_engine.clone();
let rtc_engine = self.inner.rtc_engine.clone();
async move {
let _ = rtc_engine.negotiate_publisher().await;
}
});
self.inner
.add_track_publication(TrackPublication::Local(publication.clone()));
.add_publication(TrackPublication::Local(publication.clone()));
self.inner
.dispatcher
@@ -125,9 +131,10 @@ impl LocalParticipant {
) -> RoomResult<LocalTrackPublication> {
let mut tracks = self.inner.tracks.write();
if let Some(TrackPublication::Local(publication)) = tracks.remove(&track) {
let track = publication.track().unwrap();
let track = publication.track();
let sender = track.transceiver().unwrap().sender();
self.rtc_engine.remove_track(sender).await?;
self.inner.rtc_engine.remove_track(sender).await?;
track.update_transceiver(None);
self.inner
@@ -135,10 +142,10 @@ impl LocalParticipant {
.dispatch(&ParticipantEvent::LocalTrackUnpublished {
publication: publication.clone(),
});
publication.update_track(None);
// publication.update_track(None);
tokio::spawn({
let rtc_engine = self.rtc_engine.clone();
let rtc_engine = self.inner.rtc_engine.clone();
async move {
let _ = rtc_engine.negotiate_publisher().await;
}
@@ -152,20 +159,21 @@ impl LocalParticipant {
pub async fn publish_data(
&self,
data: &[u8],
kind: proto::data_packet::Kind,
) -> Result<(), RoomError> {
data: Vec<u8>,
kind: DataPacketKind,
destination_sids: Vec<String>,
) -> RoomResult<()> {
let data = proto::DataPacket {
kind: kind as i32,
value: Some(proto::data_packet::Value::User(proto::UserPacket {
participant_sid: self.sid().to_string(),
payload: data.to_vec(),
destination_sids: vec![],
payload: data,
destination_sids: destination_sids.to_owned(),
..Default::default()
})),
};
self.rtc_engine
self.inner
.rtc_engine
.publish_data(&data, kind)
.await
.map_err(Into::into)
livekit/src/room/participant/mod.rs
+59 -44
View File
@@ -1,14 +1,15 @@
use crate::prelude::*;
use crate::rtc_engine::RtcEngine;
use crate::track::TrackError;
use livekit_protocol as proto;
use livekit_protocol::enum_dispatch;
use livekit_protocol::observer::Dispatcher;
use parking_lot::{Mutex, RwLock, RwLockReadGuard};
use parking_lot::{RwLock, RwLockReadGuard};
use std::collections::HashMap;
use std::fmt::Debug;
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU8, Ordering};
use std::sync::Arc;
use tokio::sync::mpsc;
use std::thread::JoinHandle;
use tokio::sync::{mpsc, oneshot};
mod local_participant;
mod remote_participant;
@@ -38,7 +39,7 @@ pub enum ParticipantEvent {
},
DataReceived {
payload: Arc<Vec<u8>>,
kind: proto::data_packet::Kind,
kind: DataPacketKind,
},
SpeakingChanged {
speaking: bool,
@@ -109,7 +110,6 @@ impl Participant {
pub fn tracks(self: &Self) -> RwLockReadGuard<HashMap<TrackSid, TrackPublication>>;
pub fn register_observer(self: &Self) -> mpsc::UnboundedReceiver<ParticipantEvent>;
// Internal functions
pub(crate) fn set_speaking(self: &Self, speaking: bool) -> ();
pub(crate) fn set_audio_level(self: &Self, level: f32) -> ();
pub(crate) fn set_connection_quality(self: &Self, quality: ConnectionQuality) -> ();
@@ -118,56 +118,76 @@ impl Participant {
}
#[derive(Debug)]
pub(crate) struct ParticipantInner {
pub sid: Mutex<ParticipantSid>,
pub identity: Mutex<ParticipantIdentity>,
pub name: Mutex<String>,
pub metadata: Mutex<String>,
pub speaking: AtomicBool,
pub tracks: RwLock<HashMap<TrackSid, TrackPublication>>,
pub audio_level: AtomicU32,
pub connection_quality: AtomicU8,
pub dispatcher: Dispatcher<ParticipantEvent>,
pub(crate) struct ParticipantInfo {
pub sid: ParticipantSid,
pub identity: ParticipantIdentity,
pub name: String,
pub metadata: String,
pub speaking: bool,
pub audio_level: f32,
pub connection_quality: ConnectionQuality,
}
impl ParticipantInner {
#[derive(Debug)]
pub(crate) struct ParticipantInternal {
pub(super) rtc_engine: Arc<RtcEngine>,
pub(super) dispatcher: Dispatcher<ParticipantEvent>,
info: RwLock<ParticipantInfo>,
tracks: RwLock<HashMap<TrackSid, TrackPublication>>,
tracks_tasks: RwLock<HashMap<TrackSid, (JoinHandle<()>, oneshot::Sender<()>)>>,
}
impl ParticipantInternal {
pub fn new(
rtc_engine: Arc<RtcEngine>,
sid: ParticipantSid,
identity: ParticipantIdentity,
name: String,
metadata: String,
) -> Self {
Self {
sid: Mutex::new(sid),
identity: Mutex::new(identity),
name: Mutex::new(name),
metadata: Mutex::new(metadata),
tracks: Default::default(),
speaking: Default::default(),
audio_level: Default::default(),
connection_quality: AtomicU8::new(ConnectionQuality::Unknown as u8),
rtc_engine,
info: RwLock::new(ParticipantInfo {
sid,
identity,
name,
metadata,
speaking: false,
audio_level: 0.0,
connection_quality: ConnectionQuality::Unknown,
}),
dispatcher: Default::default(),
tracks: Default::default(),
tracks_tasks: Default::default(),
}
}
pub fn update_info(&self, new_info: proto::ParticipantInfo) {
let mut info = self.info.write();
info.sid = new_info.sid.into();
info.name = new_info.name;
info.identity = new_info.identity.into();
info.metadata = new_info.metadata; // TODO(theomonnom): callback MetadataChanged
}
pub fn sid(&self) -> ParticipantSid {
self.sid.lock().clone()
self.info.read().sid.clone()
}
pub fn identity(&self) -> ParticipantIdentity {
self.identity.lock().clone()
self.info.read().identity.clone()
}
pub fn name(&self) -> String {
self.name.lock().clone()
self.info.read().name.clone()
}
pub fn metadata(&self) -> String {
self.metadata.lock().clone()
self.info.read().metadata.clone()
}
pub fn is_speaking(&self) -> bool {
self.speaking.load(Ordering::SeqCst)
self.info.read().speaking
}
pub fn tracks(&self) -> RwLockReadGuard<HashMap<TrackSid, TrackPublication>> {
@@ -175,39 +195,34 @@ impl ParticipantInner {
}
pub fn audio_level(&self) -> f32 {
f32::from_bits(self.audio_level.load(Ordering::SeqCst))
self.info.read().audio_level
}
pub fn connection_quality(&self) -> ConnectionQuality {
self.connection_quality.load(Ordering::SeqCst).into()
self.info.read().connection_quality
}
pub fn register_observer(&self) -> mpsc::UnboundedReceiver<ParticipantEvent> {
self.dispatcher.register()
}
pub fn update_info(&self, info: proto::ParticipantInfo) {
*self.sid.lock() = info.sid.into();
*self.identity.lock() = info.identity.into();
*self.name.lock() = info.name;
*self.metadata.lock() = info.metadata; // TODO(theomonnom): callback MetadataChanged
}
pub fn set_speaking(&self, speaking: bool) {
self.speaking.store(speaking, Ordering::SeqCst);
self.info.write().speaking = speaking;
}
pub fn set_audio_level(&self, audio_level: f32) {
self.audio_level
.store(audio_level.to_bits(), Ordering::SeqCst)
self.info.write().audio_level = audio_level;
}
pub fn set_connection_quality(&self, quality: ConnectionQuality) {
self.connection_quality
.store(quality as u8, Ordering::SeqCst);
self.info.write().connection_quality = quality;
}
pub fn add_track_publication(&self, publication: TrackPublication) {
pub fn remove_publication(&self, sid: &TrackSid) {
self.tracks.write().remove(sid);
}
pub fn add_publication(&self, publication: TrackPublication) {
self.tracks.write().insert(publication.sid(), publication);
}
}
livekit/src/room/participant/remote_participant.rs
+34 -27
View File
@@ -1,6 +1,8 @@
use super::{ConnectionQuality, ParticipantInner};
use crate::prelude::*;
use super::TrackKind;
use super::{ConnectionQuality, ParticipantInternal};
use crate::rtc_engine::RtcEngine;
use crate::track::TrackError;
use crate::{prelude::*, DataPacketKind};
use livekit_protocol as proto;
use livekit_webrtc::prelude::*;
use parking_lot::RwLockReadGuard;
@@ -10,13 +12,12 @@ use std::sync::Arc;
use std::time::Duration;
use tokio::sync::mpsc;
use tokio::time::timeout;
use tracing::{debug, error, instrument, Level};
const ADD_TRACK_TIMEOUT: Duration = Duration::from_secs(5);
#[derive(Clone)]
pub struct RemoteParticipant {
inner: Arc<ParticipantInner>,
inner: Arc<ParticipantInternal>,
}
impl Debug for RemoteParticipant {
@@ -31,29 +32,22 @@ impl Debug for RemoteParticipant {
impl RemoteParticipant {
pub(crate) fn new(
rtc_engine: Arc<RtcEngine>,
sid: ParticipantSid,
identity: ParticipantIdentity,
name: String,
metadata: String,
) -> Self {
Self {
inner: Arc::new(ParticipantInner::new(sid, identity, name, metadata)),
inner: Arc::new(ParticipantInternal::new(
rtc_engine, sid, identity, name, metadata,
)),
}
}
#[inline]
pub fn get_track_publication(&self, sid: &TrackSid) -> Option<RemoteTrackPublication> {
self.inner.tracks.read().get(sid).map(|track| {
if let TrackPublication::Remote(remote) = track {
return remote.clone();
}
unreachable!()
})
}
/// Called by the RoomSession when receiving data from the RrcSession
/// Called by the RoomSession when receiving data from the RtcSession
/// It is just used to emit the Data event on the participant dispatcher.
pub(crate) fn on_data_received(&self, data: Arc<Vec<u8>>, kind: proto::data_packet::Kind) {
pub(crate) fn on_data_received(&self, data: Arc<Vec<u8>>, kind: DataPacketKind) {
self.inner
.dispatcher
.dispatch(&ParticipantEvent::DataReceived {
@@ -62,7 +56,6 @@ impl RemoteParticipant {
});
}
#[instrument(level = Level::DEBUG)]
pub(crate) async fn add_subscribed_media_track(
&self,
sid: TrackSid,
@@ -78,7 +71,7 @@ impl RemoteParticipant {
return publication;
}
tokio::task::yield_now().await; // Remove yield
tokio::time::sleep(Duration::from_millis(50)).await;
}
}
};
@@ -111,10 +104,10 @@ impl RemoteParticipant {
}
};
debug!("starting track: {:?}", sid);
log::debug!("starting track: {:?}", sid);
remote_publication.update_track(Some(track.clone().into()));
track.set_muted(remote_publication.is_muted());
//track.set_muted(remote_publication.is_muted());
track.update_info(proto::TrackInfo {
sid: remote_publication.sid().to_string(),
name: remote_publication.name().to_string(),
@@ -124,8 +117,9 @@ impl RemoteParticipant {
});
self.inner
.add_track_publication(TrackPublication::Remote(remote_publication.clone()));
track.start();
.add_publication(TrackPublication::Remote(remote_publication.clone()));
// track.start();
track.enable();
self.inner
.dispatcher
@@ -134,7 +128,7 @@ impl RemoteParticipant {
publication: remote_publication,
});
} else {
error!("could not find published track with sid: {:?}", sid);
log::error!("could not find published track with sid: {:?}", sid);
self.inner
.dispatcher
@@ -149,7 +143,7 @@ impl RemoteParticipant {
if let Some(publication) = self.get_track_publication(sid) {
// Unsubscribe to the track if needed
if let Some(track) = publication.track() {
track.stop();
track.disable();
self.inner
.dispatcher
@@ -159,6 +153,8 @@ impl RemoteParticipant {
});
}
self.inner.remove_publication(sid);
self.inner
.dispatcher
.dispatch(&ParticipantEvent::TrackUnpublished {
@@ -177,9 +173,10 @@ impl RemoteParticipant {
if let Some(publication) = self.get_track_publication(&track.sid.clone().into()) {
publication.update_info(track.clone());
} else {
let publication = RemoteTrackPublication::new(track.clone(), None);
let publication =
RemoteTrackPublication::new(track.clone(), Arc::downgrade(&self.inner), None);
self.inner
.add_track_publication(TrackPublication::Remote(publication.clone()));
.add_publication(TrackPublication::Remote(publication.clone()));
// This is a new track, dispatch publish event
self.inner
@@ -200,6 +197,16 @@ impl RemoteParticipant {
}
}
#[inline]
pub fn get_track_publication(&self, sid: &TrackSid) -> Option<RemoteTrackPublication> {
self.inner.tracks.read().get(sid).map(|track| {
if let TrackPublication::Remote(remote) = track {
return remote.clone();
}
unreachable!()
})
}
#[inline]
pub fn sid(&self) -> ParticipantSid {
self.inner.sid()
livekit/src/room/publication/local.rs
+42 -14
View File
@@ -1,17 +1,13 @@
use super::TrackPublicationInner;
use crate::id::TrackSid;
use crate::options::TrackPublishOptions;
use crate::track::{LocalTrack, Track, TrackDimension, TrackKind, TrackSource};
use crate::participant::ParticipantInternal;
use crate::track::{LocalTrack, TrackDimension, TrackKind, TrackSource};
use livekit_protocol as proto;
use parking_lot::Mutex;
use std::sync::Arc;
use std::sync::{Arc, Weak};
#[derive(Debug)]
struct LocalTrackPublicationInner {
publication_inner: TrackPublicationInner,
#[allow(unused)] // TODO(theomonnom)
options: Mutex<TrackPublishOptions>,
}
#[derive(Clone, Debug)]
@@ -22,17 +18,47 @@ pub struct LocalTrackPublication {
impl LocalTrackPublication {
pub(crate) fn new(
info: proto::TrackInfo,
participant: Weak<ParticipantInternal>,
track: LocalTrack,
options: TrackPublishOptions,
) -> Self {
Self {
inner: Arc::new(LocalTrackPublicationInner {
publication_inner: TrackPublicationInner::new(info, Some(track.into())),
options: Mutex::new(options),
publication_inner: TrackPublicationInner::new(
info,
participant,
Some(track.into()),
),
}),
}
}
pub async fn mute(&self) {}
pub async fn unmute(&self) {}
pub async fn pause_upstream(&self) {}
pub async fn resume_upstream(&self) {}
/*pub fn set_muted(&self, muted: bool) {
if self.is_muted() == muted {
return;
}
self.track().rtc_track().set_enabled(!muted);
let participant = self.inner.publication_inner.participant().upgrade();
if participant.is_none() {
log::warn!("publication's participant is invalid, set_muted failed");
return;
}
let participant = participant.unwrap();
// Engine update muted
// Participant MUTED/UNMUTED event
}*/
#[inline]
pub fn sid(&self) -> TrackSid {
self.inner.publication_inner.sid()
@@ -64,11 +90,13 @@ impl LocalTrackPublication {
}
#[inline]
pub fn track(&self) -> Option<LocalTrack> {
pub fn track(&self) -> LocalTrack {
self.inner
.publication_inner
.track()
.map(|track| track.try_into().unwrap())
.unwrap()
.try_into()
.unwrap()
}
#[inline]
@@ -86,10 +114,10 @@ impl LocalTrackPublication {
false
}
#[inline]
/*#[inline]
pub(crate) fn update_track(&self, track: Option<Track>) {
self.inner.publication_inner.update_track(track);
}
}*/
#[allow(dead_code)]
#[inline]
livekit/src/room/publication/mod.rs
+204 -124
View File
@@ -1,140 +1,49 @@
use super::track::{TrackDimension, TrackEvent};
use super::track::TrackDimension;
use crate::participant::ParticipantInternal;
use crate::prelude::*;
use crate::track::Track;
use futures_util::stream::StreamExt;
use livekit_protocol as proto;
use livekit_protocol::enum_dispatch;
use livekit_protocol::observer::Dispatcher;
use parking_lot::Mutex;
use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
use parking_lot::{Mutex, RwLock};
use proto::observer::Dispatcher;
use std::sync::Arc;
use std::sync::Weak;
use tokio::sync::Notify;
use tokio_stream::wrappers::UnboundedReceiverStream;
mod local;
pub use local::*;
mod remote;
pub use local::*;
pub use remote::*;
#[derive(Debug)]
pub(crate) struct TrackPublicationInner {
track: Mutex<Option<Track>>,
name: Mutex<String>,
sid: Mutex<TrackSid>,
kind: AtomicU8, // Casted to TrackKind
source: AtomicU8, // Casted to TrackSource
simulcasted: AtomicBool,
dimension: Mutex<TrackDimension>,
mime_type: Mutex<String>,
muted: AtomicBool,
dispatcher: Dispatcher<TrackEvent>,
close_notifier: Arc<Notify>,
#[derive(Debug, Clone)]
pub enum PublicationEvent {
Muted,
Unmuted,
Subscribed,
Unsubscribed,
SubscriptionStatusChanged {
old_state: SubscriptionStatus,
new_state: SubscriptionStatus,
},
SubscriptionPermissionChanged {
old_state: PermissionStatus,
new_state: PermissionStatus,
},
SubscriptionFailed,
}
impl TrackPublicationInner {
pub fn new(info: proto::TrackInfo, track: Option<Track>) -> Self {
Self {
track: Mutex::new(track),
name: Mutex::new(info.name),
sid: Mutex::new(info.sid.into()),
kind: AtomicU8::new(
TrackKind::try_from(proto::TrackType::from_i32(info.r#type).unwrap()).unwrap()
as u8,
),
source: AtomicU8::new(TrackSource::from(
proto::TrackSource::from_i32(info.source).unwrap(),
) as u8),
simulcasted: AtomicBool::new(info.simulcast),
dimension: Mutex::new(TrackDimension(info.width, info.height)),
mime_type: Mutex::new(info.mime_type),
muted: AtomicBool::new(info.muted),
dispatcher: Default::default(),
close_notifier: Default::default(),
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SubscriptionStatus {
Desired,
Subscribed,
Unsubscribed,
}
pub fn update_track(&self, track: Option<Track>) {
let mut old_track = self.track.lock();
*old_track = track.clone();
self.close_notifier.notify_waiters();
if let Some(track) = track.as_ref() {
let track_stream = UnboundedReceiverStream::new(track.register_observer());
tokio::spawn({
let dispatcher = self.dispatcher.clone();
let notifier = self.close_notifier.clone();
async move {
let notified = notifier.notified();
futures_util::pin_mut!(notified);
futures_util::future::select(
track_stream.map(Ok).forward(dispatcher),
notified,
)
.await;
}
});
}
}
pub fn update_info(&self, info: proto::TrackInfo) {
*self.name.lock() = info.name;
*self.sid.lock() = info.sid.into();
*self.dimension.lock() = TrackDimension(info.width, info.height);
*self.mime_type.lock() = info.mime_type;
self.kind.store(
TrackKind::try_from(proto::TrackType::from_i32(info.r#type).unwrap()).unwrap() as u8,
Ordering::SeqCst,
);
self.source.store(
TrackSource::from(proto::TrackSource::from_i32(info.source).unwrap()) as u8,
Ordering::SeqCst,
);
self.simulcasted.store(info.simulcast, Ordering::SeqCst);
self.muted.store(info.muted, Ordering::SeqCst);
if let Some(track) = self.track.lock().as_ref() {
track.set_muted(info.muted);
}
}
pub fn sid(&self) -> TrackSid {
self.sid.lock().clone()
}
pub fn name(&self) -> String {
self.name.lock().clone()
}
pub fn kind(&self) -> TrackKind {
self.kind.load(Ordering::SeqCst).try_into().unwrap()
}
pub fn source(&self) -> TrackSource {
self.source.load(Ordering::SeqCst).into()
}
pub fn simulcasted(&self) -> bool {
self.simulcasted.load(Ordering::Relaxed)
}
pub fn dimension(&self) -> TrackDimension {
self.dimension.lock().clone()
}
pub fn mime_type(&self) -> String {
self.mime_type.lock().clone()
}
pub fn track(&self) -> Option<Track> {
self.track.lock().clone()
}
pub fn is_muted(&self) -> bool {
self.muted.load(Ordering::Relaxed)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PermissionStatus {
Allowed,
NotAllowed,
}
#[derive(Clone, Debug)]
@@ -159,8 +68,179 @@ impl TrackPublication {
pub fn track(&self) -> Option<Track> {
match self {
TrackPublication::Local(p) => p.track().map(Into::into),
TrackPublication::Local(p) => Some(p.track().into()),
TrackPublication::Remote(p) => p.track().map(Into::into),
}
}
}
#[derive(Debug)]
pub(crate) struct PublicationInfo {
track: Option<Track>,
name: String,
sid: TrackSid,
kind: TrackKind,
source: TrackSource,
simulcasted: bool,
dimension: TrackDimension,
mime_type: String,
muted: bool,
}
#[derive(Debug)]
pub(crate) struct TrackPublicationInner {
info: RwLock<PublicationInfo>,
dispatcher: Dispatcher<PublicationEvent>,
participant: Weak<ParticipantInternal>,
//forward_handle: Mutex<Option<tokio::task::JoinHandle<()>>>,
forward_close: Arc<Notify>,
}
impl TrackPublicationInner {
pub fn new(
info: proto::TrackInfo,
participant: Weak<ParticipantInternal>,
track: Option<Track>,
) -> Self {
let info = PublicationInfo {
track,
name: info.name,
sid: info.sid.into(),
kind: proto::TrackType::from_i32(info.r#type)
.unwrap()
.try_into()
.unwrap(),
source: proto::TrackSource::from_i32(info.source)
.unwrap()
.try_into()
.unwrap(),
simulcasted: info.simulcast,
dimension: TrackDimension(info.width, info.height),
mime_type: info.mime_type,
muted: info.muted,
};
Self {
info: RwLock::new(info),
dispatcher: Default::default(),
participant,
//forward_handle: Default::default(),
forward_close: Default::default(),
}
}
// Forward track events to the publication events
// e.g: this also allow us to access the signal_client and notify the server if
// a local track changed mute state
async fn track_forward_task(
close_notifier: Weak<Notify>,
track: Track,
dispatcher: Dispatcher<PublicationEvent>,
) {
let mut track_events = track.register_observer();
loop {
let notifier = close_notifier.upgrade();
if notifier.is_none() {
break;
}
let notified = notifier.as_ref().unwrap().notified();
tokio::select! {
_ = notified => {
break;
}
Some(event) = track_events.recv() => {
match event {
TrackEvent::Muted => {
dispatcher.dispatch(&PublicationEvent::Muted);
}
TrackEvent::Unmuted => {
dispatcher.dispatch(&PublicationEvent::Unmuted);
}
}
}
}
}
}
pub fn update_track(&self, track: Option<Track>) {
//let forward_task = self.forward_handle.lock().take();
//if let Some(task) = forward_task {
// Make sure to close the old forwarder before changing the track
self.forward_close.notify_waiters();
//let _ = task.await;
// }
let mut info = self.info.write();
info.track = track.clone();
if let Some(track) = track {
let _handle = tokio::spawn(Self::track_forward_task(
Arc::downgrade(&self.forward_close),
track,
self.dispatcher.clone(),
));
//let mut forward_handle = self.forward_handle.lock();
//*forward_handle = Some(handle);
}
}
// Called when updating a participant info
pub fn update_info(&self, new_info: proto::TrackInfo) {
let mut info = self.info.write();
info.name = new_info.name;
info.sid = new_info.sid.into();
info.dimension = TrackDimension(new_info.width, new_info.height);
info.mime_type = new_info.mime_type;
info.kind =
TrackKind::try_from(proto::TrackType::from_i32(new_info.r#type).unwrap()).unwrap();
info.source = TrackSource::from(proto::TrackSource::from_i32(new_info.source).unwrap());
info.simulcasted = new_info.simulcast;
// TODO MUTE ?????????????????
// info.muted = new_info.muted;
// if let Some(track) = info.track.as_ref() {
// track.set_muted(info.muted);
// }
}
pub fn participant(&self) -> Weak<ParticipantInternal> {
self.participant.clone()
}
pub fn sid(&self) -> TrackSid {
self.info.read().sid.clone()
}
pub fn name(&self) -> String {
self.info.read().name.clone()
}
pub fn kind(&self) -> TrackKind {
self.info.read().kind
}
pub fn source(&self) -> TrackSource {
self.info.read().source
}
pub fn simulcasted(&self) -> bool {
self.info.read().simulcasted
}
pub fn dimension(&self) -> TrackDimension {
self.info.read().dimension.clone()
}
pub fn mime_type(&self) -> String {
self.info.read().mime_type.clone()
}
pub fn track(&self) -> Option<Track> {
self.info.read().track.clone()
}
pub fn is_muted(&self) -> bool {
self.info.read().muted
}
}
livekit/src/room/publication/remote.rs
+122 -16
View File
@@ -1,64 +1,170 @@
use super::TrackPublicationInner;
use super::{PermissionStatus, SubscriptionStatus, TrackPublicationInner};
use crate::id::TrackSid;
use crate::participant::ParticipantInternal;
use crate::publication::PublicationEvent;
use crate::track::{RemoteTrack, Track, TrackDimension, TrackKind, TrackSource};
use livekit_protocol as proto;
use std::sync::Arc;
use parking_lot::RwLock;
use std::sync::{Arc, Weak};
#[derive(Debug)]
struct RemoteInfo {
subscribed: bool,
allowed: bool,
// TODO(theomonnom): other remote info
}
#[derive(Debug)]
struct RemoteInner {
publication_inner: TrackPublicationInner,
info: RwLock<RemoteInfo>,
}
#[derive(Clone, Debug)]
pub struct RemoteTrackPublication {
inner: Arc<TrackPublicationInner>,
inner: Arc<RemoteInner>,
}
impl RemoteTrackPublication {
pub(crate) fn new(info: proto::TrackInfo, track: Option<RemoteTrack>) -> Self {
pub(crate) fn new(
info: proto::TrackInfo,
participant: Weak<ParticipantInternal>,
track: Option<RemoteTrack>,
) -> Self {
Self {
inner: Arc::new(TrackPublicationInner::new(info, track.map(Into::into))),
inner: Arc::new(RemoteInner {
publication_inner: TrackPublicationInner::new(
info,
participant,
track.map(Into::into),
),
info: RwLock::new(RemoteInfo {
subscribed: false,
allowed: false,
}),
}),
}
}
pub fn set_subscribed(&self, subscribed: bool) {
let old_subscription_state = self.subscription_status();
let old_permission_state = self.permission_status();
let mut info = self.inner.info.write();
info.subscribed = subscribed;
if subscribed {
info.allowed = true;
}
let participant = self.inner.publication_inner.participant.upgrade();
if participant.is_none() {
log::warn!("publication's participant is invalid, set_subscribed failed");
return;
}
let participant = participant.unwrap();
let update_subscription = proto::UpdateSubscription {
track_sids: vec![self.sid().0],
subscribe: subscribed,
participant_tracks: vec![proto::ParticipantTracks {
participant_sid: participant.sid().0,
track_sids: vec![self.sid().0],
}],
};
// Engine update subscription
if old_subscription_state != self.subscription_status() {
self.inner.publication_inner.dispatcher.dispatch(
&PublicationEvent::SubscriptionStatusChanged {
old_state: old_subscription_state,
new_state: self.subscription_status(),
},
)
}
if old_permission_state != self.permission_status() {
self.inner.publication_inner.dispatcher.dispatch(
&PublicationEvent::SubscriptionPermissionChanged {
old_state: old_permission_state,
new_state: self.permission_status(),
},
)
}
}
#[inline]
pub fn subscription_status(&self) -> SubscriptionStatus {
if !self.inner.info.read().subscribed {
return SubscriptionStatus::Unsubscribed;
}
if self.track().is_none() {
return SubscriptionStatus::Desired;
}
SubscriptionStatus::Subscribed
}
#[inline]
pub fn permission_status(&self) -> PermissionStatus {
if self.inner.info.read().allowed {
PermissionStatus::Allowed
} else {
PermissionStatus::NotAllowed
}
}
pub fn is_subscribed(&self) -> bool {
self.inner.info.read().allowed && self.track().is_some()
}
#[inline]
pub fn sid(&self) -> TrackSid {
self.inner.sid()
self.inner.publication_inner.sid()
}
#[inline]
pub fn name(&self) -> String {
self.inner.name()
self.inner.publication_inner.name()
}
#[inline]
pub fn kind(&self) -> TrackKind {
self.inner.kind()
self.inner.publication_inner.kind()
}
#[inline]
pub fn source(&self) -> TrackSource {
self.inner.source()
self.inner.publication_inner.source()
}
#[inline]
pub fn simulcasted(&self) -> bool {
self.inner.simulcasted()
self.inner.publication_inner.simulcasted()
}
#[inline]
pub fn dimension(&self) -> TrackDimension {
self.inner.dimension()
self.inner.publication_inner.dimension()
}
#[inline]
pub fn track(&self) -> Option<RemoteTrack> {
self.inner.track().map(|track| track.try_into().unwrap())
self.inner
.publication_inner
.track()
.map(|track| track.try_into().unwrap())
}
#[inline]
pub fn mime_type(&self) -> String {
self.inner.mime_type()
self.inner.publication_inner.mime_type()
}
#[inline]
pub fn is_muted(&self) -> bool {
self.inner.is_muted()
self.inner.publication_inner.is_muted()
}
#[inline]
@@ -68,11 +174,11 @@ impl RemoteTrackPublication {
#[inline]
pub(crate) fn update_track(&self, track: Option<Track>) {
self.inner.update_track(track);
self.inner.publication_inner.update_track(track);
}
#[inline]
pub(crate) fn update_info(&self, info: proto::TrackInfo) {
self.inner.update_info(info);
self.inner.publication_inner.update_info(info);
}
}
livekit/src/room/track/local_audio_track.rs
+51 -56
View File
@@ -1,24 +1,17 @@
use super::TrackInner;
use crate::options::AudioCaptureOptions;
use crate::prelude::*;
use crate::rtc_engine::lk_runtime::LkRuntime;
use crate::webrtc::peer_connection_factory::native::PeerConnectionFactoryExt;
use core::panic;
use livekit_protocol as proto;
use livekit_webrtc::prelude::*;
use parking_lot::Mutex;
use std::fmt::Debug;
use std::sync::Arc;
use tokio::sync::mpsc;
#[derive(Debug)]
pub struct LocalAudioTrackInner {
track_inner: TrackInner,
capture_options: Mutex<AudioCaptureOptions>,
}
#[derive(Clone)]
pub struct LocalAudioTrack {
inner: Arc<LocalAudioTrackInner>,
inner: Arc<TrackInner>,
source: RtcAudioSource,
}
impl Debug for LocalAudioTrack {
@@ -32,85 +25,84 @@ impl Debug for LocalAudioTrack {
}
impl LocalAudioTrack {
pub(crate) fn new(
name: String,
rtc_track: RtcAudioTrack,
capture_options: AudioCaptureOptions,
) -> Self {
pub(crate) fn new(name: String, rtc_track: RtcAudioTrack, source: RtcAudioSource) -> Self {
Self {
inner: Arc::new(LocalAudioTrackInner {
track_inner: TrackInner::new(
"unknown".to_string().into(), // sid
name,
TrackKind::Audio,
MediaStreamTrack::Audio(rtc_track),
),
capture_options: Mutex::new(capture_options),
}),
inner: Arc::new(TrackInner::new(
"unknown".to_string().into(), // sid
name,
TrackKind::Audio,
MediaStreamTrack::Audio(rtc_track),
)),
source,
}
}
#[inline]
pub fn capture_options(&self) -> AudioCaptureOptions {
self.inner.capture_options.lock().clone()
}
#[inline]
pub fn sid(&self) -> TrackSid {
self.inner.track_inner.sid()
self.inner.sid()
}
#[inline]
pub fn name(&self) -> String {
self.inner.track_inner.name()
self.inner.name()
}
#[inline]
pub fn kind(&self) -> TrackKind {
self.inner.track_inner.kind()
self.inner.kind()
}
#[inline]
pub fn source(&self) -> TrackSource {
self.inner.track_inner.source()
self.inner.source()
}
#[inline]
pub fn stream_state(&self) -> StreamState {
self.inner.track_inner.stream_state()
self.inner.stream_state()
}
#[inline]
pub fn start(&self) {
self.inner.track_inner.start()
pub fn enable(&self) {
self.inner.enable()
}
#[inline]
pub fn stop(&self) {
self.inner.track_inner.stop()
pub fn disable(&self) {
self.inner.disable()
}
#[inline]
pub fn is_muted(&self) -> bool {
self.inner.track_inner.is_muted()
self.inner.is_muted()
}
#[inline]
pub fn set_muted(&self, muted: bool) {
self.inner.track_inner.set_muted(muted)
pub fn mute(&self) {
self.inner.set_muted(true);
}
#[inline]
pub fn unmute(&self) {
self.inner.set_muted(false);
}
#[inline]
pub fn rtc_track(&self) -> RtcAudioTrack {
if let MediaStreamTrack::Audio(audio) = self.inner.track_inner.rtc_track() {
if let MediaStreamTrack::Audio(audio) = self.inner.rtc_track() {
return audio;
}
unreachable!()
}
#[inline]
pub fn rtc_source(&self) -> RtcAudioSource {
self.source.clone()
}
#[inline]
pub fn register_observer(&self) -> mpsc::UnboundedReceiver<TrackEvent> {
self.inner.track_inner.register_observer()
self.inner.register_observer()
}
#[inline]
@@ -120,30 +112,33 @@ impl LocalAudioTrack {
#[inline]
pub(crate) fn transceiver(&self) -> Option<RtpTransceiver> {
self.inner.track_inner.transceiver()
self.inner.transceiver()
}
#[inline]
pub(crate) fn update_transceiver(&self, transceiver: Option<RtpTransceiver>) {
self.inner.track_inner.update_transceiver(transceiver)
self.inner.update_transceiver(transceiver)
}
#[inline]
pub(crate) fn update_info(&self, info: proto::TrackInfo) {
self.inner.track_inner.update_info(info)
self.inner.update_info(info)
}
}
impl LocalAudioTrack {
pub fn create_audio_track(
name: &str,
options: AudioCaptureOptions,
source: livekit_webrtc::audio_source::native::NativeAudioSource,
) -> LocalAudioTrack {
let rtc_track = LkRuntime::instance()
.pc_factory()
.create_audio_track(&livekit_webrtc::native::create_random_uuid(), source);
Self::new(name.to_string(), rtc_track, options)
pub fn create_audio_track(name: &str, source: RtcAudioSource) -> LocalAudioTrack {
let rtc_track = match source.clone() {
#[cfg(not(target_arch = "wasm32"))]
RtcAudioSource::Native(native_source) => {
use livekit_webrtc::peer_connection_factory::native::PeerConnectionFactoryExt;
LkRuntime::instance().pc_factory().create_audio_track(
&livekit_webrtc::native::create_random_uuid(),
native_source,
)
}
_ => panic!("unsupported audio source"),
};
Self::new(name.to_string(), rtc_track, source)
}
}
livekit/src/room/track/local_track.rs
+33
View File
@@ -0,0 +1,33 @@
use super::TrackInner;
use super::{track_dispatch, LocalAudioTrack, LocalVideoTrack};
use crate::prelude::*;
use crate::track::TrackEvent;
use livekit_protocol as proto;
use livekit_protocol::enum_dispatch;
use livekit_webrtc::prelude::*;
use std::sync::Arc;
use tokio::sync::mpsc;
#[derive(Clone, Debug)]
pub enum LocalTrack {
Audio(LocalAudioTrack),
Video(LocalVideoTrack),
}
impl LocalTrack {
track_dispatch!([Audio, Video]);
enum_dispatch!(
[Audio, Video];
pub fn mute(self: &Self) -> ();
pub fn unmute(self: &Self) -> ();
);
#[inline]
pub fn rtc_track(&self) -> MediaStreamTrack {
match self {
Self::Audio(track) => track.rtc_track().into(),
Self::Video(track) => track.rtc_track().into(),
}
}
}
livekit/src/room/track/local_video_track.rs
+51 -55
View File
@@ -1,23 +1,16 @@
use super::TrackInner;
use crate::prelude::*;
use crate::rtc_engine::lk_runtime::LkRuntime;
use crate::{options::VideoCaptureOptions, prelude::*};
use livekit_protocol as proto;
use livekit_webrtc::peer_connection_factory::native::PeerConnectionFactoryExt;
use livekit_webrtc::prelude::*;
use parking_lot::Mutex;
use std::fmt::Debug;
use std::sync::Arc;
use tokio::sync::mpsc;
#[derive(Debug)]
struct LocalVideoTrackInner {
track_inner: TrackInner,
capture_options: Mutex<VideoCaptureOptions>,
}
#[derive(Clone)]
pub struct LocalVideoTrack {
inner: Arc<LocalVideoTrackInner>,
inner: Arc<TrackInner>,
source: RtcVideoSource,
}
impl Debug for LocalVideoTrack {
@@ -31,85 +24,84 @@ impl Debug for LocalVideoTrack {
}
impl LocalVideoTrack {
pub fn new(
name: String,
rtc_track: RtcVideoTrack,
capture_options: VideoCaptureOptions,
) -> Self {
pub fn new(name: String, rtc_track: RtcVideoTrack, source: RtcVideoSource) -> Self {
Self {
inner: Arc::new(LocalVideoTrackInner {
track_inner: TrackInner::new(
"unknown".to_string().into(), // sid
name,
TrackKind::Video,
MediaStreamTrack::Video(rtc_track),
),
capture_options: Mutex::new(capture_options),
}),
inner: Arc::new(TrackInner::new(
"unknown".to_string().into(), // sid
name,
TrackKind::Video,
MediaStreamTrack::Video(rtc_track),
)),
source,
}
}
#[inline]
pub fn capture_options(&self) -> VideoCaptureOptions {
self.inner.capture_options.lock().clone()
}
#[inline]
pub fn sid(&self) -> TrackSid {
self.inner.track_inner.sid()
self.inner.sid()
}
#[inline]
pub fn name(&self) -> String {
self.inner.track_inner.name()
self.inner.name()
}
#[inline]
pub fn kind(&self) -> TrackKind {
self.inner.track_inner.kind()
self.inner.kind()
}
#[inline]
pub fn source(&self) -> TrackSource {
self.inner.track_inner.source()
self.inner.source()
}
#[inline]
pub fn stream_state(&self) -> StreamState {
self.inner.track_inner.stream_state()
self.inner.stream_state()
}
#[inline]
pub fn start(&self) {
self.inner.track_inner.start()
pub fn enable(&self) {
self.inner.enable()
}
#[inline]
pub fn stop(&self) {
self.inner.track_inner.stop()
pub fn disable(&self) {
self.inner.disable()
}
#[inline]
pub fn is_muted(&self) -> bool {
self.inner.track_inner.is_muted()
self.inner.is_muted()
}
#[inline]
pub fn set_muted(&self, muted: bool) {
self.inner.track_inner.set_muted(muted)
pub fn mute(&self) {
self.inner.set_muted(true);
}
#[inline]
pub fn unmute(&self) {
self.inner.set_muted(false);
}
#[inline]
pub fn rtc_track(&self) -> RtcVideoTrack {
if let MediaStreamTrack::Video(video) = self.inner.track_inner.rtc_track() {
if let MediaStreamTrack::Video(video) = self.inner.rtc_track() {
return video;
}
unreachable!()
}
#[inline]
pub fn rtc_source(&self) -> RtcVideoSource {
self.source.clone()
}
#[inline]
pub fn register_observer(&self) -> mpsc::UnboundedReceiver<TrackEvent> {
self.inner.track_inner.register_observer()
self.inner.register_observer()
}
#[inline]
@@ -119,30 +111,34 @@ impl LocalVideoTrack {
#[inline]
pub(crate) fn transceiver(&self) -> Option<RtpTransceiver> {
self.inner.track_inner.transceiver()
self.inner.transceiver()
}
#[inline]
pub(crate) fn update_transceiver(&self, transceiver: Option<RtpTransceiver>) {
self.inner.track_inner.update_transceiver(transceiver)
self.inner.update_transceiver(transceiver)
}
#[inline]
pub(crate) fn update_info(&self, info: proto::TrackInfo) {
self.inner.track_inner.update_info(info)
self.inner.update_info(info)
}
}
impl LocalVideoTrack {
pub fn create_video_track(
name: &str,
options: VideoCaptureOptions,
source: livekit_webrtc::video_source::native::NativeVideoSource,
) -> LocalVideoTrack {
let rtc_track = LkRuntime::instance()
.pc_factory()
.create_video_track(&livekit_webrtc::native::create_random_uuid(), source);
pub fn create_video_track(name: &str, source: RtcVideoSource) -> LocalVideoTrack {
let rtc_track = match source.clone() {
#[cfg(not(target_arch = "wasm32"))]
RtcVideoSource::Native(native_source) => {
use livekit_webrtc::peer_connection_factory::native::PeerConnectionFactoryExt;
LkRuntime::instance().pc_factory().create_video_track(
&livekit_webrtc::native::create_random_uuid(),
native_source,
)
}
_ => panic!("unsupported video source"),
};
Self::new(name.to_string(), rtc_track, options)
Self::new(name.to_string(), rtc_track, source)
}
}
livekit/src/room/track/mod.rs
+72 -144
View File
@@ -3,19 +3,22 @@ use livekit_protocol as proto;
use livekit_protocol::enum_dispatch;
use livekit_protocol::observer::Dispatcher;
use livekit_webrtc::prelude::*;
use parking_lot::Mutex;
use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
use parking_lot::RwLock;
use thiserror::Error;
use tokio::sync::mpsc;
mod local_audio_track;
mod local_track;
mod local_video_track;
mod remote_audio_track;
mod remote_track;
mod remote_video_track;
pub use local_audio_track::*;
pub use local_track::*;
pub use local_video_track::*;
pub use remote_audio_track::*;
pub use remote_track::*;
pub use remote_video_track::*;
#[derive(Error, Debug, Clone)]
@@ -45,10 +48,10 @@ pub enum TrackSource {
ScreenshareAudio,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(Debug, Clone)]
pub enum TrackEvent {
Mute,
Unmute,
Muted,
Unmuted,
}
#[derive(Clone, Copy, Debug)]
@@ -62,18 +65,6 @@ pub enum Track {
RemoteVideo(RemoteVideoTrack),
}
#[derive(Clone, Debug)]
pub enum LocalTrack {
Audio(LocalAudioTrack),
Video(LocalVideoTrack),
}
#[derive(Clone, Debug)]
pub enum RemoteTrack {
Audio(RemoteAudioTrack),
Video(RemoteVideoTrack),
}
#[derive(Clone, Debug)]
pub enum VideoTrack {
Local(LocalVideoTrack),
@@ -95,12 +86,11 @@ macro_rules! track_dispatch {
pub fn kind(self: &Self) -> TrackKind;
pub fn source(self: &Self) -> TrackSource;
pub fn stream_state(self: &Self) -> StreamState;
pub fn start(self: &Self) -> ();
pub fn stop(self: &Self) -> ();
pub fn enable(self: &Self) -> ();
pub fn disable(self: &Self) -> ();
pub fn is_muted(self: &Self) -> bool;
pub fn set_muted(self: &Self, muted: bool) -> ();
pub fn register_observer(self: &Self) -> mpsc::UnboundedReceiver<TrackEvent>;
pub fn is_remote(self: &Self) -> bool;
pub fn register_observer(self: &Self) -> mpsc::UnboundedReceiver<TrackEvent>;
pub(crate) fn transceiver(self: &Self) -> Option<RtpTransceiver>;
pub(crate) fn update_transceiver(self: &Self, transceiver: Option<RtpTransceiver>) -> ();
@@ -109,6 +99,8 @@ macro_rules! track_dispatch {
};
}
pub(crate) use track_dispatch;
impl Track {
track_dispatch!([LocalAudio, LocalVideo, RemoteAudio, RemoteVideo]);
@@ -123,30 +115,6 @@ impl Track {
}
}
impl LocalTrack {
track_dispatch!([Audio, Video]);
#[inline]
pub fn rtc_track(&self) -> MediaStreamTrack {
match self {
Self::Audio(track) => track.rtc_track().into(),
Self::Video(track) => track.rtc_track().into(),
}
}
}
impl RemoteTrack {
track_dispatch!([Audio, Video]);
#[inline]
pub fn rtc_track(&self) -> MediaStreamTrack {
match self {
Self::Audio(track) => track.rtc_track().into(),
Self::Video(track) => track.rtc_track().into(),
}
}
}
impl VideoTrack {
track_dispatch!([Local, Remote]);
@@ -171,90 +139,73 @@ impl AudioTrack {
}
}
#[derive(Debug)]
struct TrackInfo {
sid: TrackSid,
name: String,
kind: TrackKind,
source: TrackSource,
stream_state: StreamState,
muted: bool,
transceiver: Option<RtpTransceiver>,
}
#[derive(Debug)]
pub(crate) struct TrackInner {
pub sid: Mutex<TrackSid>,
pub name: Mutex<String>,
pub kind: AtomicU8, // TrackKind
pub source: AtomicU8, // TrackSource
pub stream_state: AtomicU8, // StreamState
pub muted: AtomicBool,
pub rtc_track: MediaStreamTrack,
pub transceiver: Mutex<Option<RtpTransceiver>>,
pub dispatcher: Dispatcher<TrackEvent>,
info: RwLock<TrackInfo>,
rtc_track: MediaStreamTrack,
dispatcher: Dispatcher<TrackEvent>,
}
impl TrackInner {
pub fn new(sid: TrackSid, name: String, kind: TrackKind, rtc_track: MediaStreamTrack) -> Self {
Self {
sid: Mutex::new(sid),
name: Mutex::new(name),
kind: AtomicU8::new(kind as u8),
source: AtomicU8::new(TrackSource::Unknown as u8),
stream_state: AtomicU8::new(StreamState::Active as u8),
muted: AtomicBool::new(false),
info: RwLock::new(TrackInfo {
sid,
name,
kind,
source: TrackSource::Unknown,
stream_state: StreamState::Active,
muted: false,
transceiver: None,
}),
rtc_track,
transceiver: Default::default(),
dispatcher: Default::default(),
}
}
pub fn sid(&self) -> TrackSid {
self.sid.lock().clone()
self.info.read().sid.clone()
}
pub fn name(&self) -> String {
self.name.lock().clone()
self.info.read().name.clone()
}
pub fn kind(&self) -> TrackKind {
self.kind.load(Ordering::SeqCst).try_into().unwrap()
self.info.read().kind
}
pub fn source(&self) -> TrackSource {
self.source.load(Ordering::SeqCst).into()
self.info.read().source
}
pub fn stream_state(&self) -> StreamState {
self.stream_state.load(Ordering::SeqCst).try_into().unwrap()
self.info.read().stream_state
}
pub fn is_muted(&self) -> bool {
self.muted.load(Ordering::SeqCst)
self.info.read().muted
}
pub fn start(&self) {
pub fn enable(&self) {
self.rtc_track.set_enabled(true);
}
pub fn stop(&self) {
pub fn disable(&self) {
self.rtc_track.set_enabled(false);
}
pub fn set_muted(&self, muted: bool) {
if self
.muted
.compare_exchange(!muted, muted, Ordering::SeqCst, Ordering::SeqCst)
.is_err()
{
return;
}
if !muted {
self.start();
} else {
self.stop();
}
let event = if muted {
TrackEvent::Mute
} else {
TrackEvent::Unmute
};
self.dispatcher.dispatch(&event);
}
pub fn rtc_track(&self) -> MediaStreamTrack {
self.rtc_track.clone()
}
@@ -264,24 +215,39 @@ impl TrackInner {
}
pub fn transceiver(&self) -> Option<RtpTransceiver> {
self.transceiver.lock().clone()
self.info.read().transceiver.clone()
}
pub fn update_transceiver(&self, transceiver: Option<RtpTransceiver>) {
*self.transceiver.lock() = transceiver;
self.info.write().transceiver = transceiver;
}
pub fn update_info(&self, info: proto::TrackInfo) {
*self.name.lock() = info.name;
*self.sid.lock() = info.sid.into();
self.kind.store(
TrackKind::try_from(proto::TrackType::from_i32(info.r#type).unwrap()).unwrap() as u8,
Ordering::SeqCst,
);
self.source.store(
TrackSource::from(proto::TrackSource::from_i32(info.source).unwrap()) as u8,
Ordering::SeqCst,
);
pub fn set_muted(&self, muted: bool) {
log::debug!("set_muted: {} {}", self.sid(), muted);
if self.is_muted() == muted {
return;
}
if muted {
self.disable();
} else {
self.enable();
}
self.dispatcher.dispatch(if muted {
&TrackEvent::Muted
} else {
&TrackEvent::Unmuted
});
}
pub fn update_info(&self, new_info: proto::TrackInfo) {
let mut info = self.info.write();
info.name = new_info.name;
info.sid = new_info.sid.into();
info.kind =
TrackKind::try_from(proto::TrackType::from_i32(new_info.r#type).unwrap()).unwrap();
info.source = TrackSource::from(proto::TrackSource::from_i32(new_info.source).unwrap());
// Muted and StreamState are not handled separately (events)
}
}
@@ -370,44 +336,6 @@ impl TryFrom<Track> for AudioTrack {
}
}
// Conversions from integers (Useful since we're using atomic values to represent our enums)
impl TryFrom<u8> for TrackKind {
type Error = &'static str;
fn try_from(kind: u8) -> Result<Self, Self::Error> {
match kind {
0 => Ok(Self::Audio),
1 => Ok(Self::Video),
_ => Err("invalid track kind"),
}
}
}
impl TryFrom<u8> for StreamState {
type Error = &'static str;
fn try_from(state: u8) -> Result<Self, Self::Error> {
match state {
0 => Ok(Self::Active),
1 => Ok(Self::Paused),
_ => Err("invalid stream state"),
}
}
}
impl From<u8> for TrackSource {
fn from(source: u8) -> Self {
match source {
1 => Self::Camera,
2 => Self::Microphone,
3 => Self::Screenshare,
4 => Self::ScreenshareAudio,
_ => Self::Unknown,
}
}
}
impl From<TrackKind> for MediaType {
fn from(kind: TrackKind) -> Self {
match kind {
livekit/src/room/track/remote_audio_track.rs
+6 -10
View File
@@ -1,3 +1,4 @@
use super::remote_track;
use super::TrackInner;
use crate::prelude::*;
use livekit_protocol as proto;
@@ -59,13 +60,13 @@ impl RemoteAudioTrack {
}
#[inline]
pub fn start(&self) {
self.inner.start()
pub fn enable(&self) {
self.inner.enable()
}
#[inline]
pub fn stop(&self) {
self.inner.stop()
pub fn disable(&self) {
self.inner.disable()
}
#[inline]
@@ -73,11 +74,6 @@ impl RemoteAudioTrack {
self.inner.is_muted()
}
#[inline]
pub fn set_muted(&self, muted: bool) {
self.inner.set_muted(muted)
}
#[inline]
pub fn rtc_track(&self) -> RtcAudioTrack {
if let MediaStreamTrack::Audio(audio) = self.inner.rtc_track() {
@@ -110,6 +106,6 @@ impl RemoteAudioTrack {
#[inline]
pub(crate) fn update_info(&self, info: proto::TrackInfo) {
self.inner.update_info(info)
remote_track::update_info(&self.inner, info);
}
}
livekit/src/room/track/remote_track.rs
+33
View File
@@ -0,0 +1,33 @@
use super::track_dispatch;
use super::TrackInner;
use super::{RemoteAudioTrack, RemoteVideoTrack};
use crate::prelude::*;
use crate::track::TrackEvent;
use livekit_protocol as proto;
use livekit_protocol::enum_dispatch;
use livekit_webrtc::prelude::*;
use std::sync::Arc;
use tokio::sync::mpsc;
#[derive(Clone, Debug)]
pub enum RemoteTrack {
Audio(RemoteAudioTrack),
Video(RemoteVideoTrack),
}
impl RemoteTrack {
track_dispatch!([Audio, Video]);
#[inline]
pub fn rtc_track(&self) -> MediaStreamTrack {
match self {
Self::Audio(track) => track.rtc_track().into(),
Self::Video(track) => track.rtc_track().into(),
}
}
}
pub(crate) fn update_info(track: &Arc<TrackInner>, new_info: proto::TrackInfo) {
track.update_info(new_info.clone());
track.set_muted(new_info.muted);
}
livekit/src/room/track/remote_video_track.rs
+6 -11
View File
@@ -1,4 +1,4 @@
use super::TrackInner;
use super::{remote_track, TrackInner};
use crate::prelude::*;
use livekit_protocol as proto;
use livekit_webrtc::prelude::*;
@@ -59,13 +59,13 @@ impl RemoteVideoTrack {
}
#[inline]
pub fn start(&self) {
self.inner.start()
pub fn enable(&self) {
self.inner.enable()
}
#[inline]
pub fn stop(&self) {
self.inner.stop()
pub fn disable(&self) {
self.inner.disable()
}
#[inline]
@@ -73,11 +73,6 @@ impl RemoteVideoTrack {
self.inner.is_muted()
}
#[inline]
pub fn set_muted(&self, muted: bool) {
self.inner.set_muted(muted)
}
#[inline]
pub fn rtc_track(&self) -> RtcVideoTrack {
if let MediaStreamTrack::Video(video) = self.inner.rtc_track() {
@@ -110,6 +105,6 @@ impl RemoteVideoTrack {
#[inline]
pub(crate) fn update_info(&self, info: proto::TrackInfo) {
self.inner.update_info(info);
remote_track::update_info(&self.inner, info);
}
}
livekit/src/rtc_engine/lk_runtime.rs
+2 -3
View File
@@ -3,7 +3,6 @@ use livekit_webrtc::prelude::*;
use parking_lot::Mutex;
use std::fmt::{Debug, Formatter};
use std::sync::{Arc, Weak};
use tracing::trace;
lazy_static! {
static ref LK_RUNTIME: Mutex<Weak<LkRuntime>> = Mutex::new(Weak::new());
@@ -25,7 +24,7 @@ impl LkRuntime {
if let Some(lk_runtime) = lk_runtime_ref.upgrade() {
lk_runtime
} else {
trace!("LkRuntime::new()");
log::trace!("LkRuntime::new()");
let new_runtime = Arc::new(Self {
pc_factory: PeerConnectionFactory::default(),
});
@@ -41,6 +40,6 @@ impl LkRuntime {
impl Drop for LkRuntime {
fn drop(&mut self) {
trace!("LkRuntime::drop()");
log::trace!("LkRuntime::drop()");
}
}
livekit/src/rtc_engine/mod.rs
+153 -153
View File
@@ -1,8 +1,9 @@
use crate::options::TrackPublishOptions;
use crate::prelude::LocalTrack;
use crate::rtc_engine::lk_runtime::LkRuntime;
use crate::rtc_engine::rtc_session::{RtcSession, SessionEvent, SessionEvents, SessionInfo};
use crate::rtc_engine::rtc_session::{RtcSession, SessionEvent, SessionEvents};
use crate::signal_client::{SignalError, SignalOptions};
use crate::DataPacketKind;
use livekit_protocol as proto;
use livekit_webrtc::prelude::*;
use livekit_webrtc::session_description::SdpParseError;
@@ -14,9 +15,9 @@ use std::time::Duration;
use thiserror::Error;
use tokio::sync::RwLock as AsyncRwLock;
use tokio::sync::{mpsc, oneshot};
use tokio::sync::{Mutex as AsyncMutex, Notify};
use tokio::task::JoinHandle;
use tokio::time::{interval, Interval};
use tracing::{error, info, trace, warn};
use tokio::time::{interval, Interval, MissedTickBehavior};
pub mod lk_runtime;
mod peer_transport;
@@ -72,7 +73,7 @@ pub enum EngineEvent {
Data {
participant_sid: String,
payload: Vec<u8>,
kind: proto::data_packet::Kind,
kind: DataPacketKind,
},
SpeakersChanged {
speakers: Vec<proto::SpeakerInfo>,
@@ -90,7 +91,6 @@ pub enum EngineEvent {
pub const RECONNECT_ATTEMPTS: u32 = 10;
pub const RECONNECT_INTERVAL: Duration = Duration::from_secs(5);
///
/// Represents a running RTCSession with the ability to close the session
/// and the engine_task
#[derive(Debug)]
@@ -101,23 +101,30 @@ struct EngineHandle {
}
struct EngineInner {
// Keep a strong reference to LkRuntime to avoid creating a new RtcRuntime or PeerConnection factory accross multiple Rtc sessions
#[allow(dead_code)]
lk_runtime: Arc<LkRuntime>,
session_info: Mutex<Option<SessionInfo>>, // Last/Current Sessioninfo
running_handle: AsyncRwLock<Option<EngineHandle>>,
opened: AtomicBool,
engine_emitter: EngineEmitter,
// Last/current session JoinResponse
// We keep a clone of the join response here because the room needs it
// (directly accessing the running_handle requires an async context to lock the Mutex and a getter needs a short lock)
// Maybe there is a better way to do it?
join_response: Mutex<proto::JoinResponse>,
running_handle: AsyncRwLock<Option<EngineHandle>>,
// Reconnecting fields
closed: AtomicBool, // True if closed or the reconnection failed (Note that this is false when reconnecting or resuming)
reconnecting: AtomicBool,
full_reconnect: AtomicBool,
reconnect_interval: Mutex<Interval>,
full_reconnect: AtomicBool, // If true, the next reconnect attempt will skip resume and directly try a full reconnect
reconnect_interval: AsyncMutex<Interval>,
reconnect_notifier: Arc<Notify>, // Called when the reconnection task finisehd, successful or not
}
impl Debug for EngineInner {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
f.debug_struct("EngineInner")
.field("session_info", &self.session_info)
.field("opened", &self.opened)
.field("closed", &self.closed)
.field("reconnecting", &self.reconnecting)
.field("full_reconnect", &self.full_reconnect)
.finish()
@@ -130,93 +137,68 @@ pub struct RtcEngine {
}
impl RtcEngine {
pub fn new() -> (Self, EngineEvents) {
let (engine_emitter, engine_events) = mpsc::channel(8);
let inner = Arc::new(EngineInner {
lk_runtime: LkRuntime::instance(),
session_info: Default::default(),
running_handle: Default::default(),
opened: Default::default(),
engine_emitter,
reconnecting: Default::default(),
full_reconnect: Default::default(),
reconnect_interval: Mutex::new(interval(RECONNECT_INTERVAL)),
});
(Self { inner }, engine_events)
}
#[tracing::instrument]
pub async fn connect(
&self,
url: &str,
token: &str,
options: SignalOptions,
) -> EngineResult<()> {
self.inner.connect(url, token, options).await
) -> EngineResult<(Self, EngineEvents)> {
let (engine_emitter, engine_events) = mpsc::channel(8);
let mut reconnect_interval = interval(RECONNECT_INTERVAL);
reconnect_interval.set_missed_tick_behavior(MissedTickBehavior::Delay);
let inner = Arc::new(EngineInner {
lk_runtime: LkRuntime::instance(),
running_handle: Default::default(),
engine_emitter,
join_response: Default::default(), // Will directly be replaced by the connect method below
closed: Default::default(),
reconnecting: Default::default(),
full_reconnect: Default::default(),
reconnect_interval: AsyncMutex::new(reconnect_interval),
reconnect_notifier: Arc::new(Notify::new()),
});
inner.connect(url, token, options).await?;
Ok((Self { inner }, engine_events))
}
#[tracing::instrument]
pub async fn close(&self) {
self.inner.close().await
}
#[tracing::instrument(skip(data))]
pub async fn publish_data(
&self,
data: &proto::DataPacket,
kind: proto::data_packet::Kind,
kind: DataPacketKind,
) -> EngineResult<()> {
// Make sure we are connected before trying to send data
self.inner.wait_reconnection().await?;
self.inner
.running_handle
.read()
.await
.as_ref()
.unwrap()
.session
.publish_data(data, kind)
.await
let handle = self.inner.running_handle.read().await;
let session = &handle.as_ref().unwrap().session;
session.publish_data(data, kind).await
}
pub async fn simulate_scenario(&self, scenario: SimulateScenario) -> EngineResult<()> {
self.inner.wait_reconnection().await?;
self.inner
.running_handle
.read()
.await
.as_ref()
.unwrap()
.session
.simulate_scenario(scenario)
.await;
let handle = self.inner.running_handle.read().await;
let session = &handle.as_ref().unwrap().session;
session.simulate_scenario(scenario).await;
Ok(())
}
pub async fn add_track(&self, req: proto::AddTrackRequest) -> EngineResult<proto::TrackInfo> {
self.inner.wait_reconnection().await?;
self.inner
.running_handle
.read()
.await
.as_ref()
.unwrap()
.session
.add_track(req)
.await
let handle = self.inner.running_handle.read().await;
let session = &handle.as_ref().unwrap().session;
session.add_track(req).await
}
pub async fn remove_track(&self, sender: RtpSender) -> EngineResult<()> {
self.inner.wait_reconnection().await?;
self.inner
.running_handle
.read()
.await
.as_ref()
.unwrap()
.session
.remove_track(sender)
.await
let handle = self.inner.running_handle.read().await;
let session = &handle.as_ref().unwrap().session;
session.remove_track(sender).await
}
pub async fn create_sender(
@@ -226,37 +208,33 @@ impl RtcEngine {
encodings: Vec<RtpEncodingParameters>,
) -> EngineResult<RtpTransceiver> {
self.inner.wait_reconnection().await?;
self.inner
.running_handle
.read()
.await
.as_ref()
.unwrap()
.session
.create_sender(track, options, encodings)
.await
let handle = self.inner.running_handle.read().await;
let session = &handle.as_ref().unwrap().session;
session.create_sender(track, options, encodings).await
}
pub async fn negotiate_publisher(&self) -> EngineResult<()> {
// TODO(theomonnom): guard for reconnection
self.inner.wait_reconnection().await?;
self.inner
.running_handle
.read()
.await
.as_ref()
.unwrap()
.session
.negotiate_publisher()
.await
let handle = self.inner.running_handle.read().await;
let session = &handle.as_ref().unwrap().session;
session.negotiate_publisher().await
}
pub fn join_response(&self) -> Option<proto::JoinResponse> {
if let Some(info) = self.inner.session_info.lock().as_ref() {
Some(info.join_response.clone())
} else {
None
pub async fn send_request(&self, msg: proto::signal_request::Message) -> EngineResult<()> {
if self.inner.reconnecting.load(Ordering::Acquire) {
// When doing a full reconnect, it is safe to ignore the messages, we don't wait for reconnection here
return Ok(()); // TODO(theomonnom): Maybe we should still return an error instead?
}
let handle = self.inner.running_handle.read().await;
let session = &handle.as_ref().unwrap().session; // Unwrap should be OK here (running_handle is always valid when not reconnecting)
session.signal_client().send(msg).await;
Ok(())
}
pub fn join_response(&self) -> proto::JoinResponse {
self.inner.join_response.lock().clone()
}
}
@@ -271,12 +249,12 @@ impl EngineInner {
res = session_events.recv() => {
if let Some(event) = res {
if let Err(err) = self.on_session_event(event).await {
error!("failed to handle session event: {:?}", err);
log::error!("failed to handle session event: {:?}", err);
}
}
},
_ = &mut close_receiver => {
trace!("closing engine task");
log::trace!("closing engine task");
break;
}
}
@@ -292,12 +270,12 @@ impl EngineInner {
retry_now,
full_reconnect,
} => {
info!("received session close: {}, {:?}", source, reason);
log::info!("received session close: {}, {:?}", source, reason);
if can_reconnect {
self.clone().try_reconnect(retry_now, full_reconnect);
self.try_reconnect(retry_now, full_reconnect);
} else {
// Spawning a new task because the close function wait for the engine_task to
// finish.
// finish. (Where this function is called from)
tokio::spawn({
let inner = self.clone();
async move {
@@ -363,26 +341,23 @@ impl EngineInner {
token: &str,
options: SignalOptions,
) -> EngineResult<()> {
let (session_emitter, session_events) = mpsc::unbounded_channel();
let session = RtcSession::connect(
url,
token,
options,
self.lk_runtime.clone(),
session_emitter,
)
.await?;
let mut running_handle = self.running_handle.write().await;
let (session, join_response, session_events) =
RtcSession::connect(url, token, options).await?;
let (close_sender, close_receiver) = oneshot::channel();
let engine_task = tokio::spawn(self.clone().engine_task(session_events, close_receiver));
*self.session_info.lock() = Some(session.info().clone());
*self.running_handle.write().await = Some(EngineHandle {
let engine_handle = EngineHandle {
session,
engine_task,
close_sender,
});
};
self.opened.store(true, Ordering::SeqCst);
// Always update the join response after a new session is created (first session or full reconnect)
*self.join_response.lock() = join_response;
*running_handle = Some(engine_handle);
Ok(())
}
@@ -395,7 +370,7 @@ impl EngineInner {
}
async fn close(&self) {
self.opened.store(false, Ordering::SeqCst);
self.closed.store(true, Ordering::Release);
self.terminate_session().await;
let _ = self.engine_emitter.send(EngineEvent::Disconnected).await;
}
@@ -403,14 +378,18 @@ impl EngineInner {
// Wait for the reconnection task to finish
// Return directly if no open RTCSession
async fn wait_reconnection(&self) -> EngineResult<()> {
if !self.opened.load(Ordering::SeqCst) {
Err(EngineError::Connection("not opened".to_owned()))?
if self.closed.load(Ordering::SeqCst) {
Err(EngineError::Connection("engine is closed".to_owned()))?
}
while self.reconnecting.load(Ordering::Acquire) {
tokio::task::yield_now().await; // TODO(theomonnom): Remove yield
if self.reconnecting.load(Ordering::Acquire) {
// If currently reconnecting, wait for the reconnect task to finish
self.reconnect_notifier.notified().await;
}
// reconnect_task is finished here, so it is fine to try to read the RwLock here (should be a short lock)
// (the reconnection logic can lock the running_handle for a long time, e.g when resuming)
if self.running_handle.read().await.is_none() {
Err(EngineError::Connection("reconnection failed".to_owned()))?
}
@@ -419,45 +398,64 @@ impl EngineInner {
}
/// Start the reconnect task if not already started
fn try_reconnect(self: Arc<Self>, retry_now: bool, full_reconnect: bool) {
if !self.opened.load(Ordering::SeqCst) {
/// Ask to retry directly if `retry_now` is true
/// Ask for a full reconnect if `full_reconnect` is true
fn try_reconnect(self: &Arc<Self>, retry_now: bool, full_reconnect: bool) {
if self.closed.load(Ordering::Acquire) {
return;
}
if self.reconnecting.load(Ordering::SeqCst) {
let inner = self.clone();
if retry_now {
self.reconnect_interval.lock().reset();
self.full_reconnect.store(full_reconnect, Ordering::SeqCst);
tokio::spawn(async move {
inner.reconnect_interval.lock().await.reset(); // Retry directly
});
self.full_reconnect.store(full_reconnect, Ordering::Release);
}
return;
}
warn!("reconnecting RTCEngine...");
log::warn!("reconnecting RTCEngine...");
self.reconnecting.store(true, Ordering::SeqCst);
self.full_reconnect.store(full_reconnect, Ordering::SeqCst);
self.reconnect_interval.lock().reset();
tokio::spawn({
let inner = self.clone();
async move {
let res = inner.reconnect_task().await;
inner.reconnecting.store(false, Ordering::SeqCst);
// Reconnetion logic
inner.reconnect_interval.lock().await.reset(); // Retry directly
inner.reconnecting.store(true, Ordering::Release);
inner
.full_reconnect
.store(full_reconnect, Ordering::Release);
let res = inner.reconnect_task().await; // Wait for the reconnection task to finish
inner.reconnecting.store(false, Ordering::Release);
if res.is_ok() {
warn!("RTCEngine successfully reconnected")
log::warn!("RTCEngine successfully reconnected")
} else {
error!("failed to reconnect after {} attemps", RECONNECT_ATTEMPTS);
log::error!("failed to reconnect after {} attemps", RECONNECT_ATTEMPTS);
inner.close().await;
}
inner.reconnect_notifier.notify_waiters();
}
});
}
/// Called every time the PeerConnection or the SignalClient is closed
/// Runned every time the PeerConnection or the SignalClient is closed
/// We first try to resume the connection, if it fails, we start a full reconnect.
async fn reconnect_task(self: &Arc<Self>) -> EngineResult<()> {
// Get the latest connection info from the signal_client (including the refreshed token because the initial join token may have expired)
let running_handle = self.running_handle.read().await;
let signal_client = running_handle.as_ref().unwrap().session.signal_client();
let url = signal_client.url();
let token = signal_client.token();
let options = signal_client.options();
drop(running_handle);
for i in 0..RECONNECT_ATTEMPTS {
if !self.opened.load(Ordering::Acquire) {
if self.closed.load(Ordering::Acquire) {
// The user closed the RTCEngine, cancel the reconnection task
return Ok(());
}
@@ -467,9 +465,12 @@ impl EngineInner {
let _ = self.engine_emitter.send(EngineEvent::Restarting).await;
}
info!("restarting connection... attempt: {}", i);
if let Err(err) = self.try_restart_connection().await {
error!("restarting connection failed: {}", err);
log::info!("restarting connection... attempt: {}", i);
if let Err(err) = self
.try_restart_connection(&url, &token, options.clone())
.await
{
log::error!("restarting connection failed: {}", err);
} else {
let _ = self.engine_emitter.send(EngineEvent::Restarted).await;
return Ok(());
@@ -479,9 +480,9 @@ impl EngineInner {
let _ = self.engine_emitter.send(EngineEvent::Resuming).await;
}
info!("resuming connection... attempt: {}", i);
log::info!("resuming connection... attempt: {}", i);
if let Err(err) = self.try_resume_connection().await {
error!("resuming connection failed: {}", err);
log::error!("resuming connection failed: {}", err);
if let EngineError::Signal(_) = err {
self.full_reconnect.store(true, Ordering::SeqCst);
}
@@ -491,34 +492,33 @@ impl EngineInner {
}
}
self.reconnect_interval.lock().tick().await;
self.reconnect_interval.lock().await.tick().await;
}
Err(EngineError::Connection("failed to reconnect".to_owned()))
}
/// Try to recover the connection by doing a full reconnect.
/// It recreates a new RTCSession
async fn try_restart_connection(self: &Arc<Self>) -> EngineResult<()> {
let info = self.session_info.lock().clone().unwrap();
/// It recreates a new RtcSession
async fn try_restart_connection(
self: &Arc<Self>,
url: &str,
token: &str,
options: SignalOptions,
) -> EngineResult<()> {
self.terminate_session().await;
self.connect(&info.url, &info.token, info.options).await?;
self.running_handle
.read()
.await
.as_ref()
.unwrap()
.session
.wait_pc_connection()
.await
self.connect(url, token, options).await?;
// TODO(theomonnom): Resend SignalClient queue
let handle = self.running_handle.read().await;
let session = &handle.as_ref().unwrap().session;
session.wait_pc_connection().await
}
/// Try to restart the current session
async fn try_resume_connection(&self) -> EngineResult<()> {
let handle = self.running_handle.read().await;
handle.as_ref().unwrap().session.restart().await?;
handle.as_ref().unwrap().session.wait_pc_connection().await
let session = &handle.as_ref().unwrap().session;
session.restart().await?;
session.wait_pc_connection().await
}
}
livekit/src/rtc_engine/peer_transport.rs
+3 -11
View File
@@ -1,8 +1,8 @@
use livekit_protocol as proto;
use livekit_webrtc::prelude::*;
use log::{debug, error};
use std::fmt::{Debug, Formatter};
use std::time::Duration;
use tracing::{event, Level};
const _NEGOTIATION_FREQUENCY: Duration = Duration::from_millis(150);
@@ -64,7 +64,6 @@ impl PeerTransport {
self.peer_connection.close();
}
#[tracing::instrument(level = Level::DEBUG)]
pub async fn add_ice_candidate(&mut self, ice_candidate: IceCandidate) -> Result<(), RtcError> {
if self.peer_connection.current_remote_description().is_some() && !self.restarting_ice {
self.peer_connection
@@ -78,7 +77,6 @@ impl PeerTransport {
Ok(())
}
#[tracing::instrument(level = Level::DEBUG)]
pub async fn set_remote_description(
&mut self,
remote_description: SessionDescription,
@@ -100,13 +98,11 @@ impl PeerTransport {
Ok(())
}
#[tracing::instrument(level = Level::DEBUG)]
pub async fn negotiate(&mut self) -> Result<(), RtcError> {
// TODO(theomonnom) Debounce here with NEGOTIATION_FREQUENCY
self.create_and_send_offer(OfferOptions::default()).await
}
#[tracing::instrument(level = Level::DEBUG)]
pub async fn create_anwser(
&mut self,
offer: SessionDescription,
@@ -121,14 +117,13 @@ impl PeerTransport {
Ok(answer)
}
#[tracing::instrument(level = Level::DEBUG)]
pub async fn create_and_send_offer(&mut self, options: OfferOptions) -> Result<(), RtcError> {
if self.on_offer_handler.is_none() {
return Ok(());
}
if options.ice_restart {
event!(Level::TRACE, "restarting ICE");
debug!("restarting ICE");
self.restarting_ice = true;
}
@@ -140,10 +135,7 @@ impl PeerTransport {
.set_remote_description(remote_description)
.await?;
} else {
event!(
Level::ERROR,
"trying to restart ICE when the pc doesn't have remote description"
);
error!("trying to restart ICE when the pc doesn't have remote description");
}
} else {
self.renegotiate = true;
livekit/src/rtc_engine/rtc_events.rs
+1 -1
View File
@@ -2,8 +2,8 @@ use super::peer_transport::PeerTransport;
use crate::rtc_engine::peer_transport::OnOfferCreated;
use livekit_protocol as proto;
use livekit_webrtc::{self as rtc, prelude::*};
use log::error;
use tokio::sync::mpsc;
use tracing::{error};
pub type RtcEmitter = mpsc::UnboundedSender<RtcEvent>;
pub type RtcEvents = mpsc::UnboundedReceiver<RtcEvent>;
livekit/src/rtc_engine/rtc_session.rs
+43 -90
View File
@@ -4,9 +4,9 @@ use crate::prelude::TrackKind;
use crate::rtc_engine::lk_runtime::LkRuntime;
use crate::rtc_engine::peer_transport::PeerTransport;
use crate::rtc_engine::rtc_events::{RtcEvent, RtcEvents};
use crate::signal_client;
use crate::signal_client::{SignalClient, SignalEvent, SignalEvents, SignalOptions};
use crate::track::LocalTrack;
use crate::DataPacketKind;
use livekit_protocol as proto;
use livekit_webrtc::prelude::*;
use parking_lot::Mutex;
@@ -22,7 +22,6 @@ use tokio::sync::Mutex as AsyncMutex;
use tokio::sync::{mpsc, oneshot, watch};
use tokio::task::JoinHandle;
use tokio::time::sleep;
use tracing::{debug, error, trace, warn};
pub const ICE_CONNECT_TIMEOUT: Duration = Duration::from_secs(15);
pub const TRACK_PUBLISH_TIMEOUT: Duration = Duration::from_secs(10);
@@ -40,7 +39,7 @@ pub enum SessionEvent {
Data {
participant_sid: String,
payload: Vec<u8>,
kind: proto::data_packet::Kind,
kind: DataPacketKind,
},
MediaTrack {
track: MediaStreamTrack,
@@ -96,19 +95,10 @@ struct IceCandidateJson {
pub candidate: String,
}
#[derive(Debug, Clone, Default)]
pub struct SessionInfo {
pub url: String,
pub token: String,
pub options: SignalOptions,
pub join_response: proto::JoinResponse,
}
/// Fields shared with rtc_task and signal_task
struct SessionInner {
info: SessionInfo,
signal_client: Arc<SignalClient>,
pc_state: AtomicU8, // PCState
pc_state: AtomicU8, // PcState
has_published: AtomicBool,
publisher_pc: AsyncMutex<PeerTransport>,
@@ -117,7 +107,7 @@ struct SessionInner {
pending_tracks: Mutex<HashMap<String, oneshot::Sender<proto::TrackInfo>>>,
// Publisher data channels
// used to send data to other participants ( The SFU forwards the messages )
// used to send data to other participants (The SFU forwards the messages)
lossy_dc: DataChannel,
reliable_dc: DataChannel,
@@ -132,7 +122,6 @@ struct SessionInner {
impl Debug for SessionInner {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("SessionInner")
.field("info", &self.info)
.field("pc_state", &self.pc_state)
.field("has_published", &self.has_published)
.field("closed", &self.closed)
@@ -146,8 +135,6 @@ impl Debug for SessionInner {
/// RTCSession is also responsable for the signaling and the negotation
#[derive(Debug)]
pub struct RtcSession {
#[allow(dead_code)]
lk_runtime: Arc<LkRuntime>,
inner: Arc<SessionInner>,
close_tx: watch::Sender<bool>, // false = is_running
signal_task: JoinHandle<()>,
@@ -159,15 +146,13 @@ impl RtcSession {
url: &str,
token: &str,
options: SignalOptions,
lk_runtime: Arc<LkRuntime>,
session_emitter: SessionEmitter,
) -> EngineResult<Self> {
// Connect to the SignalClient
let (signal_client, mut signal_events) = SignalClient::new();
) -> EngineResult<(Self, proto::JoinResponse, SessionEvents)> {
let (session_emitter, session_events) = mpsc::unbounded_channel();
let (signal_client, join_response, signal_events) =
SignalClient::connect(url, token, options).await?;
let signal_client = Arc::new(signal_client);
signal_client.connect(url, token, options.clone()).await?;
let join_response = signal_client::utils::next_join_response(&mut signal_events).await?;
debug!("received JoinResponse: {:?}", join_response);
log::debug!("received JoinResponse: {:?}", join_response);
let (rtc_emitter, rtc_events) = mpsc::unbounded_channel();
let rtc_config = RtcConfiguration {
@@ -186,6 +171,7 @@ impl RtcSession {
ice_transport_type: IceTransportsType::All,
};
let lk_runtime = LkRuntime::instance();
let mut publisher_pc = PeerTransport::new(
lk_runtime
.pc_factory()
@@ -223,16 +209,8 @@ impl RtcSession {
rtc_events::forward_dc_events(&mut lossy_dc, rtc_emitter.clone());
rtc_events::forward_dc_events(&mut reliable_dc, rtc_emitter.clone());
let session_info = SessionInfo {
url: url.to_owned(),
token: token.to_owned(),
options,
join_response,
};
let (close_tx, close_rx) = watch::channel(false);
let inner = Arc::new(SessionInner {
info: session_info,
pc_state: AtomicU8::new(PeerState::New as u8),
has_published: Default::default(),
signal_client,
@@ -250,19 +228,14 @@ impl RtcSession {
let signal_task = tokio::spawn(inner.clone().signal_task(signal_events, close_rx.clone()));
let rtc_task = tokio::spawn(inner.clone().rtc_session_task(rtc_events, close_rx.clone()));
if !inner.info.join_response.subscriber_primary {
inner.negotiate_publisher().await?;
}
let session = Self {
lk_runtime,
inner: inner.clone(),
close_tx,
signal_task,
rtc_task,
};
Ok(session)
Ok((session, join_response, session_events))
}
#[inline]
@@ -291,7 +264,6 @@ impl RtcSession {
}
/// Close the PeerConnections and the SignalClient
#[tracing::instrument]
pub async fn close(self) {
// Close the tasks
self.inner.close().await;
@@ -304,7 +276,7 @@ impl RtcSession {
pub async fn publish_data(
&self,
data: &proto::DataPacket,
kind: proto::data_packet::Kind,
kind: DataPacketKind,
) -> Result<(), EngineError> {
self.inner.publish_data(data, kind).await
}
@@ -324,11 +296,6 @@ impl RtcSession {
self.inner.simulate_scenario(scenario).await
}
#[inline]
pub fn info(&self) -> &SessionInfo {
&self.inner.info
}
#[allow(dead_code)]
#[inline]
pub fn state(&self) -> PeerState {
@@ -359,7 +326,7 @@ impl RtcSession {
#[allow(dead_code)]
#[inline]
pub fn data_channel(&self, kind: proto::data_packet::Kind) -> &DataChannel {
pub fn data_channel(&self, kind: DataPacketKind) -> &DataChannel {
&self.inner.data_channel(kind)
}
}
@@ -375,11 +342,11 @@ impl SessionInner {
res = rtc_events.recv() => {
if let Some(event) = res {
if let Err(err) = self.on_rtc_event(event).await {
error!("failed to handle rtc event: {:?}", err);
log::error!("failed to handle rtc event: {:?}", err);
}
} },
_ = close_rx.changed() => {
trace!("closing rtc_session_task");
log::trace!("closing rtc_session_task");
break;
}
}
@@ -399,7 +366,7 @@ impl SessionInner {
SignalEvent::Open => {}
SignalEvent::Signal(signal) => {
if let Err(err) = self.on_signal_event(signal).await {
error!("failed to handle signal: {:?}", err);
log::error!("failed to handle signal: {:?}", err);
}
}
SignalEvent::Close => {
@@ -415,7 +382,7 @@ impl SessionInner {
}
},
_ = close_rx.changed() => {
trace!("closing signal_task");
log::trace!("closing signal_task");
break;
}
}
@@ -425,7 +392,7 @@ impl SessionInner {
async fn on_signal_event(&self, event: proto::signal_response::Message) -> EngineResult<()> {
match event {
proto::signal_response::Message::Answer(answer) => {
trace!("received publisher answer: {:?}", answer);
log::debug!("received publisher answer: {:?}", answer);
let answer =
SessionDescription::parse(&answer.sdp, answer.r#type.parse().unwrap())?;
self.publisher_pc
@@ -435,7 +402,7 @@ impl SessionInner {
.await?;
}
proto::signal_response::Message::Offer(offer) => {
trace!("received subscriber offer: {:?}", offer);
log::debug!("received subscriber offer: {:?}", offer);
let offer = SessionDescription::parse(&offer.sdp, offer.r#type.parse().unwrap())?;
let answer = self
.subscriber_pc
@@ -460,7 +427,7 @@ impl SessionInner {
IceCandidate::parse(&json.sdp_mid, json.sdp_m_line_index, &json.candidate)?
};
debug!("received ice_candidate {:?} {:?}", target, ice_candidate);
log::debug!("received ice_candidate {:?} {:?}", target, ice_candidate);
if target == proto::SignalTarget::Publisher {
self.publisher_pc
@@ -478,7 +445,7 @@ impl SessionInner {
}
proto::signal_response::Message::Leave(leave) => {
self.on_session_disconnected(
"received leave",
"server request to leave",
leave.reason(),
leave.can_reconnect,
true,
@@ -533,11 +500,12 @@ impl SessionInner {
.await;
}
RtcEvent::ConnectionChange { state, target } => {
debug!("connection change, {:?} {:?}", state, target);
let is_primary = self.info.join_response.subscriber_primary
&& target == proto::SignalTarget::Subscriber;
log::debug!("connection change, {:?} {:?}", state, target);
if is_primary && state == PeerConnectionState::Connected {
// The subscriber is always the primary peer connection
if target == proto::SignalTarget::Subscriber
&& state == PeerConnectionState::Connected
{
let old_state = self
.pc_state
.swap(PeerState::Connected as u8, Ordering::SeqCst);
@@ -565,7 +533,7 @@ impl SessionInner {
}
RtcEvent::Offer { offer, target: _ } => {
// Send the publisher offer to the server
debug!("sending publisher offer: {:?}", offer);
log::debug!("sending publisher offer: {:?}", offer);
self.signal_client
.send(proto::signal_request::Message::Offer(
proto::SessionDescription {
@@ -589,7 +557,7 @@ impl SessionInner {
receiver,
});
} else {
warn!("Track event with no streams");
log::warn!("Track event with no streams");
}
}
RtcEvent::Data { data, binary } => {
@@ -605,7 +573,9 @@ impl SessionInner {
let _ = self.emitter.send(SessionEvent::Data {
participant_sid: user.participant_sid,
payload: user.payload,
kind: proto::data_packet::Kind::from_i32(data.kind).unwrap(),
kind: proto::data_packet::Kind::from_i32(data.kind)
.unwrap()
.into(),
});
}
proto::data_packet::Value::Speaker(_) => {}
@@ -735,7 +705,6 @@ impl SessionInner {
});
}
#[tracing::instrument]
async fn close(&self) {
self.closed.store(true, Ordering::Release);
self.signal_client.close().await;
@@ -743,7 +712,6 @@ impl SessionInner {
self.subscriber_pc.lock().await.close();
}
#[tracing::instrument]
async fn simulate_scenario(&self, scenario: SimulateScenario) {
match scenario {
SimulateScenario::SignalReconnect => {
@@ -814,11 +782,10 @@ impl SessionInner {
}
}
#[tracing::instrument(skip(data))]
async fn publish_data(
&self,
data: &proto::DataPacket,
kind: proto::data_packet::Kind,
kind: DataPacketKind,
) -> Result<(), EngineError> {
self.ensure_publisher_connected(kind).await?;
self.data_channel(kind)
@@ -827,18 +794,9 @@ impl SessionInner {
}
/// Try to restart the session by doing an ICE Restart (The SignalClient is also restarted)
/// This reconnection if more seemless than the full reconnection implemented in ['RTCEngine']
/// This reconnection if more seemless compared to the full reconnection implemented in ['RTCEngine']
async fn restart_session(&self) -> EngineResult<()> {
self.signal_client.close().await;
let mut options = self.info.options.clone();
options.sid = self.info.join_response.participant.clone().unwrap().sid;
options.reconnect = true;
self.signal_client
.connect(&self.info.url, &self.info.token, options)
.await?;
self.signal_client.restart().await?;
self.subscriber_pc.lock().await.prepare_ice_restart();
if self.has_published.load(Ordering::Acquire) {
@@ -854,11 +812,10 @@ impl SessionInner {
self.wait_pc_connection().await?;
self.signal_client.flush_queue().await;
Ok(())
}
// Wait for PCState to become PCState::Connected
// Wait for PeerState to become PeerState::Connected
// Timeout after ['MAX_ICE_CONNECT_TIMEOUT']
async fn wait_pc_connection(&self) -> EngineResult<()> {
let wait_connected = async move {
@@ -867,7 +824,7 @@ impl SessionInner {
return Err(EngineError::Connection("closed".to_string()));
}
tokio::task::yield_now().await;
tokio::time::sleep(Duration::from_millis(50)).await;
}
Ok(())
@@ -887,18 +844,14 @@ impl SessionInner {
self.has_published.store(true, Ordering::Release);
let res = self.publisher_pc.lock().await.negotiate().await;
if let Err(err) = &res {
error!("failed to negotiate the publisher: {:?}", err);
log::error!("failed to negotiate the publisher: {:?}", err);
}
res.map_err(Into::into)
}
/// Ensure the Publisher PC is connected, if not, start the negotiation
/// This is required when sending data to the server
async fn ensure_publisher_connected(&self, kind: proto::data_packet::Kind) -> EngineResult<()> {
if !self.info.join_response.subscriber_primary {
return Ok(());
}
async fn ensure_publisher_connected(&self, kind: DataPacketKind) -> EngineResult<()> {
if !self.publisher_pc.lock().await.is_connected()
&& self
.publisher_pc
@@ -923,7 +876,7 @@ impl SessionInner {
return Err(EngineError::Connection("closed".to_string()));
}
tokio::task::yield_now().await;
tokio::time::sleep(Duration::from_millis(50)).await;
}
Ok(())
@@ -933,14 +886,14 @@ impl SessionInner {
res = wait_connected => res,
_ = sleep(ICE_CONNECT_TIMEOUT) => {
let err = EngineError::Connection("could not establish publisher connection: timeout".to_string());
error!(error = ?err);
log::error!("{}", err);
Err(err)
}
}
}
fn data_channel(&self, kind: proto::data_packet::Kind) -> &DataChannel {
if kind == proto::data_packet::Kind::Reliable {
fn data_channel(&self, kind: DataPacketKind) -> &DataChannel {
if kind == DataPacketKind::Reliable {
&self.reliable_dc
} else {
&self.lossy_dc
livekit/src/signal_client/mod.rs
+109 -68
View File
@@ -1,13 +1,12 @@
use crate::signal_client::signal_stream::SignalStream;
use livekit_protocol as proto;
use parking_lot::RwLock;
use parking_lot::Mutex;
use std::fmt::Debug;
use std::time::Duration;
use thiserror::Error;
use tokio::sync::mpsc;
use tokio::sync::RwLock as AsyncRwLock;
use tokio_tungstenite::tungstenite::Error as WsError;
use tracing::{instrument, Level};
mod signal_stream;
@@ -16,9 +15,12 @@ pub(crate) type SignalEvents = mpsc::Receiver<SignalEvent>;
pub(crate) type SignalResult<T> = Result<T, SignalError>;
pub const JOIN_RESPONSE_TIMEOUT: Duration = Duration::from_secs(5);
pub const PROTOCOL_VERSION: u32 = 8;
#[derive(Error, Debug)]
pub enum SignalError {
#[error("already connected")]
AlreadyConnected,
#[error("ws failure: {0}")]
WsError(#[from] WsError),
#[error("failed to parse the url")]
@@ -39,8 +41,6 @@ pub enum SignalEvent {
#[derive(Debug, Clone)]
pub struct SignalOptions {
pub(crate) reconnect: bool,
pub(crate) sid: String,
pub auto_subscribe: bool,
pub adaptive_stream: bool,
}
@@ -48,9 +48,7 @@ pub struct SignalOptions {
impl Default for SignalOptions {
fn default() -> Self {
Self {
reconnect: false,
auto_subscribe: true,
sid: "".to_string(),
adaptive_stream: false,
}
}
@@ -58,100 +56,143 @@ impl Default for SignalOptions {
#[derive(Debug)]
pub struct SignalClient {
stream: RwLock<Option<SignalStream>>,
stream: AsyncRwLock<Option<SignalStream>>,
url: String,
token: Mutex<String>, // TODO(theomonnom): Handle token refresh
join_response: proto::JoinResponse,
options: SignalOptions,
emitter: SignalEmitter,
}
impl SignalClient {
pub fn new() -> (Self, SignalEvents) {
let (emitter, events) = mpsc::channel(8);
(
Self {
stream: Default::default(),
emitter,
},
events,
)
}
#[instrument(level = Level::DEBUG, skip(url, token, options))]
pub async fn connect(
&self,
url: &str,
token: &str,
options: SignalOptions,
) -> SignalResult<()> {
let stream = SignalStream::connect(url, token, options, self.emitter.clone()).await?;
*self.stream.write() = Some(stream);
) -> SignalResult<(Self, proto::JoinResponse, SignalEvents)> {
let (emitter, mut events) = mpsc::channel(8);
let lk_url = get_livekit_url(url, token, &options)?;
let new_stream = SignalStream::connect(lk_url, emitter.clone()).await?;
let join_response = get_join_response(&mut events).await?;
Ok((
Self {
stream: AsyncRwLock::new(Some(new_stream)),
url: url.to_string(),
token: Mutex::new(token.to_string()),
join_response: join_response.clone(),
options,
emitter,
},
join_response,
events,
))
}
// Restart is called when trying to resume the room (RtcSession resume)
// TODO(theomonom): Should this be renamed to resume?
pub async fn restart(&self) -> SignalResult<()> {
self.close().await;
let sid = &self.join_response.participant.as_ref().unwrap().sid;
let token = self.token.lock().clone();
let mut lk_url = get_livekit_url(&self.url, &token, &self.options)?;
lk_url
.query_pairs_mut()
.append_pair("reconnect", "1")
.append_pair("sid", sid);
let new_stream = SignalStream::connect(lk_url, self.emitter.clone()).await?;
*self.stream.write().await = Some(new_stream);
Ok(())
}
#[instrument(level = Level::DEBUG)]
pub async fn close(&self) {
if let Some(stream) = self.stream.write().take() {
if let Some(stream) = self.stream.write().await.take() {
stream.close().await;
}
}
#[instrument(level = Level::DEBUG)]
pub async fn send(&self, signal: proto::signal_request::Message) {
if let Some(stream) = self.stream.read().as_ref() {
// TODO: Check if currently reconnecting and queue message
if let Some(stream) = self.stream.read().await.as_ref() {
if stream.send(signal).await.is_ok() {
return;
}
}
// TODO(theomonnom): enqueue message
// TODO(theomonnom): return result?
}
/*#[allow(dead_code)]
#[allow(dead_code)]
pub async fn clear_queue(&self) {
// TODO(theomonnom): impl
}*/
// TODO(theomonnom): Clear the queue
}
#[instrument(level = Level::DEBUG)]
pub async fn flush_queue(&self) {
// TODO(theomonnom): impl
// TODO(theomonnom): Send the queue
}
pub fn join_response(&self) -> proto::JoinResponse {
self.join_response.clone()
}
pub fn options(&self) -> SignalOptions {
self.options.clone()
}
pub fn url(&self) -> String {
self.url.clone()
}
pub fn token(&self) -> String {
self.token.lock().clone()
}
}
pub mod utils {
use crate::signal_client::{SignalError, SignalEvent, SignalResult, JOIN_RESPONSE_TIMEOUT};
use livekit_protocol as proto;
use tokio::time::timeout;
use tokio_tungstenite::tungstenite::Error as WsError;
use tracing::{event, instrument, Level};
fn get_livekit_url(url: &str, token: &str, options: &SignalOptions) -> SignalResult<url::Url> {
let mut lk_url = url::Url::parse(url)?;
lk_url.set_path("/rtc");
lk_url
.query_pairs_mut()
.append_pair("access_token", token)
.append_pair("protocol", PROTOCOL_VERSION.to_string().as_str())
.append_pair(
"auto_subscribe",
if options.auto_subscribe { "1" } else { "0" },
)
.append_pair(
"adaptive_stream",
if options.adaptive_stream { "1" } else { "0" },
);
use super::SignalEvents;
Ok(lk_url)
}
#[instrument(level = Level::DEBUG, skip(receiver))]
pub(crate) async fn next_join_response(
receiver: &mut SignalEvents,
) -> SignalResult<proto::JoinResponse> {
let join = async {
while let Some(event) = receiver.recv().await {
match event {
SignalEvent::Signal(proto::signal_response::Message::Join(join)) => {
return Ok(join)
}
SignalEvent::Close => break,
SignalEvent::Open => continue,
_ => {
event!(
Level::WARN,
"received unexpected message while waiting for JoinResponse: {:?}",
event
);
continue;
}
async fn get_join_response(receiver: &mut SignalEvents) -> SignalResult<proto::JoinResponse> {
let join = async {
while let Some(event) = receiver.recv().await {
match event {
SignalEvent::Signal(proto::signal_response::Message::Join(join)) => {
return Ok(join)
}
SignalEvent::Close => break,
SignalEvent::Open => continue,
_ => {
log::warn!(
"received unexpected message while waiting for JoinResponse: {:?}",
event
);
continue;
}
}
}
Err(WsError::ConnectionClosed)?
};
Err(WsError::ConnectionClosed)?
};
timeout(JOIN_RESPONSE_TIMEOUT, join)
.await
.map_err(|_| SignalError::Timeout("failed to receive JoinResponse".to_string()))?
}
tokio::time::timeout(JOIN_RESPONSE_TIMEOUT, join)
.await
.map_err(|_| SignalError::Timeout("failed to receive JoinResponse".to_string()))?
}
livekit/src/signal_client/signal_stream.rs
+11 -33
View File
@@ -1,4 +1,4 @@
use crate::signal_client::{SignalEmitter, SignalEvent, SignalOptions, SignalResult};
use crate::signal_client::{SignalEmitter, SignalEvent, SignalResult};
use futures_util::stream::{SplitSink, SplitStream};
use futures_util::{SinkExt, StreamExt};
use livekit_protocol as proto;
@@ -10,9 +10,8 @@ use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode;
use tokio_tungstenite::tungstenite::protocol::CloseFrame;
use tokio_tungstenite::tungstenite::Message;
use tokio_tungstenite::{connect_async, MaybeTlsStream, WebSocketStream};
use tracing::{event, Level};
pub const PROTOCOL_VERSION: u32 = 8;
use super::SignalEvents;
type WebSocket = WebSocketStream<MaybeTlsStream<TcpStream>>;
@@ -46,31 +45,10 @@ impl SignalStream {
///
/// SignalStream will never try to reconnect if the connection has been
/// closed.
pub async fn connect(
url: &str,
token: &str,
options: SignalOptions,
emitter: SignalEmitter,
) -> SignalResult<Self> {
let mut lk_url = url::Url::parse(url)?;
lk_url.set_path("/rtc");
lk_url
.query_pairs_mut()
.append_pair("access_token", token)
.append_pair("protocol", PROTOCOL_VERSION.to_string().as_str())
.append_pair("reconnect", if options.reconnect { "1" } else { "0" })
.append_pair("sid", &options.sid)
.append_pair(
"auto_subscribe",
if options.auto_subscribe { "1" } else { "0" },
)
.append_pair(
"adaptive_stream",
if options.adaptive_stream { "1" } else { "0" },
);
pub async fn connect(url: url::Url, emitter: SignalEmitter) -> SignalResult<Self> {
log::info!("connecting to SignalClient: {}", url);
event!(Level::INFO, "connecting to SignalClient: {}", lk_url);
let (ws_stream, _) = connect_async(lk_url).await?;
let (ws_stream, _) = connect_async(url).await?;
let _ = emitter.send(SignalEvent::Open).await;
let (ws_writer, ws_reader) = ws_stream.split();
@@ -128,7 +106,7 @@ impl SignalStream {
signal,
response_chn,
} => {
event!(Level::TRACE, "sending SignalRequest: {:?}", signal);
log::debug!("sending SignalRequest: {:?}", signal);
let data = Message::Binary(
proto::SignalRequest {
@@ -138,7 +116,7 @@ impl SignalStream {
);
if let Err(err) = ws_writer.send(data).await {
event!(Level::ERROR, "failed to send signal: {:?}", err);
log::error!("failed to send signal: {:?}", err);
let _ = response_chn.send(Err(err.into()));
break;
}
@@ -147,7 +125,7 @@ impl SignalStream {
}
InternalMessage::Pong { ping_data } => {
if let Err(err) = ws_writer.send(Message::Pong(ping_data)).await {
event!(Level::ERROR, "failed to send pong message: {:?}", err);
log::error!("failed to send pong message: {:?}", err);
}
}
InternalMessage::Close { close_frame } => {
@@ -180,7 +158,7 @@ impl SignalStream {
.expect("failed to decode SignalResponse");
let msg = res.message.unwrap();
event!(Level::TRACE, "received SignalResponse: {:?}", msg);
log::debug!("received SignalResponse: {:?}", msg);
let _ = emitter.send(SignalEvent::Signal(msg)).await;
}
Ok(Message::Ping(data)) => {
@@ -190,11 +168,11 @@ impl SignalStream {
continue;
}
Ok(Message::Close(close)) => {
event!(Level::DEBUG, "server closed the connection: {:?}", close);
log::debug!("server closed the connection: {:?}", close);
break;
}
_ => {
event!(Level::ERROR, "unhandled websocket message {:?}", msg);
log::error!("unhandled websocket message {:?}", msg);
break;
}
}
webrtc-sys/include/livekit/audio_track.h
+12 -4
View File
@@ -18,6 +18,7 @@
#include <memory>
#include "api/audio_options.h"
#include "livekit/helper.h"
#include "livekit/media_stream_track.h"
#include "livekit/webrtc.h"
@@ -78,7 +79,7 @@ std::shared_ptr<NativeAudioSink> new_native_audio_sink(
class AudioTrackSource {
class InternalSource : public webrtc::LocalAudioSource {
public:
InternalSource();
InternalSource(const cricket::AudioOptions& options);
SourceState state() const override;
bool remote() const override;
@@ -88,6 +89,8 @@ class AudioTrackSource {
void AddSink(webrtc::AudioTrackSinkInterface* sink) override;
void RemoveSink(webrtc::AudioTrackSinkInterface* sink) override;
void set_options(const cricket::AudioOptions& options);
// AudioFrame should always contain 10 ms worth of data (see index.md of
// acm)
void on_captured_frame(rust::Slice<const int16_t> audio_data,
@@ -96,13 +99,17 @@ class AudioTrackSource {
size_t number_of_frames);
private:
webrtc::Mutex mutex_;
mutable webrtc::Mutex mutex_;
std::vector<webrtc::AudioTrackSinkInterface*> sinks_;
cricket::AudioOptions options_{};
};
public:
AudioTrackSource();
AudioTrackSource(AudioSourceOptions options);
AudioSourceOptions audio_options() const;
void set_audio_options(const AudioSourceOptions& options) const;
void on_captured_frame(rust::Slice<const int16_t> audio_data,
int sample_rate,
@@ -115,7 +122,8 @@ class AudioTrackSource {
rtc::scoped_refptr<InternalSource> source_;
};
std::shared_ptr<AudioTrackSource> new_audio_track_source();
std::shared_ptr<AudioTrackSource> new_audio_track_source(
AudioSourceOptions options);
static std::shared_ptr<MediaStreamTrack> audio_to_media(
std::shared_ptr<AudioTrack> track) {
webrtc-sys/include/livekit/video_track.h
+12 -5
View File
@@ -87,23 +87,29 @@ std::shared_ptr<NativeVideoSink> new_native_video_sink(
class VideoTrackSource {
class InternalSource : public rtc::AdaptedVideoTrackSource {
public:
InternalSource();
InternalSource(const VideoResolution&
resolution); // (0, 0) means no resolution/optional, the
// source will guess the resolution at the
// first captured frame
~InternalSource() override;
bool is_screencast() const override;
absl::optional<bool> needs_denoising() const override;
SourceState state() const override;
bool remote() const override;
VideoResolution video_resolution() const;
bool on_captured_frame(const webrtc::VideoFrame& frame);
private:
webrtc::Mutex mutex_;
mutable webrtc::Mutex mutex_;
rtc::TimestampAligner timestamp_aligner_;
VideoResolution resolution_;
};
public:
VideoTrackSource();
VideoTrackSource(const VideoResolution& resolution);
VideoResolution video_resolution() const;
bool on_captured_frame(const std::unique_ptr<VideoFrame>& frame)
const; // frames pushed from Rust (+interior mutability)
@@ -114,7 +120,8 @@ class VideoTrackSource {
rtc::scoped_refptr<InternalSource> source_;
};
std::shared_ptr<VideoTrackSource> new_video_track_source();
std::shared_ptr<VideoTrackSource> new_video_track_source(
const VideoResolution& resolution);
static std::shared_ptr<MediaStreamTrack> video_to_media(
std::shared_ptr<VideoTrack> track) {
webrtc-sys/src/audio_track.cpp
+44 -9
View File
@@ -20,6 +20,7 @@
#include <iostream>
#include <memory>
#include "api/audio_options.h"
#include "api/media_stream_interface.h"
#include "audio/remix_resample.h"
#include "common_audio/include/audio_util.h"
@@ -28,9 +29,28 @@
#include "rtc_base/synchronization/mutex.h"
#include "rtc_base/time_utils.h"
#include "rust/cxx.h"
#include "webrtc-sys/src/audio_track.rs.h"
namespace livekit {
inline cricket::AudioOptions to_native_audio_options(
const AudioSourceOptions& options) {
cricket::AudioOptions rtc_options{};
rtc_options.echo_cancellation = options.echo_cancellation;
rtc_options.noise_suppression = options.noise_suppression;
rtc_options.auto_gain_control = options.auto_gain_control;
return rtc_options;
}
inline AudioSourceOptions to_rust_audio_options(
const cricket::AudioOptions& rtc_options) {
AudioSourceOptions options{};
options.echo_cancellation = rtc_options.echo_cancellation.value_or(false);
options.noise_suppression = rtc_options.noise_suppression.value_or(false);
options.auto_gain_control = rtc_options.auto_gain_control.value_or(false);
return options;
}
AudioTrack::AudioTrack(std::shared_ptr<RtcRuntime> rtc_runtime,
rtc::scoped_refptr<webrtc::AudioTrackInterface> track)
: MediaStreamTrack(rtc_runtime, std::move(track)) {}
@@ -74,11 +94,8 @@ std::shared_ptr<NativeAudioSink> new_native_audio_sink(
return std::make_shared<NativeAudioSink>(std::move(observer));
}
AudioTrackSource::InternalSource::InternalSource() {
options_.echo_cancellation = false;
options_.auto_gain_control = false;
options_.noise_suppression = false;
}
AudioTrackSource::InternalSource::InternalSource(
const cricket::AudioOptions& options) {}
webrtc::MediaSourceInterface::SourceState
AudioTrackSource::InternalSource::state() const {
@@ -90,9 +107,16 @@ bool AudioTrackSource::InternalSource::remote() const {
}
const cricket::AudioOptions AudioTrackSource::InternalSource::options() const {
webrtc::MutexLock lock(&mutex_);
return options_;
}
void AudioTrackSource::InternalSource::set_options(
const cricket::AudioOptions& options) {
webrtc::MutexLock lock(&mutex_);
options_ = options;
}
void AudioTrackSource::InternalSource::AddSink(
webrtc::AudioTrackSinkInterface* sink) {
webrtc::MutexLock lock(&mutex_);
@@ -117,8 +141,18 @@ void AudioTrackSource::InternalSource::on_captured_frame(
}
}
AudioTrackSource::AudioTrackSource() {
source_ = rtc::make_ref_counted<InternalSource>();
AudioTrackSource::AudioTrackSource(AudioSourceOptions options) {
source_ =
rtc::make_ref_counted<InternalSource>(to_native_audio_options(options));
}
AudioSourceOptions AudioTrackSource::audio_options() const {
return to_rust_audio_options(source_->options());
}
void AudioTrackSource::set_audio_options(
const AudioSourceOptions& options) const {
source_->set_options(to_native_audio_options(options));
}
void AudioTrackSource::on_captured_frame(rust::Slice<const int16_t> audio_data,
@@ -134,8 +168,9 @@ rtc::scoped_refptr<AudioTrackSource::InternalSource> AudioTrackSource::get()
return source_;
}
std::shared_ptr<AudioTrackSource> new_audio_track_source() {
return std::make_shared<AudioTrackSource>();
std::shared_ptr<AudioTrackSource> new_audio_track_source(
AudioSourceOptions options) {
return std::make_shared<AudioTrackSource>(options);
}
} // namespace livekit
webrtc-sys/src/audio_track.rs
+10 -1
View File
@@ -3,6 +3,13 @@ use std::sync::Arc;
#[cxx::bridge(namespace = "livekit")]
pub mod ffi {
pub struct AudioSourceOptions {
pub echo_cancellation: bool,
pub noise_suppression: bool,
pub auto_gain_control: bool,
}
extern "C++" {
include!("livekit/media_stream_track.h");
@@ -27,7 +34,9 @@ pub mod ffi {
nb_channels: usize,
nb_frames: usize,
);
fn new_audio_track_source() -> SharedPtr<AudioTrackSource>;
fn audio_options(self: &AudioTrackSource) -> AudioSourceOptions;
fn set_audio_options(self: &AudioTrackSource, options: &AudioSourceOptions);
fn new_audio_track_source(options: AudioSourceOptions) -> SharedPtr<AudioTrackSource>;
fn audio_to_media(track: SharedPtr<AudioTrack>) -> SharedPtr<MediaStreamTrack>;
unsafe fn media_to_audio(track: SharedPtr<MediaStreamTrack>) -> SharedPtr<AudioTrack>;
webrtc-sys/src/video_track.cpp
+23 -9
View File
@@ -31,6 +31,7 @@
#include "rtc_base/ref_counted_object.h"
#include "rtc_base/synchronization/mutex.h"
#include "rtc_base/time_utils.h"
#include "webrtc-sys/src/video_track.rs.h"
namespace livekit {
@@ -103,8 +104,9 @@ std::shared_ptr<NativeVideoSink> new_native_video_sink(
return std::make_shared<NativeVideoSink>(std::move(observer));
}
VideoTrackSource::InternalSource::InternalSource()
: rtc::AdaptedVideoTrackSource(4) {}
VideoTrackSource::InternalSource::InternalSource(
const VideoResolution& resolution)
: rtc::AdaptedVideoTrackSource(4), resolution_(resolution) {}
VideoTrackSource::InternalSource::~InternalSource() {}
@@ -125,6 +127,11 @@ bool VideoTrackSource::InternalSource::remote() const {
return false;
}
VideoResolution VideoTrackSource::InternalSource::video_resolution() const {
webrtc::MutexLock lock(&mutex_);
return resolution_;
}
bool VideoTrackSource::InternalSource::on_captured_frame(
const webrtc::VideoFrame& frame) {
webrtc::MutexLock lock(&mutex_);
@@ -135,6 +142,11 @@ bool VideoTrackSource::InternalSource::on_captured_frame(
rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer =
frame.video_frame_buffer();
if (resolution_.height == 0 || resolution_.width == 0) {
resolution_ = VideoResolution{static_cast<uint32_t>(buffer->width()),
static_cast<uint32_t>(buffer->height())};
}
int adapted_width, adapted_height, crop_width, crop_height, crop_x, crop_y;
if (!AdaptFrame(buffer->width(), buffer->height(), aligned_timestamp_us,
&adapted_width, &adapted_height, &crop_width, &crop_height,
@@ -163,16 +175,17 @@ bool VideoTrackSource::InternalSource::on_captured_frame(
return true;
}
VideoTrackSource::VideoTrackSource() {
source_ = rtc::make_ref_counted<InternalSource>();
VideoTrackSource::VideoTrackSource(const VideoResolution& resolution) {
source_ = rtc::make_ref_counted<InternalSource>(resolution);
}
VideoResolution VideoTrackSource::video_resolution() const {
return source_->video_resolution();
}
bool VideoTrackSource::on_captured_frame(
const std::unique_ptr<VideoFrame>& frame) const {
auto rtc_frame = frame->get();
rtc_frame.set_timestamp_us(
rtc::TimeMicros()); // TODO(theomonnom): Expore capture ts to Rust
return source_->on_captured_frame(rtc_frame);
}
@@ -181,8 +194,9 @@ rtc::scoped_refptr<VideoTrackSource::InternalSource> VideoTrackSource::get()
return source_;
}
std::shared_ptr<VideoTrackSource> new_video_track_source() {
return std::make_shared<VideoTrackSource>();
std::shared_ptr<VideoTrackSource> new_video_track_source(
const VideoResolution& resolution) {
return std::make_shared<VideoTrackSource>(resolution);
}
} // namespace livekit
webrtc-sys/src/video_track.rs
+8 -2
View File
@@ -21,6 +21,12 @@ pub mod ffi {
pub max_fps: f64,
}
#[derive(Debug)]
pub struct VideoResolution {
pub width: u32,
pub height: u32,
}
extern "C++" {
include!("livekit/video_frame.h");
include!("livekit/media_stream_track.h");
@@ -44,9 +50,9 @@ pub mod ffi {
fn set_content_hint(self: &VideoTrack, hint: ContentHint);
fn new_native_video_sink(observer: Box<VideoSinkWrapper>) -> SharedPtr<NativeVideoSink>;
fn video_resolution(self: &VideoTrackSource) -> VideoResolution;
fn on_captured_frame(self: &VideoTrackSource, frame: &UniquePtr<VideoFrame>) -> bool;
fn new_video_track_source() -> SharedPtr<VideoTrackSource>;
fn new_video_track_source(resolution: &VideoResolution) -> SharedPtr<VideoTrackSource>;
fn video_to_media(track: SharedPtr<VideoTrack>) -> SharedPtr<MediaStreamTrack>;
unsafe fn media_to_video(track: SharedPtr<MediaStreamTrack>) -> SharedPtr<VideoTrack>;
fn _shared_video_track() -> SharedPtr<VideoTrack>;