talksik/client-sdk-rust
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: add ffi datachannel & mute events (#88)

Browse Source
This commit is contained in:
Théo Monnom
2023-06-18 22:45:05 +02:00
committed by GitHub
parent 33dfcb27b0
commit 25f4c9a075
69 changed files with 1934 additions and 1546 deletions
Download Patch File Download Diff File Expand all files Collapse all files
livekit-ffi/Cargo.toml
+2 -2
View File
@@ -18,12 +18,12 @@ __rustls-tls = ["livekit/__rustls-tls"]
livekit = { path = "../livekit", version = "0.1.1" }
livekit-protocol = { path = "../livekit-protocol", version = "0.1.0" }
tokio = { version = "1.0", features = ["full"] }
parking_lot = { version = "0.12.1", features = ["send_guard"] }
futures-util = { version = "0.3", default-features = false, features = ["sink"] }
parking_lot = { version = "0.12.1", features=["send_guard"] }
prost = "0.11.0"
prost-types = "0.11.1"
lazy_static = "1.4.0"
thiserror = "1.0.38"
futures-util = "0.3.23"
log = "0.4.17"
dashmap = "5.4.0"
env_logger = "0.10.0"
livekit-ffi/protocol/audio_frame.proto
+12 -6
View File
@@ -18,15 +18,16 @@ message AllocAudioBufferResponse { AudioFrameBufferInfo buffer = 1; }
// Create a new AudioStream
// AudioStream is used to receive audio frames from a track
message NewAudioStreamRequest {
FfiHandleId room_handle = 1;
string participant_sid = 2;
string track_sid = 3;
AudioStreamType type = 4;
FfiHandleId track_handle = 1;
AudioStreamType type = 2;
}
message NewAudioStreamResponse { AudioStreamInfo stream = 1; }
// Create a new AudioSource
message NewAudioSourceRequest { AudioSourceType type = 1; }
message NewAudioSourceRequest {
AudioSourceType type = 1;
optional AudioSourceOptions options = 2;
}
message NewAudioSourceResponse { AudioSourceInfo source = 1; }
// Push a frame to an AudioSource
@@ -78,7 +79,6 @@ enum AudioStreamType {
message AudioStreamInfo {
FfiHandleId handle = 1;
AudioStreamType type = 2;
string track_sid = 3;
}
message AudioStreamEvent {
@@ -94,6 +94,12 @@ message AudioFrameReceived {
/// AudioSource ///
///
message AudioSourceOptions {
bool echo_cancellation = 1;
bool noise_suppression = 2;
bool auto_gain_control = 3;
}
enum AudioSourceType {
AUDIO_SOURCE_NATIVE = 0;
}
livekit-ffi/protocol/ffi.proto
+34 -30
View File
@@ -22,26 +22,27 @@ message FfiRequest {
DisconnectRequest disconnect = 4;
PublishTrackRequest publish_track = 5;
UnpublishTrackRequest unpublish_track = 6;
PublishDataRequest publish_data = 7;
// Track
CreateVideoTrackRequest create_video_track = 7;
CreateAudioTrackRequest create_audio_track = 8;
CreateVideoTrackRequest create_video_track = 8;
CreateAudioTrackRequest create_audio_track = 9;
// Video
AllocVideoBufferRequest alloc_video_buffer = 9;
NewVideoStreamRequest new_video_stream = 10;
NewVideoSourceRequest new_video_source = 11;
CaptureVideoFrameRequest capture_video_frame = 12;
ToI420Request to_i420 = 13;
ToArgbRequest to_argb = 14;
AllocVideoBufferRequest alloc_video_buffer = 10;
NewVideoStreamRequest new_video_stream = 11;
NewVideoSourceRequest new_video_source = 12;
CaptureVideoFrameRequest capture_video_frame = 13;
ToI420Request to_i420 = 14;
ToArgbRequest to_argb = 15;
// Audio
AllocAudioBufferRequest alloc_audio_buffer = 15;
NewAudioStreamRequest new_audio_stream = 16;
NewAudioSourceRequest new_audio_source = 17;
CaptureAudioFrameRequest capture_audio_frame = 18;
NewAudioResamplerRequest new_audio_resampler = 19;
RemixAndResampleRequest remix_and_resample = 20;
AllocAudioBufferRequest alloc_audio_buffer = 16;
NewAudioStreamRequest new_audio_stream = 17;
NewAudioSourceRequest new_audio_source = 18;
CaptureAudioFrameRequest capture_audio_frame = 19;
NewAudioResamplerRequest new_audio_resampler = 20;
RemixAndResampleRequest remix_and_resample = 21;
}
}
@@ -56,26 +57,27 @@ message FfiResponse {
DisconnectResponse disconnect = 4;
PublishTrackResponse publish_track = 5;
UnpublishTrackResponse unpublish_track = 6;
PublishDataResponse publish_data = 7;
// Track
CreateVideoTrackResponse create_video_track = 7;
CreateAudioTrackResponse create_audio_track = 8;
CreateVideoTrackResponse create_video_track = 8;
CreateAudioTrackResponse create_audio_track = 9;
// Video
AllocVideoBufferResponse alloc_video_buffer = 9;
NewVideoStreamResponse new_video_stream = 10;
NewVideoSourceResponse new_video_source = 11;
CaptureVideoFrameResponse capture_video_frame = 12;
ToI420Response to_i420 = 13;
ToArgbResponse to_argb = 14;
AllocVideoBufferResponse alloc_video_buffer = 10;
NewVideoStreamResponse new_video_stream = 11;
NewVideoSourceResponse new_video_source = 12;
CaptureVideoFrameResponse capture_video_frame = 13;
ToI420Response to_i420 = 14;
ToArgbResponse to_argb = 15;
// Audio
AllocAudioBufferResponse alloc_audio_buffer = 15;
NewAudioStreamResponse new_audio_stream = 16;
NewAudioSourceResponse new_audio_source = 17;
CaptureAudioFrameResponse capture_audio_frame = 18;
NewAudioResamplerResponse new_audio_resampler = 19;
RemixAndResampleResponse remix_and_resample = 20;
AllocAudioBufferResponse alloc_audio_buffer = 16;
NewAudioStreamResponse new_audio_stream = 17;
NewAudioSourceResponse new_audio_source = 18;
CaptureAudioFrameResponse capture_audio_frame = 19;
NewAudioResamplerResponse new_audio_resampler = 20;
RemixAndResampleResponse remix_and_resample = 21;
}
}
@@ -87,8 +89,10 @@ message FfiEvent {
VideoStreamEvent video_stream_event = 4;
AudioStreamEvent audio_stream_event = 5;
ConnectCallback connect = 6;
DisposeCallback dispose = 7;
PublishTrackCallback publish_track = 8;
DisconnectCallback disconnect = 7;
DisposeCallback dispose = 8;
PublishTrackCallback publish_track = 9;
PublishDataCallback publish_data = 10;
}
}
livekit-ffi/protocol/handle.proto
+4 -1
View File
@@ -6,7 +6,10 @@ option csharp_namespace = "LiveKit.Proto";
/// # Safety
/// The foreign language is responsable for disposing handles
/// Forgetting to dispose the handle may lead to memory leaks
/// Messages in this file can contain an FFIHandle
///
/// A handle means that the foreign language may still use the corresponding object
/// Dropping a handle doesn't necessarily mean that the object is destroyed if it is still used
/// on the FfiServer (Rust)
message FfiHandleId {
uint64 id = 1;
}
livekit-ffi/protocol/participant.proto
+1
View File
@@ -6,6 +6,7 @@ option csharp_namespace = "LiveKit.Proto";
//import "handle.proto";
import "track.proto";
// Seems like we don't need a FfiHandle for participants (atm at least)
message ParticipantInfo {
string sid = 1;
string name = 2;
livekit-ffi/protocol/room.proto
+25 -7
View File
@@ -26,7 +26,7 @@ message ConnectCallback {
// Disconnect from the a room
message DisconnectRequest { FfiHandleId room_handle = 1; }
message DisconnectResponse { FfiAsyncId async_id = 1; }
message DisconnectCallback { }
message DisconnectCallback { FfiAsyncId async_id = 1; }
// Publish a track to the room
message PublishTrackRequest {
@@ -53,9 +53,27 @@ message UnpublishTrackResponse {
FfiAsyncId async_id = 1;
}
message UnpublishTrackCallback {
optional string error = 1;
FfiAsyncId async_id = 1;
optional string error = 2;
}
// Publish data to other participants
message PublishDataRequest {
FfiHandleId room_handle = 1;
uint64 data_ptr = 2;
uint64 data_size = 3;
DataPacketKind kind = 4;
repeated string destination_sids = 5; // destination
}
message PublishDataResponse {
FfiAsyncId async_id = 1;
}
message PublishDataCallback {
FfiAsyncId async_id = 1;
optional string error = 2;
}
///
/// Options
///
@@ -77,13 +95,13 @@ message TrackPublishOptions {
bool dtx = 4;
bool red = 5;
bool simulcast = 6;
string name = 7;
TrackSource source = 8;
TrackSource source = 7;
}
message RoomOptions {
bool auto_subscribe = 1;
bool adaptive_stream = 2;
bool dynacast = 3;
}
///
@@ -104,7 +122,7 @@ enum ConnectionState {
}
enum DataPacketKind {
KIND_UNRELIABLE = 0;
KIND_LOSSY = 0;
KIND_RELIABLE = 1;
}
@@ -141,7 +159,7 @@ message RoomInfo {
message DataReceived {
FfiHandleId handle = 1;
optional string participant_sid = 2;
optional string participant_sid = 2; // Can be empty if the data is sent a server SDK
uint64 data_ptr = 3;
uint64 data_size = 4;
DataPacketKind kind = 5;
@@ -196,4 +214,4 @@ message ConnectionStateChanged { ConnectionState state = 1; }
message Connected {}
message Disconnected {}
message Reconnecting {}
message Reconnected {}
message Reconnected {}
livekit-ffi/protocol/track.proto
+3 -16
View File
@@ -7,21 +7,10 @@ import "handle.proto";
import "video_frame.proto";
import "audio_frame.proto";
message VideoCaptureOptions {
VideoResolution resolution = 1;
}
message AudioCaptureOptions {
bool echo_cancellation = 1;
bool noise_suppression = 2;
bool auto_gain_control = 3;
}
// Create a new VideoTrack from a VideoSource
message CreateVideoTrackRequest {
string name = 1;
VideoCaptureOptions options = 2;
FfiHandleId source_handle = 3;
FfiHandleId source_handle = 2;
}
message CreateVideoTrackResponse {
TrackInfo track = 1;
@@ -30,8 +19,7 @@ message CreateVideoTrackResponse {
// Create a new AudioTrack from a AudioSource
message CreateAudioTrackRequest {
string name = 1;
AudioCaptureOptions options = 2;
FfiHandleId source_handle = 3;
FfiHandleId source_handle = 2;
}
message CreateAudioTrackResponse {
TrackInfo track = 1;
@@ -79,8 +67,7 @@ message TrackPublicationInfo {
}
message TrackInfo {
// Tracks created/owned by the client will have a handle
FfiHandleId opt_handle = 1;
FfiHandleId handle = 1;
string sid = 2;
string name = 3;
TrackKind kind = 4;
livekit-ffi/protocol/video_frame.proto
+14 -7
View File
@@ -16,16 +16,19 @@ message AllocVideoBufferResponse { VideoFrameBufferInfo buffer = 1; }
// Create a new VideoStream
// VideoStream is used to receive video frames from a track
message NewVideoStreamRequest {
FfiHandleId room_handle = 1;
string participant_sid = 2;
string track_sid = 3;
VideoStreamType type = 4;
FfiHandleId track_handle = 1;
VideoStreamType type = 2;
}
message NewVideoStreamResponse { VideoStreamInfo stream = 1; }
// Create a new VideoSource
// VideoSource is used to send video frame to a track
message NewVideoSourceRequest { VideoSourceType type = 1; }
message NewVideoSourceRequest {
VideoSourceType type = 1;
// Used to determine which encodings to use + simulcast layers
// Most of the time it corresponds to the source resolution
optional VideoSourceResolution resolution = 2;
}
message NewVideoSourceResponse { VideoSourceInfo source = 1; }
// Push a frame to a VideoSource
@@ -110,7 +113,7 @@ message ARGBBufferInfo {
}
message VideoFrameInfo {
int64 timestamp = 1;
int64 timestamp_us = 1; // In microseconds
VideoRotation rotation = 2;
}
@@ -168,7 +171,6 @@ enum VideoStreamType {
message VideoStreamInfo {
FfiHandleId handle = 1;
VideoStreamType type = 2;
string track_sid = 3;
}
message VideoStreamEvent {
@@ -185,6 +187,11 @@ message VideoFrameReceived {
/// VideoSource ///
///
message VideoSourceResolution {
uint32 width = 1;
uint32 height = 2;
}
enum VideoSourceType {
VIDEO_SOURCE_NATIVE = 0;
}
livekit-ffi/src/conversion/audio_frame.rs
+11 -1
View File
@@ -1,7 +1,18 @@
use crate::server::audio_frame::{FfiAudioSource, FfiAudioSream};
use crate::{proto, FfiHandleId};
use livekit::webrtc::audio_source::AudioSourceOptions;
use livekit::webrtc::prelude::*;
impl From<proto::AudioSourceOptions> for AudioSourceOptions {
fn from(opts: proto::AudioSourceOptions) -> Self {
Self {
echo_cancellation: opts.echo_cancellation,
auto_gain_control: opts.auto_gain_control,
noise_suppression: opts.noise_suppression,
}
}
}
impl proto::AudioFrameBufferInfo {
pub fn from(handle_id: FfiHandleId, buffer: &AudioFrame) -> Self {
Self {
@@ -20,7 +31,6 @@ impl From<&FfiAudioSream> for proto::AudioStreamInfo {
handle: Some(proto::FfiHandleId {
id: stream.handle_id() as u64,
}),
track_sid: stream.track_sid().clone().into(),
r#type: stream.stream_type() as i32,
}
}
livekit-ffi/src/conversion/room.rs
+16 -61
View File
@@ -1,67 +1,23 @@
use crate::{proto, FfiHandleId, INVALID_HANDLE};
use crate::{proto, FfiHandleId};
use livekit::options::{AudioEncoding, TrackPublishOptions, VideoEncoding};
use livekit::prelude::*;
impl proto::RoomEvent {
pub fn from(room_handle: FfiHandleId, event: RoomEvent) -> Option<Self> {
let message = match event {
RoomEvent::ParticipantConnected(participant) => Some(
proto::room_event::Message::ParticipantConnected(proto::ParticipantConnected {
info: Some((&participant).into()),
}),
),
RoomEvent::ParticipantDisconnected(participant) => {
Some(proto::room_event::Message::ParticipantDisconnected(
proto::ParticipantDisconnected {
info: Some((&participant).into()),
},
))
}
RoomEvent::TrackPublished {
publication,
participant,
} => Some(proto::room_event::Message::TrackPublished(
proto::TrackPublished {
participant_sid: participant.sid().to_string(),
publication: Some((&publication).into()),
},
)),
RoomEvent::TrackUnpublished {
publication,
participant,
} => Some(proto::room_event::Message::TrackUnpublished(
proto::TrackUnpublished {
participant_sid: participant.sid().to_string(),
publication_sid: publication.sid().into(),
},
)),
RoomEvent::TrackSubscribed {
track,
publication: _,
participant,
} => Some(proto::room_event::Message::TrackSubscribed(
proto::TrackSubscribed {
participant_sid: participant.sid().to_string(),
track: Some(proto::TrackInfo::from_remote_track(INVALID_HANDLE, &track)),
},
)),
RoomEvent::TrackUnsubscribed {
track,
publication: _,
participant,
} => Some(proto::room_event::Message::TrackUnsubscribed(
proto::TrackUnsubscribed {
participant_sid: participant.sid().to_string(),
track_sid: track.sid().to_string(),
},
)),
_ => None,
};
impl From<proto::RoomOptions> for RoomOptions {
fn from(value: proto::RoomOptions) -> Self {
Self {
adaptive_stream: value.adaptive_stream,
auto_subscribe: value.auto_subscribe,
dynacast: value.dynacast,
}
}
}
message.map(|message| proto::RoomEvent {
room_handle: Some(room_handle.into()),
message: Some(message),
})
impl From<proto::DataPacketKind> for DataPacketKind {
fn from(value: proto::DataPacketKind) -> Self {
match value {
proto::DataPacketKind::KindReliable => Self::Reliable,
proto::DataPacketKind::KindLossy => Self::Lossy,
}
}
}
@@ -93,7 +49,6 @@ impl From<proto::TrackPublishOptions> for TrackPublishOptions {
dtx: opts.dtx,
red: opts.red,
simulcast: opts.simulcast,
name: opts.name,
source: proto::TrackSource::from_i32(opts.source).unwrap().into(),
}
}
livekit-ffi/src/conversion/track.rs
+1 -20
View File
@@ -1,25 +1,6 @@
use crate::{proto, FfiHandleId};
use livekit::options::{AudioCaptureOptions, VideoCaptureOptions};
use livekit::prelude::*;
impl From<proto::VideoCaptureOptions> for VideoCaptureOptions {
fn from(opts: proto::VideoCaptureOptions) -> Self {
Self {
resolution: opts.resolution.unwrap_or_default().into(),
}
}
}
impl From<proto::AudioCaptureOptions> for AudioCaptureOptions {
fn from(opts: proto::AudioCaptureOptions) -> Self {
Self {
echo_cancellation: opts.echo_cancellation,
auto_gain_control: opts.auto_gain_control,
noise_suppression: opts.noise_suppression,
}
}
}
impl From<TrackSource> for proto::TrackSource {
fn from(source: TrackSource) -> proto::TrackSource {
match source {
@@ -63,7 +44,7 @@ macro_rules! impl_track_into {
#[allow(dead_code)]
pub fn $fnc(handle_id: FfiHandleId, track: $t) -> Self {
Self {
opt_handle: Some(handle_id.into()),
handle: Some(handle_id.into()),
name: track.name(),
stream_state: proto::StreamState::from(track.stream_state()).into(),
sid: track.sid().to_string(),
livekit-ffi/src/conversion/video_frame.rs
+11 -2
View File
@@ -4,6 +4,16 @@ use crate::FfiHandleId;
use livekit::options::{VideoCodec, VideoResolution};
use livekit::webrtc::prelude::*;
use livekit::webrtc::video_frame;
use livekit::webrtc::video_source::VideoResolution as VideoSourceResolution;
impl From<proto::VideoSourceResolution> for VideoSourceResolution {
fn from(res: proto::VideoSourceResolution) -> Self {
Self {
width: res.width,
height: res.height,
}
}
}
macro_rules! impl_yuv_into {
(@fields, $buffer:ident, $data_y:ident, $data_u:ident, $data_v: ident) => {
@@ -72,7 +82,7 @@ impl proto::VideoFrameInfo {
T: AsRef<dyn VideoFrameBuffer>,
{
Self {
timestamp: frame.timestamp,
timestamp_us: frame.timestamp_us,
rotation: proto::VideoRotation::from(frame.rotation).into(),
}
}
@@ -236,7 +246,6 @@ impl From<&FfiVideoStream> for proto::VideoStreamInfo {
handle: Some(proto::FfiHandleId {
id: stream.handle_id() as u64,
}),
track_sid: stream.track_sid().clone().into(),
r#type: stream.stream_type() as i32,
}
}
livekit-ffi/src/server/audio_frame.rs
+32 -34
View File
@@ -2,11 +2,9 @@ use crate::{proto, server, FfiError, FfiHandleId, FfiResult};
use futures_util::StreamExt;
use livekit::prelude::*;
use livekit::webrtc::audio_frame::AudioFrame;
use livekit::webrtc::audio_source::native::NativeAudioSource;
use livekit::webrtc::audio_stream::native::NativeAudioStream;
use livekit::webrtc::prelude::*;
use log::warn;
use server::utils;
use tokio::sync::oneshot;
// ===== FFIAudioStream =====
@@ -14,7 +12,6 @@ use tokio::sync::oneshot;
pub struct FfiAudioSream {
handle_id: FfiHandleId,
stream_type: proto::AudioStreamType,
track_sid: TrackSid,
#[allow(dead_code)]
close_tx: oneshot::Sender<()>, // Close the stream on drop
@@ -35,42 +32,45 @@ impl FfiAudioSream {
) -> FfiResult<proto::AudioStreamInfo> {
let (close_tx, close_rx) = oneshot::channel();
let stream_type = proto::AudioStreamType::from_i32(new_stream.r#type).unwrap();
let track_sid: TrackSid = new_stream.track_sid.into();
let room_handle = new_stream
.room_handle
.ok_or(FfiError::InvalidRequest("room_handle is empty"))?
let handle_id = new_stream
.track_handle
.ok_or(FfiError::InvalidRequest("track_handle is empty"))?
.id as FfiHandleId;
let track = utils::find_remote_track(
server,
&track_sid,
&new_stream.participant_sid.into(),
room_handle,
)?
.rtc_track();
let track = server
.ffi_handles()
.get(&handle_id)
.ok_or(FfiError::InvalidRequest("track not found"))?;
let MediaStreamTrack::Audio(track) = track else {
let track = track
.downcast_ref::<Track>()
.ok_or(FfiError::InvalidRequest("handle is not a Track"))?;
let rtc_track = track.rtc_track();
let MediaStreamTrack::Audio(rtc_track) = rtc_track else {
return Err(FfiError::InvalidRequest("not an audio track"));
};
let audio_stream = match stream_type {
#[cfg(not(target_arch = "wasm32"))]
proto::AudioStreamType::AudioStreamNative => {
let audio_stream = Self {
handle_id: server.next_id(),
stream_type,
close_tx,
track_sid,
};
let native_stream = NativeAudioStream::new(rtc_track);
server.async_runtime.spawn(Self::native_audio_stream_task(
server,
audio_stream.handle_id,
NativeAudioStream::new(track),
native_stream,
close_rx,
));
Ok::<FfiAudioSream, FfiError>(audio_stream)
}
// TODO(theomonnom): Support other stream types
_ => return Err(FfiError::InvalidRequest("unsupported audio stream type")),
}?;
@@ -91,10 +91,6 @@ impl FfiAudioSream {
self.stream_type
}
pub fn track_sid(&self) -> &TrackSid {
&self.track_sid
}
async fn native_audio_stream_task(
server: &'static server::FfiServer,
stream_handle_id: FfiHandleId,
@@ -139,12 +135,7 @@ impl FfiAudioSream {
pub struct FfiAudioSource {
handle_id: FfiHandleId,
source_type: proto::AudioSourceType,
source: AudioSource,
}
#[derive(Clone)]
pub enum AudioSource {
Native(NativeAudioSource),
source: RtcAudioSource,
}
impl FfiAudioSource {
@@ -153,12 +144,17 @@ impl FfiAudioSource {
new_source: proto::NewAudioSourceRequest,
) -> FfiResult<proto::AudioSourceInfo> {
let source_type = proto::AudioSourceType::from_i32(new_source.r#type).unwrap();
#[allow(unreachable_patterns)]
let source_inner = match source_type {
#[cfg(not(target_arch = "wasm32"))]
proto::AudioSourceType::AudioSourceNative => {
let audio_source = NativeAudioSource::default();
Ok::<AudioSource, FfiError>(AudioSource::Native(audio_source))
} //_ => return Err(FfiError::InvalidRequest("unsupported audio source type")),
}?;
use livekit::webrtc::audio_source::native::NativeAudioSource;
let audio_source =
NativeAudioSource::new(new_source.options.map(Into::into).unwrap_or_default());
RtcAudioSource::Native(audio_source)
}
_ => return Err(FfiError::InvalidRequest("unsupported audio source type")),
};
let audio_source = Self {
handle_id: server.next_id(),
@@ -180,7 +176,8 @@ impl FfiAudioSource {
capture: proto::CaptureAudioFrameRequest,
) -> FfiResult<()> {
match self.source {
AudioSource::Native(ref source) => {
#[cfg(not(target_arch = "wasm32"))]
RtcAudioSource::Native(ref source) => {
let buffer_handle = capture
.buffer_handle
.ok_or(FfiError::InvalidRequest("buffer_handle is empty"))?
@@ -197,6 +194,7 @@ impl FfiAudioSource {
source.capture_frame(frame);
}
_ => {}
}
Ok(())
@@ -210,7 +208,7 @@ impl FfiAudioSource {
self.source_type
}
pub fn inner_source(&self) -> &AudioSource {
pub fn inner_source(&self) -> &RtcAudioSource {
&self.source
}
}
livekit-ffi/src/server/mod.rs
+70 -30
View File
@@ -15,7 +15,6 @@ use std::sync::Arc;
pub mod audio_frame;
pub mod room;
pub mod utils;
pub mod video_frame;
#[cfg(test)]
@@ -31,7 +30,7 @@ pub struct FfiConfig {
pub struct FfiServer {
rooms: Mutex<HashMap<RoomSid, FfiHandleId>>,
/// Store all FFI handles inside an HashMap, if this isn't efficient enough
/// Store all Ffi handles inside an HashMap, if this isn't efficient enough
/// We can still use Box::into_raw & Box::from_raw in the future (but keep it safe for now)
ffi_handles: DashMap<FfiHandleId, FfiHandle>,
next_id: AtomicUsize,
@@ -116,11 +115,9 @@ impl FfiServer {
}
// # SAFETY: The foreign language is responsible for ensuring that the callback function is valid
unsafe {
*self.config.lock() = Some(FfiConfig {
callback_fn: std::mem::transmute(init.event_callback_ptr),
});
}
*self.config.lock() = Some(FfiConfig {
callback_fn: unsafe { std::mem::transmute(init.event_callback_ptr) },
});
Ok(proto::InitializeResponse::default())
}
@@ -191,9 +188,37 @@ impl FfiServer {
fn on_disconnect(
&'static self,
_disconnect: proto::DisconnectRequest,
disconnect: proto::DisconnectRequest,
) -> FfiResult<proto::DisconnectResponse> {
Ok(proto::DisconnectResponse::default())
let async_id = self.next_id() as FfiAsyncId;
let room_handle = disconnect
.room_handle
.as_ref()
.ok_or(FfiError::InvalidRequest("room_handle is empty"))?
.id as FfiHandleId;
let ffi_room = self
.ffi_handles
.remove(&room_handle)
.ok_or(FfiError::InvalidRequest("room not found"))?
.1;
let ffi_room = ffi_room
.downcast::<room::FfiRoom>()
.map_err(|_| FfiError::InvalidRequest("room is not a FfiRoom"))?;
self.async_runtime.spawn(async move {
ffi_room.close().await;
let _ = self.send_event(proto::ffi_event::Message::Disconnect(
proto::DisconnectCallback {
async_id: Some(async_id.into()),
},
));
});
Ok(proto::DisconnectResponse {
async_id: Some(async_id.into()),
})
}
fn on_publish_track(
@@ -230,14 +255,17 @@ impl FfiServer {
.ok_or(FfiError::InvalidRequest("track not found"))?;
let track = track
.downcast_ref::<LocalTrack>()
.ok_or(FfiError::InvalidRequest("track is not a LocalTrack"))?;
.downcast_ref::<Track>()
.ok_or(FfiError::InvalidRequest("track is not a Track"))?;
let local_track = LocalTrack::try_from(track.clone())
.map_err(|_| FfiError::InvalidRequest("track is not a LocalTrack"))?;
let publication = ffi_room
.room()
.local_participant()
.publish_track(
track.clone(),
local_track,
publish.options.map(Into::into).unwrap_or_default(),
)
.await?;
@@ -269,6 +297,29 @@ impl FfiServer {
Ok(proto::UnpublishTrackResponse::default())
}
fn on_publish_data(
&'static self,
publish: proto::PublishDataRequest,
) -> FfiResult<proto::PublishDataResponse> {
let room_handle = publish
.room_handle
.as_ref()
.ok_or(FfiError::InvalidRequest("room_handle is empty"))?
.id as FfiHandleId;
let ffi_room = self
.ffi_handles
.get(&room_handle)
.ok_or(FfiError::InvalidRequest("room not found"))?;
let ffi_room = ffi_room
.downcast_ref::<room::FfiRoom>()
.ok_or(FfiError::InvalidRequest("room is not a FfiRoom"))?;
// Push the data to an async queue (avoid blocking and keep the order)
ffi_room.publish_data(self, publish)
}
// Track
fn on_create_video_track(
&'static self,
@@ -276,7 +327,6 @@ impl FfiServer {
) -> FfiResult<proto::CreateVideoTrackResponse> {
let handle_id = create
.source_handle
.as_ref()
.ok_or(FfiError::InvalidRequest("source_handle is empty"))?
.id as FfiHandleId;
@@ -290,19 +340,13 @@ impl FfiServer {
.ok_or(FfiError::InvalidRequest("handle is not a video source"))?;
let source = source.inner_source().clone();
let video_track = match source {
video_frame::VideoSource::Native(native_source) => LocalVideoTrack::create_video_track(
&create.name,
create.options.unwrap_or_default().into(),
native_source,
),
};
let video_track = LocalVideoTrack::create_video_track(&create.name, source);
let handle_id = self.next_id() as FfiHandleId;
let track_info = proto::TrackInfo::from_local_video_track(handle_id, &video_track);
self.ffi_handles
.insert(handle_id, Box::new(LocalTrack::Video(video_track)));
.insert(handle_id, Box::new(Track::LocalVideo(video_track)));
Ok(proto::CreateVideoTrackResponse {
track: Some(track_info),
@@ -315,7 +359,6 @@ impl FfiServer {
) -> FfiResult<proto::CreateAudioTrackResponse> {
let handle_id = create
.source_handle
.as_ref()
.ok_or(FfiError::InvalidRequest("source_handle is empty"))?
.id as FfiHandleId;
@@ -329,19 +372,13 @@ impl FfiServer {
.ok_or(FfiError::InvalidRequest("handle is not an audio source"))?;
let source = source.inner_source().clone();
let audio_track = match source {
audio_frame::AudioSource::Native(native_source) => LocalAudioTrack::create_audio_track(
&create.name,
create.options.unwrap_or_default().into(),
native_source,
),
};
let audio_track = LocalAudioTrack::create_audio_track(&create.name, source);
let handle_id = self.next_id() as FfiHandleId;
let track_info = proto::TrackInfo::from_local_audio_track(handle_id, &audio_track);
self.ffi_handles
.insert(handle_id, Box::new(LocalTrack::Audio(audio_track)));
.insert(handle_id, Box::new(Track::LocalAudio(audio_track)));
Ok(proto::CreateAudioTrackResponse {
track: Some(track_info),
@@ -714,6 +751,9 @@ impl FfiServer {
proto::ffi_request::Message::UnpublishTrack(unpublish) => {
proto::ffi_response::Message::UnpublishTrack(self.on_unpublish_track(unpublish)?)
}
proto::ffi_request::Message::PublishData(publish) => {
proto::ffi_response::Message::PublishData(self.on_publish_data(publish)?)
}
proto::ffi_request::Message::CreateVideoTrack(create) => {
proto::ffi_response::Message::CreateVideoTrack(self.on_create_video_track(create)?)
}
livekit-ffi/src/server/room.rs
+165 -22
View File
@@ -1,14 +1,24 @@
use crate::server::FfiServer;
use crate::{proto, FfiHandleId, FfiResult};
use crate::{proto, FfiAsyncId, FfiError, FfiHandleId, FfiResult};
use livekit::prelude::*;
use std::slice;
use std::sync::Arc;
use tokio::sync::{mpsc, oneshot};
use tokio::sync::{broadcast, mpsc};
use tokio::task::JoinHandle;
struct DataPacket {
data: Vec<u8>,
kind: DataPacketKind,
destination_sids: Vec<String>,
async_id: FfiAsyncId,
}
pub struct FfiRoom {
room: Arc<Room>,
handle: JoinHandle<()>,
close_tx: oneshot::Sender<()>,
event_handle: JoinHandle<()>,
data_handle: JoinHandle<()>,
close_tx: broadcast::Sender<()>,
data_tx: mpsc::UnboundedSender<DataPacket>,
}
impl FfiRoom {
@@ -16,33 +26,77 @@ impl FfiRoom {
server: &'static FfiServer,
connect: proto::ConnectRequest,
) -> FfiResult<proto::RoomInfo> {
let (room, events) = Room::connect(&connect.url, &connect.token).await?;
let (room, events) = Room::connect(
&connect.url,
&connect.token,
connect.options.map(Into::into).unwrap_or_default(),
)
.await?;
let room = Arc::new(room);
let (close_tx, close_rx) = oneshot::channel();
let next_id = server.next_id() as FfiHandleId;
let (close_tx, close_rx) = broadcast::channel(1);
let (data_tx, data_rx) = mpsc::unbounded_channel();
let handle =
let next_id = server.next_id() as FfiHandleId;
let event_handle = server.async_runtime.spawn(room_task(
server,
room.clone(),
next_id,
events,
close_rx.resubscribe(),
));
let data_handle =
server
.async_runtime
.spawn(room_task(server, room.clone(), next_id, events, close_rx));
let room_info = proto::RoomInfo::from_room(next_id, &room);
.spawn(data_task(server, room.clone(), data_rx, close_rx));
let ffi_room = Self {
room: room.clone(),
handle,
event_handle,
data_handle,
close_tx,
data_tx,
};
server.ffi_handles().insert(next_id, Box::new(ffi_room));
server.rooms().lock().insert(room.sid(), next_id);
let room_info = proto::RoomInfo::from_room(next_id, &room);
Ok(room_info)
}
pub fn publish_data(
&self,
server: &'static FfiServer,
publish: proto::PublishDataRequest,
) -> FfiResult<proto::PublishDataResponse> {
let data = unsafe {
slice::from_raw_parts(publish.data_ptr as *const u8, publish.data_size as usize)
};
let kind = proto::DataPacketKind::from_i32(publish.kind).unwrap();
let destination_sids: Vec<String> = publish.destination_sids;
let async_id = server.next_id() as FfiAsyncId;
let packet = DataPacket {
data: data.to_vec(), // Avoid copy?
kind: kind.into(),
destination_sids,
async_id,
};
self.data_tx
.send(packet)
.map_err(|_| FfiError::InvalidRequest("failed to send data packet"))?;
Ok(proto::PublishDataResponse {
async_id: Some(async_id.into()),
})
}
pub async fn close(self) {
let _ = self.room.close().await;
let _ = self.close_tx.send(());
let _ = self.handle.await;
let _ = self.event_handle.await;
let _ = self.data_handle.await;
}
pub fn room(&self) -> &Arc<Room> {
@@ -50,12 +104,41 @@ impl FfiRoom {
}
}
async fn data_task(
server: &'static FfiServer,
room: Arc<Room>,
mut data_rx: mpsc::UnboundedReceiver<DataPacket>,
mut close_rx: broadcast::Receiver<()>,
) {
loop {
tokio::select! {
Some(event) = data_rx.recv() => {
let res = room.local_participant().publish_data(
event.data,
event.kind,
event.destination_sids,
).await;
let cb = proto::PublishDataCallback {
async_id: Some(event.async_id.into()),
error: res.err().map(|e| e.to_string()),
};
let _ = server.send_event(proto::ffi_event::Message::PublishData(cb));
},
_ = close_rx.recv() => {
break;
}
}
}
}
async fn room_task(
server: &'static FfiServer,
room: Arc<Room>,
room_handle: FfiHandleId,
mut events: mpsc::UnboundedReceiver<livekit::RoomEvent>,
mut close_rx: oneshot::Receiver<()>,
mut close_rx: broadcast::Receiver<()>,
) {
server
.async_runtime
@@ -66,18 +149,78 @@ async fn room_task(
loop {
tokio::select! {
Some(event) = events.recv() => {
if let Some(event) = proto::RoomEvent::from(room_handle, event.clone()) {
let _ = server.send_event(proto::ffi_event::Message::RoomEvent(event));
let message = match event {
RoomEvent::ParticipantConnected(participant) => {
server.async_runtime.spawn(participant_task(Participant::Remote(participant.clone())));
Some(proto::room_event::Message::ParticipantConnected(
proto::ParticipantConnected {
info: Some(proto::ParticipantInfo::from(&participant)),
}
))
},
RoomEvent::ParticipantDisconnected(participant) => {
Some(proto::room_event::Message::ParticipantDisconnected(
proto::ParticipantDisconnected {
info: Some(proto::ParticipantInfo::from(&participant)),
},
))
}
RoomEvent::TrackPublished {
publication,
participant,
} => Some(proto::room_event::Message::TrackPublished(
proto::TrackPublished {
participant_sid: participant.sid().to_string(),
publication: Some(proto::TrackPublicationInfo::from(&publication))
},
)),
RoomEvent::TrackUnpublished {
publication,
participant,
} => Some(proto::room_event::Message::TrackUnpublished(
proto::TrackUnpublished {
participant_sid: participant.sid().to_string(),
publication_sid: publication.sid().into(),
},
)),
RoomEvent::TrackSubscribed {
track,
publication: _,
participant,
} => {
let handle_id = server.next_id() as FfiHandleId;
let track_info = proto::TrackInfo::from_remote_track(handle_id, &track);
server.ffi_handles().insert(handle_id, Box::new(Track::from(track)));
Some(proto::room_event::Message::TrackSubscribed(
proto::TrackSubscribed {
participant_sid: participant.sid().to_string(),
track: Some(track_info),
},
))
},
RoomEvent::TrackUnsubscribed {
track,
publication: _,
participant,
} => Some(proto::room_event::Message::TrackUnsubscribed(
proto::TrackUnsubscribed {
participant_sid: participant.sid().to_string(),
track_sid: track.sid().to_string(),
},
)),
_ => None
};
if message.is_some() {
let _ = server.send_event(proto::ffi_event::Message::RoomEvent(proto::RoomEvent{
room_handle: Some(room_handle.into()),
message
}));
}
match event {
RoomEvent::ParticipantConnected(p) => {
server.async_runtime.spawn(participant_task(Participant::Remote(p)));
}
_ => {}
}
},
_ = &mut close_rx => {
_ = close_rx.recv() => {
break;
}
};
livekit-ffi/src/server/tests.rs
+10 -14
View File
@@ -212,18 +212,22 @@ fn publish_video_track() {
client::FfiHandle(connect.room.unwrap().handle.unwrap().id as FfiHandleId);
// Create a new VideoSource
const VIDEO_WIDTH: u32 = 640;
const VIDEO_HEIGHT: u32 = 480;
const VIDEO_FPS: f64 = 8.0;
let res = client.send_request(proto::FfiRequest {
message: Some(proto::ffi_request::Message::NewVideoSource(
proto::NewVideoSourceRequest {
r#type: proto::VideoSourceType::VideoSourceNative as i32,
resolution: Some(proto::VideoSourceResolution {
width: VIDEO_WIDTH,
height: VIDEO_HEIGHT,
}),
},
)),
});
const VIDEO_WIDTH: u32 = 640;
const VIDEO_HEIGHT: u32 = 480;
const VIDEO_FPS: f64 = 8.0;
let proto::ffi_response::Message::NewVideoSource(new_video_source) =
res.message.unwrap() else {
panic!("unexpected response");
@@ -241,13 +245,6 @@ fn publish_video_track() {
source_handle: Some(proto::FfiHandleId {
id: source_handle.0 as u64,
}),
options: Some(proto::VideoCaptureOptions {
resolution: Some(proto::VideoResolution {
width: VIDEO_WIDTH,
height: VIDEO_HEIGHT,
frame_rate: VIDEO_FPS,
}),
}),
},
)),
});
@@ -258,11 +255,10 @@ fn publish_video_track() {
};
let track_handle = client::FfiHandle(
create_video_track.track.unwrap().opt_handle.unwrap().id as FfiHandleId,
create_video_track.track.unwrap().handle.unwrap().id as FfiHandleId,
);
let publish_options = proto::TrackPublishOptions {
name: "video_test".to_string(),
video_codec: proto::VideoCodec::H264 as i32,
source: proto::TrackSource::SourceCamera as i32,
..Default::default()
@@ -320,7 +316,7 @@ fn publish_video_track() {
id: buffer_handle.0 as u64,
}),
frame: Some(proto::VideoFrameInfo {
timestamp: 0, // TODO
timestamp_us: 0,
rotation: proto::VideoRotation::VideoRotation0 as i32,
}),
},
livekit-ffi/src/server/utils.rs
-32
View File
@@ -1,32 +0,0 @@
use crate::{server, FfiError, FfiHandleId, FfiResult};
use livekit::prelude::*;
pub fn find_remote_track(
server: &'static server::FfiServer,
track_sid: &TrackSid,
participant_sid: &ParticipantSid,
room_handle: FfiHandleId,
) -> FfiResult<RemoteTrack> {
let ffi_room = server
.ffi_handles()
.get(&room_handle)
.ok_or(FfiError::InvalidRequest("room not found"))?;
let ffi_room = ffi_room
.downcast_ref::<server::room::FfiRoom>()
.ok_or(FfiError::InvalidRequest("room is not ffi room"))?;
let room = ffi_room.room();
let participants = room.participants();
let participant = participants
.get(participant_sid)
.ok_or(FfiError::InvalidRequest("participant not found"))?;
let track = participant
.get_track_publication(track_sid)
.ok_or(FfiError::InvalidRequest("publication not found"))?
.track()
.ok_or(FfiError::InvalidRequest("track not found/subscribed"))?;
Ok(track)
}
livekit-ffi/src/server/video_frame.rs
+30 -33
View File
@@ -3,10 +3,8 @@ use futures_util::StreamExt;
use livekit::prelude::*;
use livekit::webrtc::prelude::*;
use livekit::webrtc::video_frame::{BoxVideoFrameBuffer, VideoFrame};
use livekit::webrtc::video_source::native::NativeVideoSource;
use livekit::webrtc::video_stream::native::NativeVideoStream;
use log::warn;
use server::utils;
use tokio::sync::oneshot;
// ===== FFIVideoStream =====
@@ -14,7 +12,6 @@ use tokio::sync::oneshot;
pub struct FfiVideoStream {
handle_id: FfiHandleId,
stream_type: proto::VideoStreamType,
track_sid: TrackSid,
#[allow(dead_code)]
close_tx: oneshot::Sender<()>, // Close the stream on drop
@@ -35,42 +32,43 @@ impl FfiVideoStream {
) -> FfiResult<proto::VideoStreamInfo> {
let (close_tx, close_rx) = oneshot::channel();
let stream_type = proto::VideoStreamType::from_i32(new_stream.r#type).unwrap();
let track_sid: TrackSid = new_stream.track_sid.into();
let room_handle = new_stream
.room_handle
.ok_or(FfiError::InvalidRequest("room_handle is empty"))?
let handle_id = new_stream
.track_handle
.ok_or(FfiError::InvalidRequest("track_handle is empty"))?
.id as FfiHandleId;
let track = utils::find_remote_track(
server,
&track_sid,
&new_stream.participant_sid.into(),
room_handle,
)?
.rtc_track();
let track = server
.ffi_handles()
.get(&handle_id)
.ok_or(FfiError::InvalidRequest("track not found"))?;
let MediaStreamTrack::Video(track) = track else {
let track = track
.downcast_ref::<Track>()
.ok_or(FfiError::InvalidRequest("handle is not a Track"))?;
let rtc_track = track.rtc_track();
let MediaStreamTrack::Video(rtc_track) = rtc_track else {
return Err(FfiError::InvalidRequest("not a video track"));
};
let stream = match stream_type {
#[cfg(not(target_arch = "wasm32"))]
proto::VideoStreamType::VideoStreamNative => {
let video_stream = Self {
handle_id: server.next_id(),
close_tx,
stream_type,
track_sid,
};
server.async_runtime.spawn(Self::native_video_stream_task(
server,
video_stream.handle_id,
NativeVideoStream::new(track),
NativeVideoStream::new(rtc_track),
close_rx,
));
Ok::<FfiVideoStream, FfiError>(video_stream)
}
// TODO(theomonnom): Support other stream types
_ => return Err(FfiError::InvalidRequest("unsupported video stream type")),
}?;
@@ -91,10 +89,6 @@ impl FfiVideoStream {
self.stream_type
}
pub fn track_sid(&self) -> &TrackSid {
&self.track_sid
}
async fn native_video_stream_task(
server: &'static server::FfiServer,
stream_handle_id: FfiHandleId,
@@ -143,12 +137,7 @@ impl FfiVideoStream {
pub struct FfiVideoSource {
handle_id: FfiHandleId,
source_type: proto::VideoSourceType,
source: VideoSource,
}
#[derive(Clone)]
pub enum VideoSource {
Native(NativeVideoSource),
source: RtcVideoSource,
}
impl FfiVideoSource {
@@ -157,11 +146,17 @@ impl FfiVideoSource {
new_source: proto::NewVideoSourceRequest,
) -> FfiResult<proto::VideoSourceInfo> {
let source_type = proto::VideoSourceType::from_i32(new_source.r#type).unwrap();
#[allow(unreachable_patterns)]
let source_inner = match source_type {
#[cfg(not(target_arch = "wasm32"))]
proto::VideoSourceType::VideoSourceNative => {
let video_source = NativeVideoSource::default();
VideoSource::Native(video_source)
use livekit::webrtc::video_source::native::NativeVideoSource;
let video_source = NativeVideoSource::new(
new_source.resolution.map(Into::into).unwrap_or_default(),
);
RtcVideoSource::Native(video_source)
}
_ => return Err(FfiError::InvalidRequest("unsupported video source type")),
};
let video_source = Self {
@@ -184,7 +179,8 @@ impl FfiVideoSource {
capture: proto::CaptureVideoFrameRequest,
) -> FfiResult<()> {
match self.source {
VideoSource::Native(ref source) => {
#[cfg(not(target_arch = "wasm32"))]
RtcVideoSource::Native(ref source) => {
let frame_info = capture
.frame
.ok_or(FfiError::InvalidRequest("frame is empty"))?;
@@ -206,12 +202,13 @@ impl FfiVideoSource {
let rotation = proto::VideoRotation::from_i32(frame_info.rotation).unwrap();
let frame = VideoFrame {
rotation: rotation.into(),
timestamp: frame_info.timestamp,
timestamp_us: frame_info.timestamp_us,
buffer,
};
source.capture_frame(&frame);
}
_ => {}
}
Ok(())
}
@@ -224,7 +221,7 @@ impl FfiVideoSource {
self.source_type
}
pub fn inner_source(&self) -> &VideoSource {
pub fn inner_source(&self) -> &RtcVideoSource {
&self.source
}
}