talksik/client-sdk-rust
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: add ffi datachannel & mute events (#88)

Browse Source
This commit is contained in:
Théo Monnom
2023-06-18 22:45:05 +02:00
committed by GitHub
parent 33dfcb27b0
commit 25f4c9a075
69 changed files with 1934 additions and 1546 deletions
Download Patch File Download Diff File Expand all files Collapse all files
livekit-ffi/src/conversion/audio_frame.rs
+11 -1
View File
@@ -1,7 +1,18 @@
use crate::server::audio_frame::{FfiAudioSource, FfiAudioSream};
use crate::{proto, FfiHandleId};
use livekit::webrtc::audio_source::AudioSourceOptions;
use livekit::webrtc::prelude::*;
impl From<proto::AudioSourceOptions> for AudioSourceOptions {
fn from(opts: proto::AudioSourceOptions) -> Self {
Self {
echo_cancellation: opts.echo_cancellation,
auto_gain_control: opts.auto_gain_control,
noise_suppression: opts.noise_suppression,
}
}
}
impl proto::AudioFrameBufferInfo {
pub fn from(handle_id: FfiHandleId, buffer: &AudioFrame) -> Self {
Self {
@@ -20,7 +31,6 @@ impl From<&FfiAudioSream> for proto::AudioStreamInfo {
handle: Some(proto::FfiHandleId {
id: stream.handle_id() as u64,
}),
track_sid: stream.track_sid().clone().into(),
r#type: stream.stream_type() as i32,
}
}
livekit-ffi/src/conversion/room.rs
+16 -61
View File
@@ -1,67 +1,23 @@
use crate::{proto, FfiHandleId, INVALID_HANDLE};
use crate::{proto, FfiHandleId};
use livekit::options::{AudioEncoding, TrackPublishOptions, VideoEncoding};
use livekit::prelude::*;
impl proto::RoomEvent {
pub fn from(room_handle: FfiHandleId, event: RoomEvent) -> Option<Self> {
let message = match event {
RoomEvent::ParticipantConnected(participant) => Some(
proto::room_event::Message::ParticipantConnected(proto::ParticipantConnected {
info: Some((&participant).into()),
}),
),
RoomEvent::ParticipantDisconnected(participant) => {
Some(proto::room_event::Message::ParticipantDisconnected(
proto::ParticipantDisconnected {
info: Some((&participant).into()),
},
))
}
RoomEvent::TrackPublished {
publication,
participant,
} => Some(proto::room_event::Message::TrackPublished(
proto::TrackPublished {
participant_sid: participant.sid().to_string(),
publication: Some((&publication).into()),
},
)),
RoomEvent::TrackUnpublished {
publication,
participant,
} => Some(proto::room_event::Message::TrackUnpublished(
proto::TrackUnpublished {
participant_sid: participant.sid().to_string(),
publication_sid: publication.sid().into(),
},
)),
RoomEvent::TrackSubscribed {
track,
publication: _,
participant,
} => Some(proto::room_event::Message::TrackSubscribed(
proto::TrackSubscribed {
participant_sid: participant.sid().to_string(),
track: Some(proto::TrackInfo::from_remote_track(INVALID_HANDLE, &track)),
},
)),
RoomEvent::TrackUnsubscribed {
track,
publication: _,
participant,
} => Some(proto::room_event::Message::TrackUnsubscribed(
proto::TrackUnsubscribed {
participant_sid: participant.sid().to_string(),
track_sid: track.sid().to_string(),
},
)),
_ => None,
};
impl From<proto::RoomOptions> for RoomOptions {
fn from(value: proto::RoomOptions) -> Self {
Self {
adaptive_stream: value.adaptive_stream,
auto_subscribe: value.auto_subscribe,
dynacast: value.dynacast,
}
}
}
message.map(|message| proto::RoomEvent {
room_handle: Some(room_handle.into()),
message: Some(message),
})
impl From<proto::DataPacketKind> for DataPacketKind {
fn from(value: proto::DataPacketKind) -> Self {
match value {
proto::DataPacketKind::KindReliable => Self::Reliable,
proto::DataPacketKind::KindLossy => Self::Lossy,
}
}
}
@@ -93,7 +49,6 @@ impl From<proto::TrackPublishOptions> for TrackPublishOptions {
dtx: opts.dtx,
red: opts.red,
simulcast: opts.simulcast,
name: opts.name,
source: proto::TrackSource::from_i32(opts.source).unwrap().into(),
}
}
livekit-ffi/src/conversion/track.rs
+1 -20
View File
@@ -1,25 +1,6 @@
use crate::{proto, FfiHandleId};
use livekit::options::{AudioCaptureOptions, VideoCaptureOptions};
use livekit::prelude::*;
impl From<proto::VideoCaptureOptions> for VideoCaptureOptions {
fn from(opts: proto::VideoCaptureOptions) -> Self {
Self {
resolution: opts.resolution.unwrap_or_default().into(),
}
}
}
impl From<proto::AudioCaptureOptions> for AudioCaptureOptions {
fn from(opts: proto::AudioCaptureOptions) -> Self {
Self {
echo_cancellation: opts.echo_cancellation,
auto_gain_control: opts.auto_gain_control,
noise_suppression: opts.noise_suppression,
}
}
}
impl From<TrackSource> for proto::TrackSource {
fn from(source: TrackSource) -> proto::TrackSource {
match source {
@@ -63,7 +44,7 @@ macro_rules! impl_track_into {
#[allow(dead_code)]
pub fn $fnc(handle_id: FfiHandleId, track: $t) -> Self {
Self {
opt_handle: Some(handle_id.into()),
handle: Some(handle_id.into()),
name: track.name(),
stream_state: proto::StreamState::from(track.stream_state()).into(),
sid: track.sid().to_string(),
livekit-ffi/src/conversion/video_frame.rs
+11 -2
View File
@@ -4,6 +4,16 @@ use crate::FfiHandleId;
use livekit::options::{VideoCodec, VideoResolution};
use livekit::webrtc::prelude::*;
use livekit::webrtc::video_frame;
use livekit::webrtc::video_source::VideoResolution as VideoSourceResolution;
impl From<proto::VideoSourceResolution> for VideoSourceResolution {
fn from(res: proto::VideoSourceResolution) -> Self {
Self {
width: res.width,
height: res.height,
}
}
}
macro_rules! impl_yuv_into {
(@fields, $buffer:ident, $data_y:ident, $data_u:ident, $data_v: ident) => {
@@ -72,7 +82,7 @@ impl proto::VideoFrameInfo {
T: AsRef<dyn VideoFrameBuffer>,
{
Self {
timestamp: frame.timestamp,
timestamp_us: frame.timestamp_us,
rotation: proto::VideoRotation::from(frame.rotation).into(),
}
}
@@ -236,7 +246,6 @@ impl From<&FfiVideoStream> for proto::VideoStreamInfo {
handle: Some(proto::FfiHandleId {
id: stream.handle_id() as u64,
}),
track_sid: stream.track_sid().clone().into(),
r#type: stream.stream_type() as i32,
}
}
livekit-ffi/src/server/audio_frame.rs
+32 -34
View File
@@ -2,11 +2,9 @@ use crate::{proto, server, FfiError, FfiHandleId, FfiResult};
use futures_util::StreamExt;
use livekit::prelude::*;
use livekit::webrtc::audio_frame::AudioFrame;
use livekit::webrtc::audio_source::native::NativeAudioSource;
use livekit::webrtc::audio_stream::native::NativeAudioStream;
use livekit::webrtc::prelude::*;
use log::warn;
use server::utils;
use tokio::sync::oneshot;
// ===== FFIAudioStream =====
@@ -14,7 +12,6 @@ use tokio::sync::oneshot;
pub struct FfiAudioSream {
handle_id: FfiHandleId,
stream_type: proto::AudioStreamType,
track_sid: TrackSid,
#[allow(dead_code)]
close_tx: oneshot::Sender<()>, // Close the stream on drop
@@ -35,42 +32,45 @@ impl FfiAudioSream {
) -> FfiResult<proto::AudioStreamInfo> {
let (close_tx, close_rx) = oneshot::channel();
let stream_type = proto::AudioStreamType::from_i32(new_stream.r#type).unwrap();
let track_sid: TrackSid = new_stream.track_sid.into();
let room_handle = new_stream
.room_handle
.ok_or(FfiError::InvalidRequest("room_handle is empty"))?
let handle_id = new_stream
.track_handle
.ok_or(FfiError::InvalidRequest("track_handle is empty"))?
.id as FfiHandleId;
let track = utils::find_remote_track(
server,
&track_sid,
&new_stream.participant_sid.into(),
room_handle,
)?
.rtc_track();
let track = server
.ffi_handles()
.get(&handle_id)
.ok_or(FfiError::InvalidRequest("track not found"))?;
let MediaStreamTrack::Audio(track) = track else {
let track = track
.downcast_ref::<Track>()
.ok_or(FfiError::InvalidRequest("handle is not a Track"))?;
let rtc_track = track.rtc_track();
let MediaStreamTrack::Audio(rtc_track) = rtc_track else {
return Err(FfiError::InvalidRequest("not an audio track"));
};
let audio_stream = match stream_type {
#[cfg(not(target_arch = "wasm32"))]
proto::AudioStreamType::AudioStreamNative => {
let audio_stream = Self {
handle_id: server.next_id(),
stream_type,
close_tx,
track_sid,
};
let native_stream = NativeAudioStream::new(rtc_track);
server.async_runtime.spawn(Self::native_audio_stream_task(
server,
audio_stream.handle_id,
NativeAudioStream::new(track),
native_stream,
close_rx,
));
Ok::<FfiAudioSream, FfiError>(audio_stream)
}
// TODO(theomonnom): Support other stream types
_ => return Err(FfiError::InvalidRequest("unsupported audio stream type")),
}?;
@@ -91,10 +91,6 @@ impl FfiAudioSream {
self.stream_type
}
pub fn track_sid(&self) -> &TrackSid {
&self.track_sid
}
async fn native_audio_stream_task(
server: &'static server::FfiServer,
stream_handle_id: FfiHandleId,
@@ -139,12 +135,7 @@ impl FfiAudioSream {
pub struct FfiAudioSource {
handle_id: FfiHandleId,
source_type: proto::AudioSourceType,
source: AudioSource,
}
#[derive(Clone)]
pub enum AudioSource {
Native(NativeAudioSource),
source: RtcAudioSource,
}
impl FfiAudioSource {
@@ -153,12 +144,17 @@ impl FfiAudioSource {
new_source: proto::NewAudioSourceRequest,
) -> FfiResult<proto::AudioSourceInfo> {
let source_type = proto::AudioSourceType::from_i32(new_source.r#type).unwrap();
#[allow(unreachable_patterns)]
let source_inner = match source_type {
#[cfg(not(target_arch = "wasm32"))]
proto::AudioSourceType::AudioSourceNative => {
let audio_source = NativeAudioSource::default();
Ok::<AudioSource, FfiError>(AudioSource::Native(audio_source))
} //_ => return Err(FfiError::InvalidRequest("unsupported audio source type")),
}?;
use livekit::webrtc::audio_source::native::NativeAudioSource;
let audio_source =
NativeAudioSource::new(new_source.options.map(Into::into).unwrap_or_default());
RtcAudioSource::Native(audio_source)
}
_ => return Err(FfiError::InvalidRequest("unsupported audio source type")),
};
let audio_source = Self {
handle_id: server.next_id(),
@@ -180,7 +176,8 @@ impl FfiAudioSource {
capture: proto::CaptureAudioFrameRequest,
) -> FfiResult<()> {
match self.source {
AudioSource::Native(ref source) => {
#[cfg(not(target_arch = "wasm32"))]
RtcAudioSource::Native(ref source) => {
let buffer_handle = capture
.buffer_handle
.ok_or(FfiError::InvalidRequest("buffer_handle is empty"))?
@@ -197,6 +194,7 @@ impl FfiAudioSource {
source.capture_frame(frame);
}
_ => {}
}
Ok(())
@@ -210,7 +208,7 @@ impl FfiAudioSource {
self.source_type
}
pub fn inner_source(&self) -> &AudioSource {
pub fn inner_source(&self) -> &RtcAudioSource {
&self.source
}
}
livekit-ffi/src/server/mod.rs
+70 -30
View File
@@ -15,7 +15,6 @@ use std::sync::Arc;
pub mod audio_frame;
pub mod room;
pub mod utils;
pub mod video_frame;
#[cfg(test)]
@@ -31,7 +30,7 @@ pub struct FfiConfig {
pub struct FfiServer {
rooms: Mutex<HashMap<RoomSid, FfiHandleId>>,
/// Store all FFI handles inside an HashMap, if this isn't efficient enough
/// Store all Ffi handles inside an HashMap, if this isn't efficient enough
/// We can still use Box::into_raw & Box::from_raw in the future (but keep it safe for now)
ffi_handles: DashMap<FfiHandleId, FfiHandle>,
next_id: AtomicUsize,
@@ -116,11 +115,9 @@ impl FfiServer {
}
// # SAFETY: The foreign language is responsible for ensuring that the callback function is valid
unsafe {
*self.config.lock() = Some(FfiConfig {
callback_fn: std::mem::transmute(init.event_callback_ptr),
});
}
*self.config.lock() = Some(FfiConfig {
callback_fn: unsafe { std::mem::transmute(init.event_callback_ptr) },
});
Ok(proto::InitializeResponse::default())
}
@@ -191,9 +188,37 @@ impl FfiServer {
fn on_disconnect(
&'static self,
_disconnect: proto::DisconnectRequest,
disconnect: proto::DisconnectRequest,
) -> FfiResult<proto::DisconnectResponse> {
Ok(proto::DisconnectResponse::default())
let async_id = self.next_id() as FfiAsyncId;
let room_handle = disconnect
.room_handle
.as_ref()
.ok_or(FfiError::InvalidRequest("room_handle is empty"))?
.id as FfiHandleId;
let ffi_room = self
.ffi_handles
.remove(&room_handle)
.ok_or(FfiError::InvalidRequest("room not found"))?
.1;
let ffi_room = ffi_room
.downcast::<room::FfiRoom>()
.map_err(|_| FfiError::InvalidRequest("room is not a FfiRoom"))?;
self.async_runtime.spawn(async move {
ffi_room.close().await;
let _ = self.send_event(proto::ffi_event::Message::Disconnect(
proto::DisconnectCallback {
async_id: Some(async_id.into()),
},
));
});
Ok(proto::DisconnectResponse {
async_id: Some(async_id.into()),
})
}
fn on_publish_track(
@@ -230,14 +255,17 @@ impl FfiServer {
.ok_or(FfiError::InvalidRequest("track not found"))?;
let track = track
.downcast_ref::<LocalTrack>()
.ok_or(FfiError::InvalidRequest("track is not a LocalTrack"))?;
.downcast_ref::<Track>()
.ok_or(FfiError::InvalidRequest("track is not a Track"))?;
let local_track = LocalTrack::try_from(track.clone())
.map_err(|_| FfiError::InvalidRequest("track is not a LocalTrack"))?;
let publication = ffi_room
.room()
.local_participant()
.publish_track(
track.clone(),
local_track,
publish.options.map(Into::into).unwrap_or_default(),
)
.await?;
@@ -269,6 +297,29 @@ impl FfiServer {
Ok(proto::UnpublishTrackResponse::default())
}
fn on_publish_data(
&'static self,
publish: proto::PublishDataRequest,
) -> FfiResult<proto::PublishDataResponse> {
let room_handle = publish
.room_handle
.as_ref()
.ok_or(FfiError::InvalidRequest("room_handle is empty"))?
.id as FfiHandleId;
let ffi_room = self
.ffi_handles
.get(&room_handle)
.ok_or(FfiError::InvalidRequest("room not found"))?;
let ffi_room = ffi_room
.downcast_ref::<room::FfiRoom>()
.ok_or(FfiError::InvalidRequest("room is not a FfiRoom"))?;
// Push the data to an async queue (avoid blocking and keep the order)
ffi_room.publish_data(self, publish)
}
// Track
fn on_create_video_track(
&'static self,
@@ -276,7 +327,6 @@ impl FfiServer {
) -> FfiResult<proto::CreateVideoTrackResponse> {
let handle_id = create
.source_handle
.as_ref()
.ok_or(FfiError::InvalidRequest("source_handle is empty"))?
.id as FfiHandleId;
@@ -290,19 +340,13 @@ impl FfiServer {
.ok_or(FfiError::InvalidRequest("handle is not a video source"))?;
let source = source.inner_source().clone();
let video_track = match source {
video_frame::VideoSource::Native(native_source) => LocalVideoTrack::create_video_track(
&create.name,
create.options.unwrap_or_default().into(),
native_source,
),
};
let video_track = LocalVideoTrack::create_video_track(&create.name, source);
let handle_id = self.next_id() as FfiHandleId;
let track_info = proto::TrackInfo::from_local_video_track(handle_id, &video_track);
self.ffi_handles
.insert(handle_id, Box::new(LocalTrack::Video(video_track)));
.insert(handle_id, Box::new(Track::LocalVideo(video_track)));
Ok(proto::CreateVideoTrackResponse {
track: Some(track_info),
@@ -315,7 +359,6 @@ impl FfiServer {
) -> FfiResult<proto::CreateAudioTrackResponse> {
let handle_id = create
.source_handle
.as_ref()
.ok_or(FfiError::InvalidRequest("source_handle is empty"))?
.id as FfiHandleId;
@@ -329,19 +372,13 @@ impl FfiServer {
.ok_or(FfiError::InvalidRequest("handle is not an audio source"))?;
let source = source.inner_source().clone();
let audio_track = match source {
audio_frame::AudioSource::Native(native_source) => LocalAudioTrack::create_audio_track(
&create.name,
create.options.unwrap_or_default().into(),
native_source,
),
};
let audio_track = LocalAudioTrack::create_audio_track(&create.name, source);
let handle_id = self.next_id() as FfiHandleId;
let track_info = proto::TrackInfo::from_local_audio_track(handle_id, &audio_track);
self.ffi_handles
.insert(handle_id, Box::new(LocalTrack::Audio(audio_track)));
.insert(handle_id, Box::new(Track::LocalAudio(audio_track)));
Ok(proto::CreateAudioTrackResponse {
track: Some(track_info),
@@ -714,6 +751,9 @@ impl FfiServer {
proto::ffi_request::Message::UnpublishTrack(unpublish) => {
proto::ffi_response::Message::UnpublishTrack(self.on_unpublish_track(unpublish)?)
}
proto::ffi_request::Message::PublishData(publish) => {
proto::ffi_response::Message::PublishData(self.on_publish_data(publish)?)
}
proto::ffi_request::Message::CreateVideoTrack(create) => {
proto::ffi_response::Message::CreateVideoTrack(self.on_create_video_track(create)?)
}
livekit-ffi/src/server/room.rs
+165 -22
View File
@@ -1,14 +1,24 @@
use crate::server::FfiServer;
use crate::{proto, FfiHandleId, FfiResult};
use crate::{proto, FfiAsyncId, FfiError, FfiHandleId, FfiResult};
use livekit::prelude::*;
use std::slice;
use std::sync::Arc;
use tokio::sync::{mpsc, oneshot};
use tokio::sync::{broadcast, mpsc};
use tokio::task::JoinHandle;
struct DataPacket {
data: Vec<u8>,
kind: DataPacketKind,
destination_sids: Vec<String>,
async_id: FfiAsyncId,
}
pub struct FfiRoom {
room: Arc<Room>,
handle: JoinHandle<()>,
close_tx: oneshot::Sender<()>,
event_handle: JoinHandle<()>,
data_handle: JoinHandle<()>,
close_tx: broadcast::Sender<()>,
data_tx: mpsc::UnboundedSender<DataPacket>,
}
impl FfiRoom {
@@ -16,33 +26,77 @@ impl FfiRoom {
server: &'static FfiServer,
connect: proto::ConnectRequest,
) -> FfiResult<proto::RoomInfo> {
let (room, events) = Room::connect(&connect.url, &connect.token).await?;
let (room, events) = Room::connect(
&connect.url,
&connect.token,
connect.options.map(Into::into).unwrap_or_default(),
)
.await?;
let room = Arc::new(room);
let (close_tx, close_rx) = oneshot::channel();
let next_id = server.next_id() as FfiHandleId;
let (close_tx, close_rx) = broadcast::channel(1);
let (data_tx, data_rx) = mpsc::unbounded_channel();
let handle =
let next_id = server.next_id() as FfiHandleId;
let event_handle = server.async_runtime.spawn(room_task(
server,
room.clone(),
next_id,
events,
close_rx.resubscribe(),
));
let data_handle =
server
.async_runtime
.spawn(room_task(server, room.clone(), next_id, events, close_rx));
let room_info = proto::RoomInfo::from_room(next_id, &room);
.spawn(data_task(server, room.clone(), data_rx, close_rx));
let ffi_room = Self {
room: room.clone(),
handle,
event_handle,
data_handle,
close_tx,
data_tx,
};
server.ffi_handles().insert(next_id, Box::new(ffi_room));
server.rooms().lock().insert(room.sid(), next_id);
let room_info = proto::RoomInfo::from_room(next_id, &room);
Ok(room_info)
}
pub fn publish_data(
&self,
server: &'static FfiServer,
publish: proto::PublishDataRequest,
) -> FfiResult<proto::PublishDataResponse> {
let data = unsafe {
slice::from_raw_parts(publish.data_ptr as *const u8, publish.data_size as usize)
};
let kind = proto::DataPacketKind::from_i32(publish.kind).unwrap();
let destination_sids: Vec<String> = publish.destination_sids;
let async_id = server.next_id() as FfiAsyncId;
let packet = DataPacket {
data: data.to_vec(), // Avoid copy?
kind: kind.into(),
destination_sids,
async_id,
};
self.data_tx
.send(packet)
.map_err(|_| FfiError::InvalidRequest("failed to send data packet"))?;
Ok(proto::PublishDataResponse {
async_id: Some(async_id.into()),
})
}
pub async fn close(self) {
let _ = self.room.close().await;
let _ = self.close_tx.send(());
let _ = self.handle.await;
let _ = self.event_handle.await;
let _ = self.data_handle.await;
}
pub fn room(&self) -> &Arc<Room> {
@@ -50,12 +104,41 @@ impl FfiRoom {
}
}
async fn data_task(
server: &'static FfiServer,
room: Arc<Room>,
mut data_rx: mpsc::UnboundedReceiver<DataPacket>,
mut close_rx: broadcast::Receiver<()>,
) {
loop {
tokio::select! {
Some(event) = data_rx.recv() => {
let res = room.local_participant().publish_data(
event.data,
event.kind,
event.destination_sids,
).await;
let cb = proto::PublishDataCallback {
async_id: Some(event.async_id.into()),
error: res.err().map(|e| e.to_string()),
};
let _ = server.send_event(proto::ffi_event::Message::PublishData(cb));
},
_ = close_rx.recv() => {
break;
}
}
}
}
async fn room_task(
server: &'static FfiServer,
room: Arc<Room>,
room_handle: FfiHandleId,
mut events: mpsc::UnboundedReceiver<livekit::RoomEvent>,
mut close_rx: oneshot::Receiver<()>,
mut close_rx: broadcast::Receiver<()>,
) {
server
.async_runtime
@@ -66,18 +149,78 @@ async fn room_task(
loop {
tokio::select! {
Some(event) = events.recv() => {
if let Some(event) = proto::RoomEvent::from(room_handle, event.clone()) {
let _ = server.send_event(proto::ffi_event::Message::RoomEvent(event));
let message = match event {
RoomEvent::ParticipantConnected(participant) => {
server.async_runtime.spawn(participant_task(Participant::Remote(participant.clone())));
Some(proto::room_event::Message::ParticipantConnected(
proto::ParticipantConnected {
info: Some(proto::ParticipantInfo::from(&participant)),
}
))
},
RoomEvent::ParticipantDisconnected(participant) => {
Some(proto::room_event::Message::ParticipantDisconnected(
proto::ParticipantDisconnected {
info: Some(proto::ParticipantInfo::from(&participant)),
},
))
}
RoomEvent::TrackPublished {
publication,
participant,
} => Some(proto::room_event::Message::TrackPublished(
proto::TrackPublished {
participant_sid: participant.sid().to_string(),
publication: Some(proto::TrackPublicationInfo::from(&publication))
},
)),
RoomEvent::TrackUnpublished {
publication,
participant,
} => Some(proto::room_event::Message::TrackUnpublished(
proto::TrackUnpublished {
participant_sid: participant.sid().to_string(),
publication_sid: publication.sid().into(),
},
)),
RoomEvent::TrackSubscribed {
track,
publication: _,
participant,
} => {
let handle_id = server.next_id() as FfiHandleId;
let track_info = proto::TrackInfo::from_remote_track(handle_id, &track);
server.ffi_handles().insert(handle_id, Box::new(Track::from(track)));
Some(proto::room_event::Message::TrackSubscribed(
proto::TrackSubscribed {
participant_sid: participant.sid().to_string(),
track: Some(track_info),
},
))
},
RoomEvent::TrackUnsubscribed {
track,
publication: _,
participant,
} => Some(proto::room_event::Message::TrackUnsubscribed(
proto::TrackUnsubscribed {
participant_sid: participant.sid().to_string(),
track_sid: track.sid().to_string(),
},
)),
_ => None
};
if message.is_some() {
let _ = server.send_event(proto::ffi_event::Message::RoomEvent(proto::RoomEvent{
room_handle: Some(room_handle.into()),
message
}));
}
match event {
RoomEvent::ParticipantConnected(p) => {
server.async_runtime.spawn(participant_task(Participant::Remote(p)));
}
_ => {}
}
},
_ = &mut close_rx => {
_ = close_rx.recv() => {
break;
}
};
livekit-ffi/src/server/tests.rs
+10 -14
View File
@@ -212,18 +212,22 @@ fn publish_video_track() {
client::FfiHandle(connect.room.unwrap().handle.unwrap().id as FfiHandleId);
// Create a new VideoSource
const VIDEO_WIDTH: u32 = 640;
const VIDEO_HEIGHT: u32 = 480;
const VIDEO_FPS: f64 = 8.0;
let res = client.send_request(proto::FfiRequest {
message: Some(proto::ffi_request::Message::NewVideoSource(
proto::NewVideoSourceRequest {
r#type: proto::VideoSourceType::VideoSourceNative as i32,
resolution: Some(proto::VideoSourceResolution {
width: VIDEO_WIDTH,
height: VIDEO_HEIGHT,
}),
},
)),
});
const VIDEO_WIDTH: u32 = 640;
const VIDEO_HEIGHT: u32 = 480;
const VIDEO_FPS: f64 = 8.0;
let proto::ffi_response::Message::NewVideoSource(new_video_source) =
res.message.unwrap() else {
panic!("unexpected response");
@@ -241,13 +245,6 @@ fn publish_video_track() {
source_handle: Some(proto::FfiHandleId {
id: source_handle.0 as u64,
}),
options: Some(proto::VideoCaptureOptions {
resolution: Some(proto::VideoResolution {
width: VIDEO_WIDTH,
height: VIDEO_HEIGHT,
frame_rate: VIDEO_FPS,
}),
}),
},
)),
});
@@ -258,11 +255,10 @@ fn publish_video_track() {
};
let track_handle = client::FfiHandle(
create_video_track.track.unwrap().opt_handle.unwrap().id as FfiHandleId,
create_video_track.track.unwrap().handle.unwrap().id as FfiHandleId,
);
let publish_options = proto::TrackPublishOptions {
name: "video_test".to_string(),
video_codec: proto::VideoCodec::H264 as i32,
source: proto::TrackSource::SourceCamera as i32,
..Default::default()
@@ -320,7 +316,7 @@ fn publish_video_track() {
id: buffer_handle.0 as u64,
}),
frame: Some(proto::VideoFrameInfo {
timestamp: 0, // TODO
timestamp_us: 0,
rotation: proto::VideoRotation::VideoRotation0 as i32,
}),
},
livekit-ffi/src/server/utils.rs
-32
View File
@@ -1,32 +0,0 @@
use crate::{server, FfiError, FfiHandleId, FfiResult};
use livekit::prelude::*;
pub fn find_remote_track(
server: &'static server::FfiServer,
track_sid: &TrackSid,
participant_sid: &ParticipantSid,
room_handle: FfiHandleId,
) -> FfiResult<RemoteTrack> {
let ffi_room = server
.ffi_handles()
.get(&room_handle)
.ok_or(FfiError::InvalidRequest("room not found"))?;
let ffi_room = ffi_room
.downcast_ref::<server::room::FfiRoom>()
.ok_or(FfiError::InvalidRequest("room is not ffi room"))?;
let room = ffi_room.room();
let participants = room.participants();
let participant = participants
.get(participant_sid)
.ok_or(FfiError::InvalidRequest("participant not found"))?;
let track = participant
.get_track_publication(track_sid)
.ok_or(FfiError::InvalidRequest("publication not found"))?
.track()
.ok_or(FfiError::InvalidRequest("track not found/subscribed"))?;
Ok(track)
}
livekit-ffi/src/server/video_frame.rs
+30 -33
View File
@@ -3,10 +3,8 @@ use futures_util::StreamExt;
use livekit::prelude::*;
use livekit::webrtc::prelude::*;
use livekit::webrtc::video_frame::{BoxVideoFrameBuffer, VideoFrame};
use livekit::webrtc::video_source::native::NativeVideoSource;
use livekit::webrtc::video_stream::native::NativeVideoStream;
use log::warn;
use server::utils;
use tokio::sync::oneshot;
// ===== FFIVideoStream =====
@@ -14,7 +12,6 @@ use tokio::sync::oneshot;
pub struct FfiVideoStream {
handle_id: FfiHandleId,
stream_type: proto::VideoStreamType,
track_sid: TrackSid,
#[allow(dead_code)]
close_tx: oneshot::Sender<()>, // Close the stream on drop
@@ -35,42 +32,43 @@ impl FfiVideoStream {
) -> FfiResult<proto::VideoStreamInfo> {
let (close_tx, close_rx) = oneshot::channel();
let stream_type = proto::VideoStreamType::from_i32(new_stream.r#type).unwrap();
let track_sid: TrackSid = new_stream.track_sid.into();
let room_handle = new_stream
.room_handle
.ok_or(FfiError::InvalidRequest("room_handle is empty"))?
let handle_id = new_stream
.track_handle
.ok_or(FfiError::InvalidRequest("track_handle is empty"))?
.id as FfiHandleId;
let track = utils::find_remote_track(
server,
&track_sid,
&new_stream.participant_sid.into(),
room_handle,
)?
.rtc_track();
let track = server
.ffi_handles()
.get(&handle_id)
.ok_or(FfiError::InvalidRequest("track not found"))?;
let MediaStreamTrack::Video(track) = track else {
let track = track
.downcast_ref::<Track>()
.ok_or(FfiError::InvalidRequest("handle is not a Track"))?;
let rtc_track = track.rtc_track();
let MediaStreamTrack::Video(rtc_track) = rtc_track else {
return Err(FfiError::InvalidRequest("not a video track"));
};
let stream = match stream_type {
#[cfg(not(target_arch = "wasm32"))]
proto::VideoStreamType::VideoStreamNative => {
let video_stream = Self {
handle_id: server.next_id(),
close_tx,
stream_type,
track_sid,
};
server.async_runtime.spawn(Self::native_video_stream_task(
server,
video_stream.handle_id,
NativeVideoStream::new(track),
NativeVideoStream::new(rtc_track),
close_rx,
));
Ok::<FfiVideoStream, FfiError>(video_stream)
}
// TODO(theomonnom): Support other stream types
_ => return Err(FfiError::InvalidRequest("unsupported video stream type")),
}?;
@@ -91,10 +89,6 @@ impl FfiVideoStream {
self.stream_type
}
pub fn track_sid(&self) -> &TrackSid {
&self.track_sid
}
async fn native_video_stream_task(
server: &'static server::FfiServer,
stream_handle_id: FfiHandleId,
@@ -143,12 +137,7 @@ impl FfiVideoStream {
pub struct FfiVideoSource {
handle_id: FfiHandleId,
source_type: proto::VideoSourceType,
source: VideoSource,
}
#[derive(Clone)]
pub enum VideoSource {
Native(NativeVideoSource),
source: RtcVideoSource,
}
impl FfiVideoSource {
@@ -157,11 +146,17 @@ impl FfiVideoSource {
new_source: proto::NewVideoSourceRequest,
) -> FfiResult<proto::VideoSourceInfo> {
let source_type = proto::VideoSourceType::from_i32(new_source.r#type).unwrap();
#[allow(unreachable_patterns)]
let source_inner = match source_type {
#[cfg(not(target_arch = "wasm32"))]
proto::VideoSourceType::VideoSourceNative => {
let video_source = NativeVideoSource::default();
VideoSource::Native(video_source)
use livekit::webrtc::video_source::native::NativeVideoSource;
let video_source = NativeVideoSource::new(
new_source.resolution.map(Into::into).unwrap_or_default(),
);
RtcVideoSource::Native(video_source)
}
_ => return Err(FfiError::InvalidRequest("unsupported video source type")),
};
let video_source = Self {
@@ -184,7 +179,8 @@ impl FfiVideoSource {
capture: proto::CaptureVideoFrameRequest,
) -> FfiResult<()> {
match self.source {
VideoSource::Native(ref source) => {
#[cfg(not(target_arch = "wasm32"))]
RtcVideoSource::Native(ref source) => {
let frame_info = capture
.frame
.ok_or(FfiError::InvalidRequest("frame is empty"))?;
@@ -206,12 +202,13 @@ impl FfiVideoSource {
let rotation = proto::VideoRotation::from_i32(frame_info.rotation).unwrap();
let frame = VideoFrame {
rotation: rotation.into(),
timestamp: frame_info.timestamp,
timestamp_us: frame_info.timestamp_us,
buffer,
};
source.capture_frame(&frame);
}
_ => {}
}
Ok(())
}
@@ -224,7 +221,7 @@ impl FfiVideoSource {
self.source_type
}
pub fn inner_source(&self) -> &VideoSource {
pub fn inner_source(&self) -> &RtcVideoSource {
&self.source
}
}