feat: video publishing (#42)

- Prepare webrtc abstraction ( for future wasm support )
- Added track publish support for videos
  - Added LogoTrack example to simple_room demo
- Lot of cleanup
- There are compiler warnings I'll solve on our v1 release

livekit-ffi/src/server/conversion/mod.rs

@@ -1,12 +1,12 @@
 use crate::{proto, server::FFIHandleId};
-use livekit::{
-    prelude::*,
-    webrtc::video_frame_buffer::{
-        BiplanarYuv8Buffer, BiplanarYuvBuffer, I010Buffer, I420ABuffer, I420Buffer, I422Buffer,
-        I444Buffer, NV12Buffer, PlanarYuv16BBuffer, PlanarYuv8Buffer, PlanarYuvBuffer,
-    },
-};
-use std::sync::Arc;
+use livekit::prelude::*;
+use livekit::webrtc::prelude::*;
+use std::any::Any;
+
+pub mod participant;
+pub mod publication;
+pub mod room;
+pub mod video_frame;
 
 impl From<FFIHandleId> for proto::FfiHandleId {
     fn from(id: FFIHandleId) -> Self {
@@ -30,8 +30,8 @@ macro_rules! impl_participant_into {
     };
 }
 
-impl_participant_into!(&Arc<LocalParticipant>);
-impl_participant_into!(&Arc<RemoteParticipant>);
+impl_participant_into!(&LocalParticipant);
+impl_participant_into!(&RemoteParticipant);
 impl_participant_into!(&Participant);
 
 impl From<TrackSource> for proto::TrackSource {
@@ -92,14 +92,13 @@ impl_track_into!(&LocalAudioTrack);
 impl_track_into!(&LocalVideoTrack);
 impl_track_into!(&RemoteAudioTrack);
 impl_track_into!(&RemoteVideoTrack);
-impl_track_into!(&TrackHandle);
-impl_track_into!(&LocalTrackHandle);
-impl_track_into!(&RemoteTrackHandle);
+impl_track_into!(&Track);
+impl_track_into!(&LocalTrack);
+impl_track_into!(&RemoteTrack);
 
 impl From<TrackKind> for proto::TrackKind {
     fn from(kind: TrackKind) -> Self {
         match kind {
-            TrackKind::Unknown => proto::TrackKind::KindUnknown,
             TrackKind::Audio => proto::TrackKind::KindAudio,
             TrackKind::Video => proto::TrackKind::KindVideo,
         }
@@ -109,7 +108,6 @@ impl From<TrackKind> for proto::TrackKind {
 impl From<StreamState> for proto::StreamState {
     fn from(state: StreamState) -> Self {
         match state {
-            StreamState::Unknown => Self::StateUnknown,
             StreamState::Active => Self::StateActive,
             StreamState::Paused => Self::StatePaused,
         }
@@ -182,136 +180,6 @@ impl proto::RoomEvent {
     }
 }
 
-impl From<VideoRotation> for proto::VideoRotation {
-    fn from(rotation: VideoRotation) -> proto::VideoRotation {
-        match rotation {
-            VideoRotation::VideoRotation0 => Self::VideoRotation0,
-            VideoRotation::VideoRotation90 => Self::VideoRotation90,
-            VideoRotation::VideoRotation180 => Self::VideoRotation180,
-            VideoRotation::VideoRotation270 => Self::VideoRotation270,
-        }
-    }
-}
-
-impl From<VideoFrame> for proto::VideoFrameInfo {
-    fn from(frame: VideoFrame) -> Self {
-        Self {
-            width: frame.width(),
-            height: frame.height(),
-            size: frame.size(),
-            id: frame.id() as u32,
-            timestamp_us: frame.timestamp_us(),
-            ntp_time_ms: frame.ntp_time_ms(),
-            transport_frame_id: frame.transport_frame_id(),
-            timestamp: frame.timestamp(),
-            rotation: proto::VideoRotation::from(frame.rotation()).into(),
-        }
-    }
-}
-
-impl From<VideoFrameBufferType> for proto::VideoFrameBufferType {
-    fn from(buffer_type: VideoFrameBufferType) -> Self {
-        match buffer_type {
-            VideoFrameBufferType::Native => Self::Native,
-            VideoFrameBufferType::I420 => Self::I420,
-            VideoFrameBufferType::I420A => Self::I420a,
-            VideoFrameBufferType::I422 => Self::I422,
-            VideoFrameBufferType::I444 => Self::I444,
-            VideoFrameBufferType::I010 => Self::I010,
-            VideoFrameBufferType::NV12 => Self::Nv12,
-        }
-    }
-}
-
-macro_rules! impl_yuv_into {
-    ($b:ty) => {
-        impl From<$b> for proto::PlanarYuvBufferInfo {
-            fn from(buffer: $b) -> Self {
-                Self {
-                    chroma_width: buffer.chroma_width(),
-                    chroma_height: buffer.chroma_height(),
-                    stride_y: buffer.stride_y(),
-                    stride_u: buffer.stride_u(),
-                    stride_v: buffer.stride_v(),
-                    data_y_ptr: buffer.data_y().as_ptr() as u64,
-                    data_u_ptr: buffer.data_u().as_ptr() as u64,
-                    data_v_ptr: buffer.data_v().as_ptr() as u64,
-                }
-            }
-        }
-    };
-}
-
-impl_yuv_into!(&I420Buffer);
-impl_yuv_into!(&I420ABuffer);
-impl_yuv_into!(&I422Buffer);
-impl_yuv_into!(&I444Buffer);
-impl_yuv_into!(&I010Buffer);
-
-macro_rules! impl_biyuv_into {
-    ($b:ty) => {
-        impl From<$b> for proto::BiplanarYuvBufferInfo {
-            fn from(buffer: $b) -> Self {
-                Self {
-                    chroma_width: buffer.chroma_width(),
-                    chroma_height: buffer.chroma_height(),
-                    stride_y: buffer.stride_y(),
-                    stride_uv: buffer.stride_uv(),
-                    data_y_ptr: buffer.data_y().as_ptr() as u64,
-                    data_uv_ptr: buffer.data_uv().as_ptr() as u64,
-                }
-            }
-        }
-    };
-}
-
-impl_biyuv_into!(&NV12Buffer);
-
-impl proto::VideoFrameBufferInfo {
-    pub fn from(handle_id: FFIHandleId, buffer: &VideoFrameBuffer) -> Self {
-        Self {
-            handle: Some(handle_id.into()),
-            buffer_type: proto::VideoFrameBufferType::from(buffer.buffer_type()).into(),
-            width: buffer.width(),
-            height: buffer.height(),
-            buffer: Some(match &buffer {
-                VideoFrameBuffer::Native(_) => {
-                    proto::video_frame_buffer_info::Buffer::Native(proto::NativeBufferInfo {})
-                }
-                VideoFrameBuffer::I420(i420) => {
-                    proto::video_frame_buffer_info::Buffer::Yuv(i420.into())
-                }
-                VideoFrameBuffer::I420A(i420a) => {
-                    proto::video_frame_buffer_info::Buffer::Yuv(i420a.into())
-                }
-                VideoFrameBuffer::I422(i422) => {
-                    proto::video_frame_buffer_info::Buffer::Yuv(i422.into())
-                }
-                VideoFrameBuffer::I444(i444) => {
-                    proto::video_frame_buffer_info::Buffer::Yuv(i444.into())
-                }
-                VideoFrameBuffer::I010(i010) => {
-                    proto::video_frame_buffer_info::Buffer::Yuv(i010.into())
-                }
-                VideoFrameBuffer::NV12(nv12) => {
-                    proto::video_frame_buffer_info::Buffer::BiYuv(nv12.into())
-                }
-            }),
-        }
-    }
-}
-
-impl From<proto::VideoFormatType> for VideoFormatType {
-    fn from(format: proto::VideoFormatType) -> Self {
-        match format {
-            proto::VideoFormatType::FormatArgb => Self::ARGB,
-            proto::VideoFormatType::FormatBgra => Self::BGRA,
-            proto::VideoFormatType::FormatAbgr => Self::ABGR,
-            proto::VideoFormatType::FormatRgba => Self::RGBA,
-        }
-    }
-}
-
 impl From<&RoomSession> for proto::RoomInfo {
     fn from(session: &RoomSession) -> Self {
         Self {

livekit-ffi/src/server/conversion/video_frame.rs

@@ -0,0 +1,203 @@
+use crate::{proto, server::FFIHandleId};
+use livekit::webrtc::prelude::*;
+use livekit::webrtc::video_frame;
+use std::any::Any;
+
+macro_rules! impl_yuv_into {
+    (@fields, $buffer:ident, $data_y:ident, $data_u:ident, $data_v: ident) => {
+        Self {
+            chroma_width: $buffer.chroma_width(),
+            chroma_height: $buffer.chroma_height(),
+            stride_y: $buffer.stride_y(),
+            stride_u: $buffer.stride_u(),
+            stride_v: $buffer.stride_v(),
+            data_y_ptr: $data_y.as_ptr() as u64,
+            data_u_ptr: $data_u.as_ptr() as u64,
+            data_v_ptr: $data_v.as_ptr() as u64,
+            ..Default::default()
+        }
+    };
+    ($buffer:ty, ALPHA) => {
+        impl From<$buffer> for proto::PlanarYuvBufferInfo {
+            fn from(buffer: $buffer) -> Self {
+                let (data_y, data_u, data_v, data_a) = buffer.data();
+                let mut proto = impl_yuv_into!(@fields, buffer, data_y, data_u, data_v);
+                proto.stride_a = buffer.stride_a();
+                proto.data_a_ptr = data_a.map(|data_a| data_a.as_ptr() as u64).unwrap_or(0);
+                proto
+            }
+        }
+    };
+    ($buffer:ty) => {
+        impl From<$buffer> for proto::PlanarYuvBufferInfo {
+            fn from(buffer: $buffer) -> Self {
+                let (data_y, data_u, data_v) = buffer.data();
+                impl_yuv_into!(@fields, buffer, data_y, data_u, data_v)
+            }
+        }
+    };
+}
+
+macro_rules! impl_biyuv_into {
+    ($b:ty) => {
+        impl From<$b> for proto::BiplanarYuvBufferInfo {
+            fn from(buffer: $b) -> Self {
+                let (data_y, data_uv) = buffer.data();
+                Self {
+                    chroma_width: buffer.chroma_width(),
+                    chroma_height: buffer.chroma_height(),
+                    stride_y: buffer.stride_y(),
+                    stride_uv: buffer.stride_uv(),
+                    data_y_ptr: data_y.as_ptr() as u64,
+                    data_uv_ptr: data_uv.as_ptr() as u64,
+                }
+            }
+        }
+    };
+}
+
+impl_yuv_into!(&I420Buffer);
+impl_yuv_into!(&I420ABuffer, ALPHA);
+impl_yuv_into!(&I422Buffer);
+impl_yuv_into!(&I444Buffer);
+impl_yuv_into!(&I010Buffer);
+impl_biyuv_into!(&NV12Buffer);
+
+impl proto::VideoFrameInfo {
+    pub fn from<T>(frame: &VideoFrame<T>) -> Self
+    where
+        T: VideoFrameBuffer,
+    {
+        Self {
+            timestamp: frame.timestamp,
+            rotation: proto::VideoRotation::from(frame.rotation).into(),
+        }
+    }
+}
+
+impl proto::VideoFrameBufferInfo {
+    pub fn from(handle: FFIHandleId, buffer: &dyn VideoFrameBuffer) -> Self {
+        match &buffer.buffer_type() {
+            #[cfg(not(target_arch = "wasm32"))]
+            VideoFrameBufferType::Native => Self::from_native(handle, buffer.as_native().unwrap()),
+            VideoFrameBufferType::I420 => Self::from_i420(handle, buffer.as_i420().unwrap()),
+            VideoFrameBufferType::I420A => Self::from_i420a(handle, buffer.as_i420a().unwrap()),
+            VideoFrameBufferType::I422 => Self::from_i422(handle, buffer.as_i422().unwrap()),
+            VideoFrameBufferType::I444 => Self::from_i444(handle, buffer.as_i444().unwrap()),
+            VideoFrameBufferType::I010 => Self::from_i010(handle, buffer.as_i010().unwrap()),
+            VideoFrameBufferType::NV12 => Self::from_nv12(handle, buffer.as_nv12().unwrap()),
+            _ => panic!("unsupported buffer type on this platform"),
+        }
+    }
+
+    #[cfg(not(target_arch = "wasm32"))]
+    pub fn from_native(handle_id: FFIHandleId, buffer: &video_frame::native::NativeBuffer) -> Self {
+        Self {
+            handle: Some(handle_id.into()),
+            buffer_type: proto::VideoFrameBufferType::Native.into(),
+            width: buffer.width(),
+            height: buffer.height(),
+            buffer: Some(proto::video_frame_buffer_info::Buffer::Native(
+                proto::NativeBufferInfo {},
+            )),
+        }
+    }
+
+    pub fn from_i420(handle_id: FFIHandleId, buffer: &I420Buffer) -> Self {
+        Self {
+            handle: Some(handle_id.into()),
+            buffer_type: proto::VideoFrameBufferType::I420.into(),
+            width: buffer.width(),
+            height: buffer.height(),
+            buffer: Some(proto::video_frame_buffer_info::Buffer::Yuv(buffer.into())),
+        }
+    }
+
+    pub fn from_i420a(handle_id: FFIHandleId, buffer: &I420ABuffer) -> Self {
+        Self {
+            handle: Some(handle_id.into()),
+            buffer_type: proto::VideoFrameBufferType::I420a.into(),
+            width: buffer.width(),
+            height: buffer.height(),
+            buffer: Some(proto::video_frame_buffer_info::Buffer::Yuv(buffer.into())),
+        }
+    }
+
+    pub fn from_i422(handle_id: FFIHandleId, buffer: &I422Buffer) -> Self {
+        Self {
+            handle: Some(handle_id.into()),
+            buffer_type: proto::VideoFrameBufferType::I422.into(),
+            width: buffer.width(),
+            height: buffer.height(),
+            buffer: Some(proto::video_frame_buffer_info::Buffer::Yuv(buffer.into())),
+        }
+    }
+
+    pub fn from_i444(handle_id: FFIHandleId, buffer: &I444Buffer) -> Self {
+        Self {
+            handle: Some(handle_id.into()),
+            buffer_type: proto::VideoFrameBufferType::I444.into(),
+            width: buffer.width(),
+            height: buffer.height(),
+            buffer: Some(proto::video_frame_buffer_info::Buffer::Yuv(buffer.into())),
+        }
+    }
+
+    pub fn from_i010(handle_id: FFIHandleId, buffer: &I010Buffer) -> Self {
+        Self {
+            handle: Some(handle_id.into()),
+            buffer_type: proto::VideoFrameBufferType::I010.into(),
+            width: buffer.width(),
+            height: buffer.height(),
+            buffer: Some(proto::video_frame_buffer_info::Buffer::Yuv(buffer.into())),
+        }
+    }
+
+    pub fn from_nv12(handle_id: FFIHandleId, buffer: &NV12Buffer) -> Self {
+        Self {
+            handle: Some(handle_id.into()),
+            buffer_type: proto::VideoFrameBufferType::Nv12.into(),
+            width: buffer.width(),
+            height: buffer.height(),
+            buffer: Some(proto::video_frame_buffer_info::Buffer::BiYuv(buffer.into())),
+        }
+    }
+}
+
+impl From<proto::VideoFormatType> for VideoFormatType {
+    fn from(format: proto::VideoFormatType) -> Self {
+        match format {
+            proto::VideoFormatType::FormatArgb => Self::ARGB,
+            proto::VideoFormatType::FormatBgra => Self::BGRA,
+            proto::VideoFormatType::FormatAbgr => Self::ABGR,
+            proto::VideoFormatType::FormatRgba => Self::RGBA,
+        }
+    }
+}
+
+impl From<VideoRotation> for proto::VideoRotation {
+    fn from(rotation: VideoRotation) -> proto::VideoRotation {
+        match rotation {
+            VideoRotation::VideoRotation0 => Self::VideoRotation0,
+            VideoRotation::VideoRotation90 => Self::VideoRotation90,
+            VideoRotation::VideoRotation180 => Self::VideoRotation180,
+            VideoRotation::VideoRotation270 => Self::VideoRotation270,
+        }
+    }
+}
+
+impl From<VideoFrameBufferType> for proto::VideoFrameBufferType {
+    fn from(buffer_type: VideoFrameBufferType) -> Self {
+        match buffer_type {
+            VideoFrameBufferType::Native => Self::Native,
+            VideoFrameBufferType::I420 => Self::I420,
+            VideoFrameBufferType::I420A => Self::I420a,
+            VideoFrameBufferType::I422 => Self::I422,
+            VideoFrameBufferType::I444 => Self::I444,
+            VideoFrameBufferType::I010 => Self::I010,
+            VideoFrameBufferType::NV12 => Self::Nv12,
+            VideoFrameBufferType::WebGl => Self::Webgl,
+            _ => panic!("unsupported buffer type on FFI server"),
+        }
+    }
+}

livekit-ffi/src/server/mod.rs

@@ -1,6 +1,7 @@
 use crate::proto;
 use lazy_static::lazy_static;
 use livekit::prelude::*;
+use livekit::webrtc::video_frame::{native::VideoFrameBufferExt, BoxVideoFrame, VideoFrameBuffer};
 use parking_lot::{Mutex, RwLock};
 use prost::Message;
 use std::any::Any;
@@ -166,11 +167,11 @@ impl FFIServer {
                 let buffer = self.release_handle(to_i420.buffer.unwrap().id as FFIHandleId);
 
                 if let Some(buffer) = buffer {
-                    if let Ok(buffer) = buffer.downcast::<VideoFrameBuffer>() {
+                    if let Ok(buffer) = buffer.downcast::<Box<dyn VideoFrameBuffer>>() {
                         let handle_id = self.next_handle_id();
-                        let i420 = VideoFrameBuffer::I420(buffer.to_i420());
-                        buffer_info = Some(proto::VideoFrameBufferInfo::from(handle_id, &i420));
-                        self.insert_handle(handle_id, Box::new(i420));
+                        let buffer = buffer.to_i420();
+                        buffer_info = Some(proto::VideoFrameBufferInfo::from(handle_id, &buffer));
+                        self.insert_handle(handle_id, Box::new(buffer));
                     }
                 }
 
@@ -188,7 +189,7 @@ impl FFIServer {
                 let buffer = ffi_owned.get(&(to_argb.buffer.unwrap().id as FFIHandleId));
 
                 if let Some(buffer) = buffer {
-                    if let Some(buffer) = buffer.downcast_ref::<VideoFrameBuffer>() {
+                    if let Some(buffer) = buffer.downcast_ref::<Box<dyn VideoFrameBuffer>>() {
                         let dst_buf = unsafe {
                             slice::from_raw_parts_mut(
                                 to_argb.dst_ptr as *mut u8,

livekit-ffi/src/server/room.rs

@@ -1,6 +1,8 @@
-use crate::proto::{self};
+use crate::proto;
 use crate::server::FFIServer;
+use futures_util::stream::StreamExt;
 use livekit::prelude::*;
+use livekit::webrtc::video_stream::native::NativeVideoStream;
 use tokio::sync::{mpsc, oneshot};
 
 pub async fn create_room(
@@ -67,9 +69,9 @@ async fn room_task(
                         publication: _,
                         participant: _,
                     } => {
-                        if let RemoteTrackHandle::Video(video_track) = track {
-                            let rtc_track = video_track.rtc_track();
-                            rtc_track.on_frame(on_video_frame(server, video_track.sid()));
+                        if let RemoteTrack::Video(video_track) = track {
+                            let video_stream = NativeVideoStream::new(video_track.rtc_track());
+                            tokio::spawn(video_frame_task(server, video_track.sid(), video_stream));
                         }
                     }
                     _ => {}
@@ -87,19 +89,20 @@ async fn room_task(
 async fn participant_task(participant: Participant) {
     let mut participant_events = participant.register_observer();
     while let Some(event) = participant_events.recv().await {
-        // TODO convert event to proto
+        // TODO(theomonnom): convert event to proto
     }
 }
 
-fn on_video_frame(server: &'static FFIServer, track_sid: TrackSid) -> OnFrameHandler {
-    // TODO(theomonnom): Should I use VideoSinkInfo here? (It'll help to have a more verbose
-    // lifetime)
-
-    Box::new(move |frame, buffer| {
-        // Frame received, create a new FFIHandle from the video buffer.
+async fn video_frame_task(
+    server: &'static FFIServer,
+    track_sid: TrackSid,
+    mut stream: NativeVideoStream,
+) {
+    while let Some(frame) = stream.next().await {
         let handle_id = server.next_handle_id();
-        let proto_buffer = proto::VideoFrameBufferInfo::from(handle_id, &buffer);
-        server.insert_handle(handle_id, Box::new(buffer));
+        let frame_info = proto::VideoFrameInfo::from(&frame);
+        let buffer_info = proto::VideoFrameBufferInfo::from(handle_id, &frame.buffer);
+        server.insert_handle(handle_id, Box::new(frame.buffer));
 
         // Send the received frame to the FFI language.
         let _ = server.send_event(
@@ -107,12 +110,12 @@ fn on_video_frame(server: &'static FFIServer, track_sid: TrackSid) -> OnFrameHan
                 track_sid: track_sid.to_string(),
                 message: Some(proto::track_event::Message::FrameReceived(
                     proto::FrameReceived {
-                        frame: Some(frame.into()),
-                        frame_buffer: Some(proto_buffer),
+                        frame: Some(frame_info),
+                        buffer: Some(buffer_info),
                     },
                 )),
             }),
             None,
         );
-    })
+    }
 }