diff --git a/Cargo.lock b/Cargo.lock index 24d3033..d10d828 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -250,6 +250,19 @@ dependencies = [ "syn 2.0.15", ] +[[package]] +name = "dashmap" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" +dependencies = [ + "cfg-if", + "hashbrown", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "digest" version = "0.10.6" @@ -794,10 +807,13 @@ dependencies = [ name = "livekit-ffi" version = "0.1.1" dependencies = [ + "dashmap", "futures-util", "lazy_static", "livekit", + "livekit-api", "livekit-protocol", + "log", "parking_lot", "prost", "prost-build", diff --git a/examples/simple_room/src/logo_track.rs b/examples/simple_room/src/logo_track.rs index 46c5c6a..eaa24e2 100644 --- a/examples/simple_room/src/logo_track.rs +++ b/examples/simple_room/src/logo_track.rs @@ -156,9 +156,7 @@ impl LogoTrack { let mut video_frame = data.video_frame.lock(); let i420_buffer = &mut video_frame.buffer; - let stride_y = i420_buffer.stride_y(); - let stride_u = i420_buffer.stride_u(); - let stride_v = i420_buffer.stride_v(); + let (stride_y, stride_u, stride_v) = i420_buffer.strides(); let (data_y, data_u, data_v) = i420_buffer.data_mut(); framebuffer.fill(0); @@ -177,7 +175,7 @@ impl LogoTrack { yuv_helper::abgr_to_i420( &framebuffer, - (FB_WIDTH * PIXEL_SIZE) as i32, + (FB_WIDTH * PIXEL_SIZE) as u32, data_y, stride_y, data_u, diff --git a/examples/simple_room/src/sine_track.rs b/examples/simple_room/src/sine_track.rs index 43231a9..29ac739 100644 --- a/examples/simple_room/src/sine_track.rs +++ b/examples/simple_room/src/sine_track.rs @@ -116,11 +116,11 @@ impl SineTrack { samples_10ms[i] = (val * 32768.0) as i16; } - rtc_source.capture_frame(AudioFrame { + rtc_source.capture_frame(&AudioFrame { data: samples_10ms.clone(), - sample_rate_hz: data.sample_rate, + sample_rate: data.sample_rate as u32, num_channels: 1, - samples_per_channel: samples_count_10ms, + samples_per_channel: samples_count_10ms as u32, }); } } diff --git a/examples/simple_room/src/video_renderer.rs b/examples/simple_room/src/video_renderer.rs index 5e24b9f..2e7e264 100644 --- a/examples/simple_room/src/video_renderer.rs +++ b/examples/simple_room/src/video_renderer.rs @@ -119,19 +119,20 @@ impl VideoRenderer { let rgba_ptr = internal.rgba_data.deref_mut(); let rgba_stride = buffer.width() * 4; + let (stride_y, stride_u, stride_v) = buffer.strides(); let (data_y, data_u, data_v) = buffer.data(); yuv_helper::i420_to_abgr( data_y, - buffer.stride_y(), + stride_y, data_u, - buffer.stride_u(), + stride_u, data_v, - buffer.stride_v(), + stride_v, rgba_ptr, rgba_stride, - buffer.width(), - buffer.height(), + buffer.width() as i32, + buffer.height() as i32, ) .unwrap(); diff --git a/livekit-api/src/access_token.rs b/livekit-api/src/access_token.rs index 977d3ef..0731ce7 100644 --- a/livekit-api/src/access_token.rs +++ b/livekit-api/src/access_token.rs @@ -64,7 +64,7 @@ impl Default for VideoGrants { room_list: false, room_record: false, room_admin: false, - room_join: false, + room_join: true, room: "".to_string(), can_publish: true, can_subscribe: true, diff --git a/livekit-ffi/Cargo.toml b/livekit-ffi/Cargo.toml index e3fc048..b40454b 100644 --- a/livekit-ffi/Cargo.toml +++ b/livekit-ffi/Cargo.toml @@ -16,9 +16,17 @@ prost-types = "0.11.1" lazy_static = "1.4.0" thiserror = "1.0.38" futures-util = "0.3.23" +log = "0.4.17" +dashmap = "5.4.0" [build-dependencies] prost-build = { version = "0.11.1" } +[dev-dependencies] +livekit-api = { path = "../livekit-api", version = "0.1.0" } + [lib] crate-type = ["cdylib", "staticlib"] + +[profile.release] +opt-level = "z" diff --git a/livekit-ffi/build.rs b/livekit-ffi/build.rs index 89dfc6e..0a68f58 100644 --- a/livekit-ffi/build.rs +++ b/livekit-ffi/build.rs @@ -1,7 +1,17 @@ use std::io::Result; fn main() -> Result<()> { - prost_build::compile_protos(&["protocol/ffi.proto"], &["protocol/"])?; + prost_build::compile_protos( + &[ + "protocol/ffi.proto", + "protocol/handle.proto", + "protocol/room.proto", + "protocol/track.proto", + "protocol/participant.proto", + "protocol/video_frame.proto", + "protocol/audio_frame.proto", + ], + &["protocol/"], + )?; Ok(()) } - diff --git a/livekit-ffi/protocol/audio_frame.proto b/livekit-ffi/protocol/audio_frame.proto new file mode 100644 index 0000000..ccedc2e --- /dev/null +++ b/livekit-ffi/protocol/audio_frame.proto @@ -0,0 +1,86 @@ +syntax = "proto3"; + +package livekit; +option csharp_namespace = "LiveKit.Proto"; + +import "handle.proto"; + +// Allocate a new AudioFrameBuffer +// This is not necessary required because the data structure is fairly simple +// But keep the API consistent with VideoFrame +message AllocAudioBufferRequest { + uint32 sample_rate = 1; + uint32 num_channels = 2; + uint32 samples_per_channel = 3; +} +message AllocAudioBufferResponse { AudioFrameBufferInfo buffer = 1; } + +// Create a new AudioStream +// AudioStream is used to receive audio frames from a track +message NewAudioStreamRequest { + FFIHandleId room_handle = 1; + string participant_sid = 2; + string track_sid = 3; + AudioStreamType type = 4; +} +message NewAudioStreamResponse { AudioStreamInfo stream = 1; } + +// Create a new AudioSource +message NewAudioSourceRequest { AudioSourceType type = 1; } +message NewAudioSourceResponse { AudioSourceInfo source = 1; } + +// Push a frame to an AudioSource +message CaptureAudioFrameRequest { + FFIHandleId source_handle = 1; + FFIHandleId buffer_handle = 2; +} +message CaptureAudioFrameResponse {} + +/// +/// AudioFrame buffer /// +/// + +message AudioFrameBufferInfo { + FFIHandleId handle = 1; + uint64 data_ptr = 2; // *const i16 + uint32 num_channels = 3; + uint32 sample_rate = 4; + uint32 samples_per_channel = 5; +} + +/// +/// AudioStream /// +/// + +enum AudioStreamType { + AUDIO_STREAM_NATIVE = 0; + AUDIO_STREAM_HTML = 1; +} + +message AudioStreamInfo { + FFIHandleId handle = 1; + AudioStreamType type = 2; + string track_sid = 3; +} + +message AudioStreamEvent { + FFIHandleId handle = 1; + oneof message { AudioFrameReceived frame_received = 2; } +} + +message AudioFrameReceived { + AudioFrameBufferInfo frame = 1; +} + +/// +/// AudioSource /// +/// + +enum AudioSourceType { + AUDIO_SOURCE_NATIVE = 0; +} + +message AudioSourceInfo { + FFIHandleId handle = 1; + AudioSourceType type = 2; +} diff --git a/livekit-ffi/protocol/ffi.proto b/livekit-ffi/protocol/ffi.proto index eb1a086..ea75af7 100644 --- a/livekit-ffi/protocol/ffi.proto +++ b/livekit-ffi/protocol/ffi.proto @@ -3,347 +3,109 @@ syntax = "proto3"; package livekit; option csharp_namespace = "LiveKit.Proto"; -/// IPC - -/// # Safety -/// The foreign language is responsable for disposing an handle -/// Forgetting to dispose the handle may lead to memory leaks -/// Messages in this file can contain an FFIHandle -message FFIHandleId { uint64 id = 1; } +import "handle.proto"; +import "track.proto"; +import "room.proto"; +import "participant.proto"; +import "video_frame.proto"; +import "audio_frame.proto"; /// This is the input of livekit_ffi_request function +/// We always expect a response (FFIResponse) message FFIRequest { oneof message { InitializeRequest initialize = 1; - // Stop all rooms synchronously (Do we need async here?). - // e.g: This is used for the Unity Editor after each assemblies reload. DisposeRequest dispose = 2; - ConnectRequest async_connect = 3; - DisconnectRequest async_disconnect = 4; - ToI420Request to_i420 = 5; - ToARGBRequest to_argb = 6; + + // Room + ConnectRequest connect = 3; + DisconnectRequest disconnect = 4; + PublishTrackRequest publish_track = 5; + UnpublishTrackRequest unpublish_track = 6; + + // Track + CreateVideoTrackRequest create_video_track = 7; + CreateAudioTrackRequest create_audio_track = 8; + + // Video + AllocVideoBufferRequest alloc_video_buffer = 9; + NewVideoStreamRequest new_video_stream = 10; + NewVideoSourceRequest new_video_source = 11; + CaptureVideoFrameRequest capture_video_frame = 12; + ToI420Request to_i420 = 13; + ToARGBRequest to_argb = 14; + + // Audio + AllocAudioBufferRequest alloc_audio_buffer = 15; + NewAudioStreamRequest new_audio_stream = 16; + NewAudioSourceRequest new_audio_source = 17; + CaptureAudioFrameRequest capture_audio_frame = 18; } } /// This is the output of livekit_ffi_request function. -/// The message field is mostly used to send result of a synchronous operation -/// to the foreign language. message FFIResponse { - optional uint64 async_id = 1; - oneof message { ToI420Response to_i420 = 2; } + oneof message { + InitializeResponse initialize = 1; + DisposeResponse dispose = 2; + + // Room + ConnectResponse connect = 3; + DisconnectResponse disconnect = 4; + PublishTrackResponse publish_track = 5; + UnpublishTrackResponse unpublish_track = 6; + + // Track + CreateVideoTrackResponse create_video_track = 7; + CreateAudioTrackResponse create_audio_track = 8; + + // Video + AllocVideoBufferResponse alloc_video_buffer = 9; + NewVideoStreamResponse new_video_stream = 10; + NewVideoSourceResponse new_video_source = 11; + CaptureVideoFrameResponse capture_video_frame = 12; + ToI420Response to_i420 = 13; + ToARGBResponse to_argb = 14; + + // Audio + AllocAudioBufferResponse alloc_audio_buffer = 15; + NewAudioStreamResponse new_audio_stream = 16; + NewAudioSourceResponse new_audio_source = 17; + CaptureAudioFrameResponse capture_audio_frame = 18; + } } -/// This message is used to receive the result of asynchronous requests -/// Or to receive events of a Room message FFIEvent { - // Used if the message is used to send the async result to the foreign - // language - optional uint64 async_id = 1; oneof message { - ConnectEvent connect_event = 2; - RoomEvent room_event = 3; - TrackEvent track_event = 4; - ParticipantEvent participant_event = 5; + RoomEvent room_event = 1; + TrackEvent track_event = 2; + ParticipantEvent participant_event = 3; + VideoStreamEvent video_stream_event = 4; + AudioStreamEvent audio_stream_event = 5; + ConnectCallback connect = 6; + DisposeCallback dispose = 7; + PublishTrackCallback publish_track = 8; } } // Setup the callback where the foreign language can receive events // and responses to asynchronous requests message InitializeRequest { uint64 event_callback_ptr = 1; } +message InitializeResponse {} -message DisposeRequest {} - -message ConnectRequest { - string url = 1; - string token = 2; - RoomOptions options = 3; +// Stop all rooms synchronously (Do we need async here?). +// e.g: This is used for the Unity Editor after each assemblies reload. +message DisposeRequest { + bool async = 1; } -message DisconnectRequest { string room_sid = 1; } - -/// Convert a VideoFrameBuffer to a I420Buffer -message ToI420Request { - FFIHandleId buffer = 1; // NOTE: This buffer will be dropped! +message DisposeResponse { + optional FFIAsyncId async_id = 1; // None if sync } -message ToARGBRequest { - FFIHandleId buffer = 1; - uint64 dst_ptr = 2; - VideoFormatType dst_format = 3; - int32 dst_stride = 4; - int32 dst_width = 5; - int32 dst_height = 6; +message DisposeCallback { + FFIAsyncId async_id = 1; } -message ConnectEvent { - bool success = 1; - optional RoomInfo room = 2; -} +// TODO(theomonnom): Debug messages (Print handles, forward logs). -message ToI420Response { optional VideoFrameBufferInfo new_buffer = 1; } - -/// Models - -message Dimension { - uint32 width = 1; - uint32 height = 2; -} - -message RoomOptions { - bool auto_subscribe = 1; - bool adaptive_stream = 2; -} - -message RoomInfo { - string sid = 1; - string name = 2; - string metadata = 3; - ParticipantInfo local_participant = 4; - repeated ParticipantInfo participants = 5; -} - -message ParticipantInfo { - string sid = 1; - string name = 2; - string identity = 3; - string metadata = 4; - repeated TrackPublicationInfo publications = 5; -} - -message TrackPublicationInfo { - string sid = 1; - string name = 2; - TrackKind kind = 3; - TrackSource source = 4; - bool simulcasted = 5; - Dimension dimension = 6; - string mime_type = 7; - bool muted = 8; -} - -message TrackInfo { - string sid = 1; - string name = 2; - TrackKind kind = 3; - StreamState stream_state = 4; - bool muted = 5; -} - -message VideoSinkInfo { - string track_sid = 1; - // More info? -} - -enum TrackKind { - KIND_UNKNOWN = 0; - KIND_AUDIO = 1; - KIND_VIDEO = 2; -} - -enum TrackSource { - SOURCE_UNKNOWN = 0; - SOURCE_CAMERA = 1; - SOURCE_MICROPHONE = 2; - SOURCE_SCREENSHARE = 3; - SOURCE_SCREENSHARE_AUDIO = 4; -} - -enum ConnectionQuality { - QUALITY_POOR = 0; - QUALITY_GOOD = 1; - QUALITY_EXCELLENT = 2; -} - -enum ConnectionState { - CONN_DISCONNECTED = 0; - CONN_CONNECTED = 1; - CONN_RECONNECTING = 2; - CONN_UNKNOWN = 3; -} - -enum StreamState { - STATE_UNKNOWN = 0; - STATE_ACTIVE = 1; - STATE_PAUSED = 2; -} - -enum VideoRotation { - VIDEO_ROTATION_0 = 0; - VIDEO_ROTATION_90 = 1; - VIDEO_ROTATION_180 = 2; - VIDEO_ROTATION_270 = 3; -} - -enum DataPacketKind { - KIND_UNRELIABLE = 0; - KIND_RELIABLE = 1; -} - -enum VideoFormatType { - FORMAT_ARGB = 0; - FORMAT_BGRA = 1; - FORMAT_ABGR = 2; - FORMAT_RGBA = 3; -} - -/// Room Events - -message RoomEvent { - string room_sid = 1; - oneof message { - ParticipantConnected participant_connected = 2; - ParticipantDisconnected participant_disconnected = 3; - TrackPublished track_published = 4; - TrackUnpublished track_unpublished = 5; - TrackSubscribed track_subscribed = 6; - TrackUnsubscribed track_unsubscribed = 7; - TrackMuted track_muted = 8; - TrackUnmuted track_unmuted = 9; - ActiveSpeakersChanged speakers_changed = 10; - ConnectionQualityChanged connection_quality_changed = 11; - DataReceived data_received = 12; - ConnectionStateChanged connection_state_changed = 13; - Connected connected = 14; - Disconnected disconnected = 15; - Reconnecting reconnecting = 16; - Reconnected reconnected = 17; - } -} - -message ParticipantConnected { ParticipantInfo info = 1; } - -message ParticipantDisconnected { ParticipantInfo info = 1; } - -message TrackPublished { - string participant_sid = 1; - TrackPublicationInfo publication = 2; -} - -message TrackUnpublished { - string participant_sid = 1; - string publication_sid = 2; -} - -// Publication isn't needed for subscription events on the FFI -// The FFI will retrieve the publication using the Track sid -message TrackSubscribed { - string participant_sid = 1; - TrackInfo track = 2; - VideoSinkInfo sink = 3; -} - -message TrackUnsubscribed { - // The FFI language can dispose/remove the VideoSink here - string participant_sid = 1; - string track_sid = 2; -} - -message TrackMuted { - string participant_sid = 1; - string track_sid = 2; -} - -message TrackUnmuted { - string participant_sid = 1; - string track_sid = 2; -} - -message ActiveSpeakersChanged { repeated string participant_sids = 1; } - -message ConnectionQualityChanged { - string participant_sid = 1; - ConnectionQuality quality = 2; -} - -message DataReceived { - FFIHandleId handle = 1; - optional string participant_sid = 2; - uint64 data_ptr = 3; - uint64 data_size = 4; - DataPacketKind kind = 5; -} - -message ConnectionStateChanged { ConnectionState state = 1; } - -message Connected {} -message Disconnected {} -message Reconnecting {} -message Reconnected {} - -/// Track Events - -message TrackEvent { - string track_sid = 1; - oneof message { FrameReceived frame_received = 2; } -} - -message FrameReceived { - VideoFrameInfo frame = 1; - VideoFrameBufferInfo buffer = 2; -} - -message VideoFrameInfo { - int64 timestamp = 1; - VideoRotation rotation = 2; -} - -message VideoFrameBufferInfo { - FFIHandleId handle = 1; - VideoFrameBufferType buffer_type = 2; - int32 width = 3; - int32 height = 4; - oneof buffer { - PlanarYuvBufferInfo yuv = 5; - BiplanarYuvBufferInfo bi_yuv = 6; - NativeBufferInfo native = 7; - } -} - -message PlanarYuvBufferInfo { - int32 chroma_width = 1; - int32 chroma_height = 2; - int32 stride_y = 3; - int32 stride_u = 4; - int32 stride_v = 5; - int32 stride_a = 6; - - // *const u8 or *const u16 - uint64 data_y_ptr = 7; - uint64 data_u_ptr = 8; - uint64 data_v_ptr = 9; - uint64 data_a_ptr = 10; // nullptr = no alpha -} - -message BiplanarYuvBufferInfo { - int32 chroma_width = 1; - int32 chroma_height = 2; - int32 stride_y = 3; - int32 stride_uv = 4; - - uint64 data_y_ptr = 5; - uint64 data_uv_ptr = 6; -} - -message NativeBufferInfo { - // TODO(theomonnom): Expose graphic context? -} - -enum VideoFrameBufferType { - NATIVE = 0; - I420 = 1; - I420A = 2; - I422 = 3; - I444 = 4; - I010 = 5; - NV12 = 6; - WEBGL = 7; -} - -/// Participant Events - -message ParticipantEvent { - string participant_sid = 1; - oneof message { IsSpeakingChanged speaking_changed = 2; } -} - -message IsSpeakingChanged { bool speaking = 1; } diff --git a/livekit-ffi/protocol/handle.proto b/livekit-ffi/protocol/handle.proto new file mode 100644 index 0000000..63fdb21 --- /dev/null +++ b/livekit-ffi/protocol/handle.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +package livekit; +option csharp_namespace = "LiveKit.Proto"; + +/// # Safety +/// The foreign language is responsable for disposing handles +/// Forgetting to dispose the handle may lead to memory leaks +/// Messages in this file can contain an FFIHandle +message FFIHandleId { + uint64 id = 1; +} + +/// Link the request/response of an asynchronous call +message FFIAsyncId { + uint64 id = 1; +} diff --git a/livekit-ffi/protocol/participant.proto b/livekit-ffi/protocol/participant.proto new file mode 100644 index 0000000..9a0b8f2 --- /dev/null +++ b/livekit-ffi/protocol/participant.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +package livekit; +option csharp_namespace = "LiveKit.Proto"; + +//import "handle.proto"; +import "track.proto"; + +message ParticipantInfo { + string sid = 1; + string name = 2; + string identity = 3; + string metadata = 4; + repeated TrackPublicationInfo publications = 5; +} + +message ParticipantEvent { + string participant_sid = 1; + oneof message { IsSpeakingChanged speaking_changed = 2; } +} + +message IsSpeakingChanged { bool speaking = 1; } diff --git a/livekit-ffi/protocol/room.proto b/livekit-ffi/protocol/room.proto new file mode 100644 index 0000000..e8154c5 --- /dev/null +++ b/livekit-ffi/protocol/room.proto @@ -0,0 +1,199 @@ +syntax = "proto3"; + +package livekit; +option csharp_namespace = "LiveKit.Proto"; + +import "handle.proto"; +import "participant.proto"; +import "track.proto"; +import "video_frame.proto"; + +// Connect to a new LiveKit room +message ConnectRequest { + string url = 1; + string token = 2; + RoomOptions options = 3; +} +message ConnectResponse { + FFIAsyncId async_id = 1; +} +message ConnectCallback { + FFIAsyncId async_id = 1; + optional string error = 2; + RoomInfo room = 3; +} + +// Disconnect from the a room +message DisconnectRequest { FFIHandleId room_handle = 1; } +message DisconnectResponse { FFIAsyncId async_id = 1; } +message DisconnectCallback { } + +// Publish a track to the room +message PublishTrackRequest { + FFIHandleId room_handle = 1; + FFIHandleId track_handle = 2; + TrackPublishOptions options = 3; +} +message PublishTrackResponse { + FFIAsyncId async_id = 1; +} +message PublishTrackCallback { + FFIAsyncId async_id = 1; + optional string error = 2; + TrackPublicationInfo publication = 3; +} + +// Unpublish a track from the room +message UnpublishTrackRequest { + FFIHandleId room_handle = 1; + string track_sid = 2; + bool stop_on_unpublish = 3; +} +message UnpublishTrackResponse { + FFIAsyncId async_id = 1; +} +message UnpublishTrackCallback { + optional string error = 1; +} + +/// +/// Options +/// + +message VideoEncoding { + uint64 max_bitrate = 1; + double max_framerate = 2; +} + +message AudioEncoding { + uint64 max_bitrate = 1; +} + +message TrackPublishOptions { + // encodings are optional + VideoEncoding video_encoding = 1; + AudioEncoding audio_encoding = 2; + VideoCodec video_codec = 3; + bool dtx = 4; + bool red = 5; + bool simulcast = 6; + string name = 7; + TrackSource source = 8; +} + +message RoomOptions { + bool auto_subscribe = 1; + bool adaptive_stream = 2; +} + +/// +/// Room +/// + +enum ConnectionQuality { + QUALITY_POOR = 0; + QUALITY_GOOD = 1; + QUALITY_EXCELLENT = 2; +} + +enum ConnectionState { + CONN_DISCONNECTED = 0; + CONN_CONNECTED = 1; + CONN_RECONNECTING = 2; + CONN_UNKNOWN = 3; +} + +enum DataPacketKind { + KIND_UNRELIABLE = 0; + KIND_RELIABLE = 1; +} + +message RoomEvent { + FFIHandleId room_handle = 1; + oneof message { + ParticipantConnected participant_connected = 2; + ParticipantDisconnected participant_disconnected = 3; + TrackPublished track_published = 4; + TrackUnpublished track_unpublished = 5; + TrackSubscribed track_subscribed = 6; + TrackUnsubscribed track_unsubscribed = 7; + TrackMuted track_muted = 8; + TrackUnmuted track_unmuted = 9; + ActiveSpeakersChanged speakers_changed = 10; + ConnectionQualityChanged connection_quality_changed = 11; + DataReceived data_received = 12; + ConnectionStateChanged connection_state_changed = 13; + Connected connected = 14; + Disconnected disconnected = 15; + Reconnecting reconnecting = 16; + Reconnected reconnected = 17; + } +} + +message RoomInfo { + FFIHandleId handle = 1; + string sid = 2; + string name = 3; + string metadata = 4; + ParticipantInfo local_participant = 5; + repeated ParticipantInfo participants = 6; +} + +message DataReceived { + FFIHandleId handle = 1; + optional string participant_sid = 2; + uint64 data_ptr = 3; + uint64 data_size = 4; + DataPacketKind kind = 5; +} + +// Publication isn't needed for subscription events on the FFI +// The FFI will retrieve the publication using the Track sid +message TrackSubscribed { + string participant_sid = 1; + TrackInfo track = 2; +} + +message TrackUnsubscribed { + // The FFI language can dispose/remove the VideoSink here + string participant_sid = 1; + string track_sid = 2; +} + +message TrackMuted { + string participant_sid = 1; + string track_sid = 2; +} + +message TrackUnmuted { + string participant_sid = 1; + string track_sid = 2; +} + +message ParticipantConnected { ParticipantInfo info = 1; } + +message ParticipantDisconnected { ParticipantInfo info = 1; } + +message TrackPublished { + string participant_sid = 1; + TrackPublicationInfo publication = 2; +} + +message TrackUnpublished { + string participant_sid = 1; + string publication_sid = 2; +} + +message ActiveSpeakersChanged { repeated string participant_sids = 1; } + +message ConnectionQualityChanged { + string participant_sid = 1; + ConnectionQuality quality = 2; +} + +message ConnectionStateChanged { ConnectionState state = 1; } + +message Connected {} +message Disconnected {} +message Reconnecting {} +message Reconnected {} diff --git a/livekit-ffi/protocol/track.proto b/livekit-ffi/protocol/track.proto new file mode 100644 index 0000000..1a34327 --- /dev/null +++ b/livekit-ffi/protocol/track.proto @@ -0,0 +1,91 @@ +syntax = "proto3"; + +package livekit; +option csharp_namespace = "LiveKit.Proto"; + +import "handle.proto"; +import "video_frame.proto"; +import "audio_frame.proto"; + +message VideoCaptureOptions { + VideoResolution resolution = 1; +} + +message AudioCaptureOptions { + bool echo_cancellation = 1; + bool noise_suppression = 2; + bool auto_gain_control = 3; +} + +// Create a new VideoTrack from a VideoSource +message CreateVideoTrackRequest { + string name = 1; + VideoCaptureOptions options = 2; + FFIHandleId source_handle = 3; +} +message CreateVideoTrackResponse { + TrackInfo track = 1; +} + +// Create a new AudioTrack from a AudioSource +message CreateAudioTrackRequest { + string name = 1; + AudioCaptureOptions options = 2; + FFIHandleId source_handle = 3; +} +message CreateAudioTrackResponse { + TrackInfo track = 1; +} + +/// +/// Track +/// + +message TrackEvent {} + +enum TrackKind { + KIND_UNKNOWN = 0; + KIND_AUDIO = 1; + KIND_VIDEO = 2; +} + +enum TrackSource { + SOURCE_UNKNOWN = 0; + SOURCE_CAMERA = 1; + SOURCE_MICROPHONE = 2; + SOURCE_SCREENSHARE = 3; + SOURCE_SCREENSHARE_AUDIO = 4; +} + +enum StreamState { + STATE_UNKNOWN = 0; + STATE_ACTIVE = 1; + STATE_PAUSED = 2; +} + +// TODO(theomonnom): Should we have a separate message whether the track is local or remote? + +message TrackPublicationInfo { + string sid = 1; + string name = 2; + TrackKind kind = 3; + TrackSource source = 4; + bool simulcasted = 5; + uint32 width = 6; + uint32 height = 7; + string mime_type = 8; + bool muted = 9; + bool remote = 10; +} + +message TrackInfo { + // Tracks created/owned by the client will have a handle + FFIHandleId opt_handle = 1; + string sid = 2; + string name = 3; + TrackKind kind = 4; + StreamState stream_state = 5; + bool muted = 6; + bool remote = 7; +} + diff --git a/livekit-ffi/protocol/video_frame.proto b/livekit-ffi/protocol/video_frame.proto new file mode 100644 index 0000000..797e8df --- /dev/null +++ b/livekit-ffi/protocol/video_frame.proto @@ -0,0 +1,198 @@ +syntax = "proto3"; + +package livekit; +option csharp_namespace = "LiveKit.Proto"; + +import "handle.proto"; + +// Allocate a new VideoFrameBuffer +message AllocVideoBufferRequest { + VideoFrameBufferType type = 1; // Only I420 is supported atm + uint32 width = 2; + uint32 height = 3; +} +message AllocVideoBufferResponse { VideoFrameBufferInfo buffer = 1; } + +// Create a new VideoStream +// VideoStream is used to receive video frames from a track +message NewVideoStreamRequest { + FFIHandleId room_handle = 1; + string participant_sid = 2; + string track_sid = 3; + VideoStreamType type = 4; +} +message NewVideoStreamResponse { VideoStreamInfo stream = 1; } + +// Create a new VideoSource +// VideoSource is used to send video frame to a track +message NewVideoSourceRequest { VideoSourceType type = 1; } +message NewVideoSourceResponse { VideoSourceInfo source = 1; } + +// Push a frame to a VideoSource +message CaptureVideoFrameRequest { + FFIHandleId source_handle = 1; + VideoFrameInfo frame = 2; + FFIHandleId buffer_handle = 3; +} +message CaptureVideoFrameResponse {} + +// Convert a RGBA frame to a I420 YUV frame +// Or convert another YUV frame format to I420 +message ToI420Request { + bool flip_y = 1; + oneof from { + ARGBBufferInfo argb = 2; + FFIHandleId buffer = 3; + } +} +message ToI420Response { VideoFrameBufferInfo buffer = 1; } + +// Convert a YUV frame to a RGBA frame +// Only I420 is supported atm +message ToARGBRequest { + FFIHandleId buffer = 1; + uint64 dst_ptr = 2; + VideoFormatType dst_format = 3; + uint32 dst_stride = 4; + uint32 dst_width = 5; + uint32 dst_height = 6; + bool flip_y = 7; +} +message ToARGBResponse {} + +/// +/// VideoFrame buffers /// +/// + +message VideoResolution { + uint32 width = 1; + uint32 height = 2; + double frame_rate = 3; +} + +enum VideoCodec { + VP8 = 0; + H264 = 1; + AV1 = 2; +} + +enum VideoRotation { + VIDEO_ROTATION_0 = 0; + VIDEO_ROTATION_90 = 1; + VIDEO_ROTATION_180 = 2; + VIDEO_ROTATION_270 = 3; +} + +enum VideoFormatType { + FORMAT_ARGB = 0; + FORMAT_BGRA = 1; + FORMAT_ABGR = 2; + FORMAT_RGBA = 3; +} + +enum VideoFrameBufferType { + NATIVE = 0; + I420 = 1; + I420A = 2; + I422 = 3; + I444 = 4; + I010 = 5; + NV12 = 6; + WEBGL = 7; +} + +message ARGBBufferInfo { + uint64 ptr = 1; + VideoFormatType format = 2; + uint32 stride = 3; + uint32 width = 4; + uint32 height = 5; +} + +message VideoFrameInfo { + int64 timestamp = 1; + VideoRotation rotation = 2; +} + +message VideoFrameBufferInfo { + FFIHandleId handle = 1; + VideoFrameBufferType buffer_type = 2; + uint32 width = 3; + uint32 height = 4; + oneof buffer { + PlanarYuvBufferInfo yuv = 5; + BiplanarYuvBufferInfo bi_yuv = 6; + NativeBufferInfo native = 7; + } +} + +message PlanarYuvBufferInfo { + uint32 chroma_width = 1; + uint32 chroma_height = 2; + uint32 stride_y = 3; + uint32 stride_u = 4; + uint32 stride_v = 5; + uint32 stride_a = 6; + + // *const u8 or *const u16 + uint64 data_y_ptr = 7; + uint64 data_u_ptr = 8; + uint64 data_v_ptr = 9; + uint64 data_a_ptr = 10; // nullptr = no alpha +} + +message BiplanarYuvBufferInfo { + uint32 chroma_width = 1; + uint32 chroma_height = 2; + uint32 stride_y = 3; + uint32 stride_uv = 4; + + uint64 data_y_ptr = 5; + uint64 data_uv_ptr = 6; +} + +message NativeBufferInfo { + // TODO(theomonnom): Expose graphic context? +} + +/// +/// VideoStream /// +/// + +enum VideoStreamType { + VIDEO_STREAM_NATIVE = 0; + VIDEO_STREAM_WEBGL = 1; + VIDEO_STREAM_HTML = 2; +} + +message VideoStreamInfo { + FFIHandleId handle = 1; + VideoStreamType type = 2; + string track_sid = 3; +} + +message VideoStreamEvent { + FFIHandleId handle = 1; + oneof message { VideoFrameReceived frame_received = 2; } +} + +message VideoFrameReceived { + VideoFrameInfo frame = 1; + VideoFrameBufferInfo buffer = 2; +} + +/// +/// VideoSource /// +/// + +enum VideoSourceType { + VIDEO_SOURCE_NATIVE = 0; +} + +message VideoSourceInfo { + // # SAFETY + // This handle must not be dropped if a track is currently using it + FFIHandleId handle = 1; + VideoSourceType type = 2; +} + diff --git a/livekit-ffi/src/conversion/audio_frame.rs b/livekit-ffi/src/conversion/audio_frame.rs new file mode 100644 index 0000000..314e4d4 --- /dev/null +++ b/livekit-ffi/src/conversion/audio_frame.rs @@ -0,0 +1,38 @@ +use crate::server::audio_frame::{FfiAudioSource, FfiAudioSream}; +use crate::{proto, FfiHandleId}; +use livekit::webrtc::prelude::*; + +impl proto::AudioFrameBufferInfo { + pub fn from(handle_id: FfiHandleId, buffer: &AudioFrame) -> Self { + Self { + handle: Some(handle_id.into()), + data_ptr: buffer.data.as_ptr() as u64, + samples_per_channel: buffer.samples_per_channel, + sample_rate: buffer.sample_rate, + num_channels: buffer.num_channels, + } + } +} + +impl From<&FfiAudioSream> for proto::AudioStreamInfo { + fn from(stream: &FfiAudioSream) -> Self { + Self { + handle: Some(proto::FfiHandleId { + id: stream.handle_id() as u64, + }), + track_sid: stream.track_sid().clone().into(), + r#type: stream.stream_type() as i32, + } + } +} + +impl From<&FfiAudioSource> for proto::AudioSourceInfo { + fn from(source: &FfiAudioSource) -> Self { + Self { + handle: Some(proto::FfiHandleId { + id: source.handle_id() as u64, + }), + r#type: source.source_type() as i32, + } + } +} diff --git a/livekit-ffi/src/conversion/mod.rs b/livekit-ffi/src/conversion/mod.rs new file mode 100644 index 0000000..090b221 --- /dev/null +++ b/livekit-ffi/src/conversion/mod.rs @@ -0,0 +1,21 @@ +use crate::proto; +use crate::{FfiAsyncId, FfiHandleId}; + +pub mod audio_frame; +pub mod participant; +pub mod publication; +pub mod room; +pub mod track; +pub mod video_frame; + +impl From for proto::FfiHandleId { + fn from(id: FfiHandleId) -> Self { + Self { id: id as u64 } + } +} + +impl From for proto::FfiAsyncId { + fn from(id: FfiAsyncId) -> Self { + Self { id: id as u64 } + } +} diff --git a/livekit-ffi/src/conversion/participant.rs b/livekit-ffi/src/conversion/participant.rs new file mode 100644 index 0000000..44b3879 --- /dev/null +++ b/livekit-ffi/src/conversion/participant.rs @@ -0,0 +1,22 @@ +use crate::proto; +use livekit::prelude::*; + +macro_rules! impl_participant_into { + ($p:ty) => { + impl From<$p> for proto::ParticipantInfo { + fn from(p: $p) -> Self { + Self { + name: p.name(), + sid: p.sid().to_string(), + identity: p.identity().to_string(), + metadata: p.metadata(), + publications: p.tracks().iter().map(|(_, p)| p.into()).collect(), + } + } + } + }; +} + +impl_participant_into!(&LocalParticipant); +impl_participant_into!(&RemoteParticipant); +impl_participant_into!(&Participant); diff --git a/livekit-ffi/src/server/conversion/publication.rs b/livekit-ffi/src/conversion/publication.rs similarity index 100% rename from livekit-ffi/src/server/conversion/publication.rs rename to livekit-ffi/src/conversion/publication.rs diff --git a/livekit-ffi/src/conversion/room.rs b/livekit-ffi/src/conversion/room.rs new file mode 100644 index 0000000..e4f4be5 --- /dev/null +++ b/livekit-ffi/src/conversion/room.rs @@ -0,0 +1,117 @@ +use crate::{proto, FfiHandleId, INVALID_HANDLE}; +use livekit::options::{AudioEncoding, TrackPublishOptions, VideoEncoding}; +use livekit::prelude::*; + +impl proto::RoomEvent { + pub fn from(room_handle: FfiHandleId, event: RoomEvent) -> Option { + let message = match event { + RoomEvent::ParticipantConnected(participant) => Some( + proto::room_event::Message::ParticipantConnected(proto::ParticipantConnected { + info: Some((&participant).into()), + }), + ), + RoomEvent::ParticipantDisconnected(participant) => { + Some(proto::room_event::Message::ParticipantDisconnected( + proto::ParticipantDisconnected { + info: Some((&participant).into()), + }, + )) + } + RoomEvent::TrackPublished { + publication, + participant, + } => Some(proto::room_event::Message::TrackPublished( + proto::TrackPublished { + participant_sid: participant.sid().to_string(), + publication: Some((&publication).into()), + }, + )), + RoomEvent::TrackUnpublished { + publication, + participant, + } => Some(proto::room_event::Message::TrackUnpublished( + proto::TrackUnpublished { + participant_sid: participant.sid().to_string(), + publication_sid: publication.sid().into(), + }, + )), + RoomEvent::TrackSubscribed { + track, + publication: _, + participant, + } => Some(proto::room_event::Message::TrackSubscribed( + proto::TrackSubscribed { + participant_sid: participant.sid().to_string(), + track: Some(proto::TrackInfo::from_remote_track(INVALID_HANDLE, &track)), + }, + )), + RoomEvent::TrackUnsubscribed { + track, + publication: _, + participant, + } => Some(proto::room_event::Message::TrackUnsubscribed( + proto::TrackUnsubscribed { + participant_sid: participant.sid().to_string(), + track_sid: track.sid().to_string(), + }, + )), + _ => None, + }; + + message.map(|message| proto::RoomEvent { + room_handle: Some(room_handle.into()), + message: Some(message), + }) + } +} + +impl proto::RoomInfo { + pub fn from_session(handle_id: FfiHandleId, session: &RoomSession) -> Self { + Self { + handle: Some(handle_id.into()), + sid: session.sid().into(), + name: session.name(), + metadata: session.metadata(), + local_participant: Some((&session.local_participant()).into()), + participants: session + .participants() + .iter() + .map(|(_, p)| p.into()) + .collect(), + } + } +} + +impl From for TrackPublishOptions { + fn from(opts: proto::TrackPublishOptions) -> Self { + Self { + video_encoding: opts.video_encoding.map(Into::into), + audio_encoding: opts.audio_encoding.map(Into::into), + video_codec: proto::VideoCodec::from_i32(opts.video_codec) + .unwrap() + .into(), + dtx: opts.dtx, + red: opts.red, + simulcast: opts.simulcast, + name: opts.name, + source: proto::TrackSource::from_i32(opts.source).unwrap().into(), + } + } +} + +impl From for VideoEncoding { + fn from(opts: proto::VideoEncoding) -> Self { + Self { + max_bitrate: opts.max_bitrate, + max_framerate: opts.max_framerate, + } + } +} + +impl From for AudioEncoding { + fn from(opts: proto::AudioEncoding) -> Self { + Self { + max_bitrate: opts.max_bitrate, + } + } +} diff --git a/livekit-ffi/src/conversion/track.rs b/livekit-ffi/src/conversion/track.rs new file mode 100644 index 0000000..e314318 --- /dev/null +++ b/livekit-ffi/src/conversion/track.rs @@ -0,0 +1,114 @@ +use crate::{proto, FfiHandleId}; +use livekit::options::{AudioCaptureOptions, VideoCaptureOptions}; +use livekit::prelude::*; + +impl From for VideoCaptureOptions { + fn from(opts: proto::VideoCaptureOptions) -> Self { + Self { + resolution: opts.resolution.unwrap_or_default().into(), + } + } +} + +impl From for AudioCaptureOptions { + fn from(opts: proto::AudioCaptureOptions) -> Self { + Self { + echo_cancellation: opts.echo_cancellation, + auto_gain_control: opts.auto_gain_control, + noise_suppression: opts.noise_suppression, + } + } +} + +impl From for proto::TrackSource { + fn from(source: TrackSource) -> proto::TrackSource { + match source { + TrackSource::Unknown => proto::TrackSource::SourceUnknown, + TrackSource::Camera => proto::TrackSource::SourceCamera, + TrackSource::Microphone => proto::TrackSource::SourceMicrophone, + TrackSource::Screenshare => proto::TrackSource::SourceScreenshare, + TrackSource::ScreenshareAudio => proto::TrackSource::SourceScreenshareAudio, + } + } +} + +macro_rules! impl_publication_into { + ($p:ty) => { + impl From<$p> for proto::TrackPublicationInfo { + fn from(p: $p) -> Self { + Self { + name: p.name(), + sid: p.sid().to_string(), + kind: proto::TrackKind::from(p.kind()).into(), + source: proto::TrackSource::from(p.source()).into(), + width: p.dimension().0, + height: p.dimension().1, + mime_type: p.mime_type(), + simulcasted: p.simulcasted(), + muted: p.is_muted(), + remote: p.is_remote(), + } + } + } + }; +} + +impl_publication_into!(&LocalTrackPublication); +impl_publication_into!(&RemoteTrackPublication); +impl_publication_into!(&TrackPublication); + +macro_rules! impl_track_into { + ($fnc:ident, $t:ty) => { + impl proto::TrackInfo { + pub fn $fnc(handle_id: FfiHandleId, track: $t) -> Self { + Self { + opt_handle: Some(handle_id.into()), + name: track.name(), + stream_state: proto::StreamState::from(track.stream_state()).into(), + sid: track.sid().to_string(), + kind: proto::TrackKind::from(track.kind()).into(), + muted: track.is_muted(), + remote: track.is_remote(), + } + } + } + }; +} + +impl_track_into!(from_local_audio_track, &LocalAudioTrack); +impl_track_into!(from_local_video_track, &LocalVideoTrack); +impl_track_into!(from_remote_audio_track, &RemoteAudioTrack); +impl_track_into!(from_remote_video_track, &RemoteVideoTrack); +impl_track_into!(from_track, &Track); +impl_track_into!(from_local_track, &LocalTrack); +impl_track_into!(from_remote_track, &RemoteTrack); + +impl From for proto::TrackKind { + fn from(kind: TrackKind) -> Self { + match kind { + TrackKind::Audio => proto::TrackKind::KindAudio, + TrackKind::Video => proto::TrackKind::KindVideo, + } + } +} + +impl From for proto::StreamState { + fn from(state: StreamState) -> Self { + match state { + StreamState::Active => Self::StateActive, + StreamState::Paused => Self::StatePaused, + } + } +} + +impl From for TrackSource { + fn from(source: proto::TrackSource) -> Self { + match source { + proto::TrackSource::SourceUnknown => TrackSource::Unknown, + proto::TrackSource::SourceCamera => TrackSource::Camera, + proto::TrackSource::SourceMicrophone => TrackSource::Microphone, + proto::TrackSource::SourceScreenshare => TrackSource::Screenshare, + proto::TrackSource::SourceScreenshareAudio => TrackSource::ScreenshareAudio, + } + } +} diff --git a/livekit-ffi/src/server/conversion/video_frame.rs b/livekit-ffi/src/conversion/video_frame.rs similarity index 60% rename from livekit-ffi/src/server/conversion/video_frame.rs rename to livekit-ffi/src/conversion/video_frame.rs index f1d36de..21ceb6c 100644 --- a/livekit-ffi/src/server/conversion/video_frame.rs +++ b/livekit-ffi/src/conversion/video_frame.rs @@ -1,5 +1,7 @@ use crate::proto; -use crate::server::FFIHandleId; +use crate::server::video_frame::{FfiVideoSource, FfiVideoStream}; +use crate::FfiHandleId; +use livekit::options::{VideoCodec, VideoResolution}; use livekit::webrtc::prelude::*; use livekit::webrtc::video_frame; @@ -8,9 +10,9 @@ macro_rules! impl_yuv_into { Self { chroma_width: $buffer.chroma_width(), chroma_height: $buffer.chroma_height(), - stride_y: $buffer.stride_y(), - stride_u: $buffer.stride_u(), - stride_v: $buffer.stride_v(), + stride_y: $buffer.strides().0, + stride_u: $buffer.strides().1, + stride_v: $buffer.strides().2, data_y_ptr: $data_y.as_ptr() as u64, data_u_ptr: $data_u.as_ptr() as u64, data_v_ptr: $data_v.as_ptr() as u64, @@ -22,7 +24,7 @@ macro_rules! impl_yuv_into { fn from(buffer: $buffer) -> Self { let (data_y, data_u, data_v, data_a) = buffer.data(); let mut proto = impl_yuv_into!(@fields, buffer, data_y, data_u, data_v); - proto.stride_a = buffer.stride_a(); + proto.stride_a = buffer.strides().3; proto.data_a_ptr = data_a.map(|data_a| data_a.as_ptr() as u64).unwrap_or(0); proto } @@ -42,12 +44,13 @@ macro_rules! impl_biyuv_into { ($b:ty) => { impl From<$b> for proto::BiplanarYuvBufferInfo { fn from(buffer: $b) -> Self { + let (stride_y, stride_uv) = buffer.strides(); let (data_y, data_uv) = buffer.data(); Self { chroma_width: buffer.chroma_width(), chroma_height: buffer.chroma_height(), - stride_y: buffer.stride_y(), - stride_uv: buffer.stride_uv(), + stride_y: stride_y, + stride_uv: stride_uv, data_y_ptr: data_y.as_ptr() as u64, data_uv_ptr: data_uv.as_ptr() as u64, } @@ -66,7 +69,7 @@ impl_biyuv_into!(&NV12Buffer); impl proto::VideoFrameInfo { pub fn from(frame: &VideoFrame) -> Self where - T: VideoFrameBuffer, + T: AsRef, { Self { timestamp: frame.timestamp, @@ -76,22 +79,36 @@ impl proto::VideoFrameInfo { } impl proto::VideoFrameBufferInfo { - pub fn from(handle: FFIHandleId, buffer: &dyn VideoFrameBuffer) -> Self { - match &buffer.buffer_type() { + pub fn from(handle: FfiHandleId, buffer: impl AsRef) -> Self { + match &buffer.as_ref().buffer_type() { #[cfg(not(target_arch = "wasm32"))] - VideoFrameBufferType::Native => Self::from_native(handle, buffer.as_native().unwrap()), - VideoFrameBufferType::I420 => Self::from_i420(handle, buffer.as_i420().unwrap()), - VideoFrameBufferType::I420A => Self::from_i420a(handle, buffer.as_i420a().unwrap()), - VideoFrameBufferType::I422 => Self::from_i422(handle, buffer.as_i422().unwrap()), - VideoFrameBufferType::I444 => Self::from_i444(handle, buffer.as_i444().unwrap()), - VideoFrameBufferType::I010 => Self::from_i010(handle, buffer.as_i010().unwrap()), - VideoFrameBufferType::NV12 => Self::from_nv12(handle, buffer.as_nv12().unwrap()), + VideoFrameBufferType::Native => { + Self::from_native(handle, buffer.as_ref().as_native().unwrap()) + } + VideoFrameBufferType::I420 => { + Self::from_i420(handle, buffer.as_ref().as_i420().unwrap()) + } + VideoFrameBufferType::I420A => { + Self::from_i420a(handle, buffer.as_ref().as_i420a().unwrap()) + } + VideoFrameBufferType::I422 => { + Self::from_i422(handle, buffer.as_ref().as_i422().unwrap()) + } + VideoFrameBufferType::I444 => { + Self::from_i444(handle, buffer.as_ref().as_i444().unwrap()) + } + VideoFrameBufferType::I010 => { + Self::from_i010(handle, buffer.as_ref().as_i010().unwrap()) + } + VideoFrameBufferType::NV12 => { + Self::from_nv12(handle, buffer.as_ref().as_nv12().unwrap()) + } _ => panic!("unsupported buffer type on this platform"), } } #[cfg(not(target_arch = "wasm32"))] - pub fn from_native(handle_id: FFIHandleId, buffer: &video_frame::native::NativeBuffer) -> Self { + pub fn from_native(handle_id: FfiHandleId, buffer: &video_frame::native::NativeBuffer) -> Self { Self { handle: Some(handle_id.into()), buffer_type: proto::VideoFrameBufferType::Native.into(), @@ -103,7 +120,7 @@ impl proto::VideoFrameBufferInfo { } } - pub fn from_i420(handle_id: FFIHandleId, buffer: &I420Buffer) -> Self { + pub fn from_i420(handle_id: FfiHandleId, buffer: &I420Buffer) -> Self { Self { handle: Some(handle_id.into()), buffer_type: proto::VideoFrameBufferType::I420.into(), @@ -113,7 +130,7 @@ impl proto::VideoFrameBufferInfo { } } - pub fn from_i420a(handle_id: FFIHandleId, buffer: &I420ABuffer) -> Self { + pub fn from_i420a(handle_id: FfiHandleId, buffer: &I420ABuffer) -> Self { Self { handle: Some(handle_id.into()), buffer_type: proto::VideoFrameBufferType::I420a.into(), @@ -123,7 +140,7 @@ impl proto::VideoFrameBufferInfo { } } - pub fn from_i422(handle_id: FFIHandleId, buffer: &I422Buffer) -> Self { + pub fn from_i422(handle_id: FfiHandleId, buffer: &I422Buffer) -> Self { Self { handle: Some(handle_id.into()), buffer_type: proto::VideoFrameBufferType::I422.into(), @@ -133,7 +150,7 @@ impl proto::VideoFrameBufferInfo { } } - pub fn from_i444(handle_id: FFIHandleId, buffer: &I444Buffer) -> Self { + pub fn from_i444(handle_id: FfiHandleId, buffer: &I444Buffer) -> Self { Self { handle: Some(handle_id.into()), buffer_type: proto::VideoFrameBufferType::I444.into(), @@ -143,7 +160,7 @@ impl proto::VideoFrameBufferInfo { } } - pub fn from_i010(handle_id: FFIHandleId, buffer: &I010Buffer) -> Self { + pub fn from_i010(handle_id: FfiHandleId, buffer: &I010Buffer) -> Self { Self { handle: Some(handle_id.into()), buffer_type: proto::VideoFrameBufferType::I010.into(), @@ -153,7 +170,7 @@ impl proto::VideoFrameBufferInfo { } } - pub fn from_nv12(handle_id: FFIHandleId, buffer: &NV12Buffer) -> Self { + pub fn from_nv12(handle_id: FfiHandleId, buffer: &NV12Buffer) -> Self { Self { handle: Some(handle_id.into()), buffer_type: proto::VideoFrameBufferType::Nv12.into(), @@ -186,6 +203,17 @@ impl From for proto::VideoRotation { } } +impl From for VideoRotation { + fn from(rotation: proto::VideoRotation) -> VideoRotation { + match rotation { + proto::VideoRotation::VideoRotation0 => Self::VideoRotation0, + proto::VideoRotation::VideoRotation90 => Self::VideoRotation90, + proto::VideoRotation::VideoRotation180 => Self::VideoRotation180, + proto::VideoRotation::VideoRotation270 => Self::VideoRotation270, + } + } +} + impl From for proto::VideoFrameBufferType { fn from(buffer_type: VideoFrameBufferType) -> Self { match buffer_type { @@ -201,3 +229,57 @@ impl From for proto::VideoFrameBufferType { } } } + +impl From<&FfiVideoStream> for proto::VideoStreamInfo { + fn from(stream: &FfiVideoStream) -> Self { + Self { + handle: Some(proto::FfiHandleId { + id: stream.handle_id() as u64, + }), + track_sid: stream.track_sid().clone().into(), + r#type: stream.stream_type() as i32, + } + } +} + +impl From<&FfiVideoSource> for proto::VideoSourceInfo { + fn from(source: &FfiVideoSource) -> Self { + Self { + handle: Some(proto::FfiHandleId { + id: source.handle_id() as u64, + }), + r#type: source.source_type() as i32, + } + } +} + +impl From for proto::VideoResolution { + fn from(resolution: VideoResolution) -> Self { + Self { + width: resolution.width, + height: resolution.height, + frame_rate: resolution.frame_rate, + } + } +} + +impl From for VideoResolution { + fn from(resolution: proto::VideoResolution) -> Self { + Self { + width: resolution.width, + height: resolution.height, + frame_rate: resolution.frame_rate, + aspect_ratio: resolution.width as f32 / resolution.height as f32, + } + } +} + +impl From for VideoCodec { + fn from(codec: proto::VideoCodec) -> Self { + match codec { + proto::VideoCodec::Vp8 => Self::VP8, + proto::VideoCodec::H264 => Self::H264, + proto::VideoCodec::Av1 => Self::AV1, + } + } +} diff --git a/livekit-ffi/src/lib.rs b/livekit-ffi/src/lib.rs index 243f07f..7823f35 100644 --- a/livekit-ffi/src/lib.rs +++ b/livekit-ffi/src/lib.rs @@ -1,5 +1,78 @@ +use livekit::prelude::*; +use prost::Message; +use std::any::Any; +use thiserror::Error; + mod proto { include!(concat!(env!("OUT_DIR"), "/livekit.rs")); } - +mod conversion; mod server; + +#[derive(Error, Debug)] +pub enum FfiError { + #[error("the server is not configured")] + NotConfigured, + #[error("the server is already initialized")] + AlreadyInitialized, + #[error("room error {0}")] + Room(#[from] RoomError), + #[error("invalid request: {0}")] + InvalidRequest(&'static str), +} + +/// # SAFTEY: The "C" callback must be threadsafe and not block +pub type FfiCallbackFn = unsafe extern "C" fn(*const u8, usize); +pub type FfiResult = Result; +pub type FfiAsyncId = usize; +pub type FfiHandleId = usize; +pub type FfiHandle = Box; + +pub const INVALID_HANDLE: FfiHandleId = 0; + +#[no_mangle] +pub(crate) extern "C" fn livekit_ffi_request( + data: *const u8, + len: usize, + res_ptr: *mut *const u8, + res_len: *mut usize, +) -> FfiHandleId { + let data = unsafe { std::slice::from_raw_parts(data, len) }; + let res = match proto::FfiRequest::decode(data) { + Ok(res) => res, + Err(err) => { + eprintln!("failed to decode request: {}", err); + return INVALID_HANDLE; + } + }; + + let res = match server::FFI_SERVER.handle_request(res) { + Ok(res) => res, + Err(err) => { + eprintln!("failed to handle request: {}", err); + return INVALID_HANDLE; + } + } + .encode_to_vec(); + + unsafe { + *res_ptr = res.as_ptr(); + *res_len = res.len(); + } + + let handle_id = server::FFI_SERVER.next_id(); + server::FFI_SERVER + .ffi_handles() + .insert(handle_id, Box::new(res)); + + handle_id +} + +#[no_mangle] +pub(crate) extern "C" fn livekit_ffi_drop_handle(handle_id: FfiHandleId) -> bool { + // Free the memory + server::FFI_SERVER + .ffi_handles() + .remove(&handle_id) + .is_some() +} diff --git a/livekit-ffi/src/server/audio_frame.rs b/livekit-ffi/src/server/audio_frame.rs new file mode 100644 index 0000000..0070820 --- /dev/null +++ b/livekit-ffi/src/server/audio_frame.rs @@ -0,0 +1,217 @@ +use crate::{proto, server, FfiError, FfiHandleId, FfiResult}; +use futures_util::StreamExt; +use livekit::prelude::*; +use livekit::webrtc::audio_frame::AudioFrame; +use livekit::webrtc::audio_source::native::NativeAudioSource; +use livekit::webrtc::audio_stream::native::NativeAudioStream; +use livekit::webrtc::media_stream::MediaStreamTrack; +use log::warn; +use server::utils; +use tokio::sync::oneshot; + +// ===== FFIAudioStream ===== + +pub struct FfiAudioSream { + handle_id: FfiHandleId, + stream_type: proto::AudioStreamType, + track_sid: TrackSid, + + #[allow(dead_code)] + close_tx: oneshot::Sender<()>, // Close the stream on drop +} + +impl FfiAudioSream { + /// Setup a new AudioStream and forward the audio data to the client/the foreign + /// language. + /// + /// When FFIAudioStream is dropped (When the corresponding handle_id is dropped), the task + /// is being closed. + /// + /// It is possible that the client receives an AudioFrame after the task is closed. The client + /// musts ignore it. + pub fn setup( + server: &'static server::FfiServer, + new_stream: proto::NewAudioStreamRequest, + ) -> FfiResult { + let (close_tx, close_rx) = oneshot::channel(); + let stream_type = proto::AudioStreamType::from_i32(new_stream.r#type).unwrap(); + let track_sid: TrackSid = new_stream.track_sid.into(); + + let room_handle = new_stream + .room_handle + .ok_or(FfiError::InvalidRequest("room_handle is empty"))? + .id as FfiHandleId; + + let track = utils::find_remote_track( + server, + &track_sid, + &new_stream.participant_sid.into(), + room_handle, + )? + .rtc_track(); + + let MediaStreamTrack::Audio(track) = track else { + return Err(FfiError::InvalidRequest("not an audio track")); + }; + + let audio_stream = match stream_type { + proto::AudioStreamType::AudioStreamNative => { + let audio_stream = Self { + handle_id: server.next_id(), + stream_type, + close_tx, + track_sid, + }; + tokio::spawn(Self::native_audio_stream_task( + server, + audio_stream.handle_id, + NativeAudioStream::new(track), + close_rx, + )); + Ok::(audio_stream) + } + // TODO(theomonnom): Support other stream types + _ => return Err(FfiError::InvalidRequest("unsupported audio stream type")), + }?; + + // Store the new audio stream and return the info + let info = proto::AudioStreamInfo::from(&audio_stream); + server + .ffi_handles() + .insert(audio_stream.handle_id, Box::new(audio_stream)); + + Ok(info) + } + + pub fn handle_id(&self) -> FfiHandleId { + self.handle_id + } + + pub fn stream_type(&self) -> proto::AudioStreamType { + self.stream_type + } + + pub fn track_sid(&self) -> &TrackSid { + &self.track_sid + } + + async fn native_audio_stream_task( + server: &'static server::FfiServer, + stream_handle_id: FfiHandleId, + mut native_stream: NativeAudioStream, + mut close_rx: oneshot::Receiver<()>, + ) { + loop { + tokio::select! { + _ = &mut close_rx => { + break; + } + frame = native_stream.next() => { + let Some(frame) = frame else { + break; + }; + + let handle_id = server.next_id(); + let buffer_info = proto::AudioFrameBufferInfo::from(handle_id, &frame); + + server.ffi_handles().insert(handle_id, Box::new(frame)); + + if let Err(err) = server.send_event(proto::ffi_event::Message::AudioStreamEvent( + proto::AudioStreamEvent { + handle: Some(stream_handle_id.into()), + message: Some(proto::audio_stream_event::Message::FrameReceived( + proto::AudioFrameReceived { + frame: Some(buffer_info), + }, + )), + }, + )) { + warn!("failed to send audio frame: {}", err); + } + } + } + } + } +} + +// ===== FFIAudioSource ===== + +pub struct FfiAudioSource { + handle_id: FfiHandleId, + source_type: proto::AudioSourceType, + source: AudioSource, +} + +#[derive(Clone)] +pub enum AudioSource { + Native(NativeAudioSource), +} + +impl FfiAudioSource { + pub fn setup( + server: &'static server::FfiServer, + new_source: proto::NewAudioSourceRequest, + ) -> FfiResult { + let source_type = proto::AudioSourceType::from_i32(new_source.r#type).unwrap(); + let source_inner = match source_type { + proto::AudioSourceType::AudioSourceNative => { + let audio_source = NativeAudioSource::default(); + Ok::(AudioSource::Native(audio_source)) + } + _ => return Err(FfiError::InvalidRequest("unsupported audio source type")), + }?; + + let audio_source = Self { + handle_id: server.next_id(), + source_type, + source: source_inner, + }; + let source_info = proto::AudioSourceInfo::from(&audio_source); + + server + .ffi_handles() + .insert(audio_source.handle_id, Box::new(audio_source)); + + Ok(source_info) + } + + pub fn capture_frame( + &self, + server: &'static server::FfiServer, + capture: proto::CaptureAudioFrameRequest, + ) -> FfiResult<()> { + match self.source { + AudioSource::Native(ref source) => { + let buffer_handle = capture + .buffer_handle + .ok_or(FfiError::InvalidRequest("buffer_handle is empty"))? + .id as FfiHandleId; + + let frame = server + .ffi_handles() + .get(&buffer_handle) + .ok_or(FfiError::InvalidRequest("handle not found"))?; + + let frame = frame + .downcast_ref::() + .ok_or(FfiError::InvalidRequest("handle is not an audio frame"))?; + + source.capture_frame(frame); + } + } + + Ok(()) + } + + pub fn handle_id(&self) -> FfiHandleId { + self.handle_id + } + + pub fn source_type(&self) -> proto::AudioSourceType { + self.source_type + } + + pub fn inner_source(&self) -> &AudioSource { + &self.source + } +} diff --git a/livekit-ffi/src/server/conversion/mod.rs b/livekit-ffi/src/server/conversion/mod.rs deleted file mode 100644 index c4081f8..0000000 --- a/livekit-ffi/src/server/conversion/mod.rs +++ /dev/null @@ -1,196 +0,0 @@ -use crate::server::FFIHandleId; -use livekit::prelude::*; -use crate::proto; - -pub mod participant; -pub mod publication; -pub mod room; -pub mod video_frame; - -impl From for proto::FfiHandleId { - fn from(id: FFIHandleId) -> Self { - Self { id: id as u64 } - } -} - -macro_rules! impl_participant_into { - ($p:ty) => { - impl From<$p> for proto::ParticipantInfo { - fn from(p: $p) -> Self { - Self { - name: p.name(), - sid: p.sid().to_string(), - identity: p.identity().to_string(), - metadata: p.metadata(), - publications: p.tracks().iter().map(|(_, p)| p.into()).collect(), - } - } - } - }; -} - -impl_participant_into!(&LocalParticipant); -impl_participant_into!(&RemoteParticipant); -impl_participant_into!(&Participant); - -impl From for proto::TrackSource { - fn from(source: TrackSource) -> proto::TrackSource { - match source { - TrackSource::Unknown => proto::TrackSource::SourceUnknown, - TrackSource::Camera => proto::TrackSource::SourceCamera, - TrackSource::Microphone => proto::TrackSource::SourceMicrophone, - TrackSource::Screenshare => proto::TrackSource::SourceScreenshare, - TrackSource::ScreenshareAudio => proto::TrackSource::SourceScreenshareAudio, - } - } -} - -macro_rules! impl_publication_into { - ($p:ty) => { - impl From<$p> for proto::TrackPublicationInfo { - fn from(p: $p) -> Self { - Self { - name: p.name(), - sid: p.sid().to_string(), - kind: proto::TrackKind::from(p.kind()).into(), - source: proto::TrackSource::from(p.source()).into(), - dimension: Some(proto::Dimension { - width: p.dimension().0, - height: p.dimension().1, - }), - mime_type: p.mime_type(), - simulcasted: p.simulcasted(), - muted: p.muted(), - } - } - } - }; -} - -impl_publication_into!(&LocalTrackPublication); -impl_publication_into!(&RemoteTrackPublication); -impl_publication_into!(&TrackPublication); - -macro_rules! impl_track_into { - ($t:ty) => { - impl From<$t> for proto::TrackInfo { - fn from(track: $t) -> Self { - Self { - name: track.name(), - stream_state: proto::StreamState::from(track.stream_state()).into(), - sid: track.sid().to_string(), - kind: proto::TrackKind::from(track.kind()).into(), - muted: track.muted(), - } - } - } - }; -} - -impl_track_into!(&LocalAudioTrack); -impl_track_into!(&LocalVideoTrack); -impl_track_into!(&RemoteAudioTrack); -impl_track_into!(&RemoteVideoTrack); -impl_track_into!(&Track); -impl_track_into!(&LocalTrack); -impl_track_into!(&RemoteTrack); - -impl From for proto::TrackKind { - fn from(kind: TrackKind) -> Self { - match kind { - TrackKind::Audio => proto::TrackKind::KindAudio, - TrackKind::Video => proto::TrackKind::KindVideo, - } - } -} - -impl From for proto::StreamState { - fn from(state: StreamState) -> Self { - match state { - StreamState::Active => Self::StateActive, - StreamState::Paused => Self::StatePaused, - } - } -} - -impl proto::RoomEvent { - pub fn from(room_sid: impl Into, event: RoomEvent) -> Option { - let message = match event { - RoomEvent::ParticipantConnected(participant) => Some( - proto::room_event::Message::ParticipantConnected(proto::ParticipantConnected { - info: Some((&participant).into()), - }), - ), - RoomEvent::ParticipantDisconnected(participant) => { - Some(proto::room_event::Message::ParticipantDisconnected( - proto::ParticipantDisconnected { - info: Some((&participant).into()), - }, - )) - } - RoomEvent::TrackPublished { - publication, - participant, - } => Some(proto::room_event::Message::TrackPublished( - proto::TrackPublished { - participant_sid: participant.sid().to_string(), - publication: Some((&publication).into()), - }, - )), - RoomEvent::TrackUnpublished { - publication, - participant, - } => Some(proto::room_event::Message::TrackUnpublished( - proto::TrackUnpublished { - participant_sid: participant.sid().to_string(), - publication_sid: publication.sid().into(), - }, - )), - RoomEvent::TrackSubscribed { - track, - publication: _, - participant, - } => Some(proto::room_event::Message::TrackSubscribed( - proto::TrackSubscribed { - participant_sid: participant.sid().to_string(), - track: Some((&track).into()), - sink: Some(proto::VideoSinkInfo { - track_sid: track.sid().to_string(), - }), - }, - )), - RoomEvent::TrackUnsubscribed { - track, - publication: _, - participant, - } => Some(proto::room_event::Message::TrackUnsubscribed( - proto::TrackUnsubscribed { - participant_sid: participant.sid().to_string(), - track_sid: track.sid().to_string(), - }, - )), - _ => None, - }; - - message.map(|message| proto::RoomEvent { - room_sid: room_sid.into(), - message: Some(message), - }) - } -} - -impl From<&RoomSession> for proto::RoomInfo { - fn from(session: &RoomSession) -> Self { - Self { - sid: session.sid().into(), - name: session.name(), - metadata: session.metadata(), - local_participant: Some((&session.local_participant()).into()), - participants: session - .participants() - .iter() - .map(|(_, p)| p.into()) - .collect(), - } - } -} diff --git a/livekit-ffi/src/server/conversion/participant.rs b/livekit-ffi/src/server/conversion/participant.rs deleted file mode 100644 index e69de29..0000000 diff --git a/livekit-ffi/src/server/conversion/room.rs b/livekit-ffi/src/server/conversion/room.rs deleted file mode 100644 index e69de29..0000000 diff --git a/livekit-ffi/src/server/mod.rs b/livekit-ffi/src/server/mod.rs index 931f9b4..2954779 100644 --- a/livekit-ffi/src/server/mod.rs +++ b/livekit-ffi/src/server/mod.rs @@ -1,272 +1,677 @@ +use crate::{proto, FfiCallbackFn}; +use crate::{FfiAsyncId, FfiError, FfiHandle, FfiHandleId, FfiResult}; +use dashmap::DashMap; use lazy_static::lazy_static; use livekit::prelude::*; -use livekit::webrtc::video_frame::{native::VideoFrameBufferExt, BoxVideoFrame, VideoFrameBuffer}; -use crate::proto; -use parking_lot::{Mutex, RwLock}; +use livekit::webrtc::native::yuv_helper; +use livekit::webrtc::prelude::*; +use livekit::webrtc::video_frame::{native::I420BufferExt, BoxVideoFrameBuffer, I420Buffer}; +use parking_lot::Mutex; use prost::Message; -use std::any::Any; use std::collections::HashMap; -use std::panic; use std::slice; -use std::sync::atomic::AtomicU64; -use std::sync::atomic::{AtomicBool, Ordering}; -use thiserror::Error; -use tokio::sync::oneshot; -use tokio::task::JoinHandle; +use std::sync::atomic::{AtomicUsize, Ordering}; -mod conversion; -mod room; +pub mod audio_frame; +pub mod room; +pub mod utils; +pub mod video_frame; -#[derive(Error, Debug)] -pub enum FFIError { - #[error("the FFIServer isn't configured")] - NotConfigured, - #[error("failed to execute the FFICallback")] - CallbackFailed, -} - -pub type FFIHandleId = usize; -pub type FFIHandle = Box; - -type CallbackFn = unsafe extern "C" fn(*const u8, usize); // This "C" callback must be threadsafe +#[cfg(test)] +mod tests; lazy_static! { - static ref FFI_SERVER: FFIServer = FFIServer::default(); + pub static ref FFI_SERVER: FfiServer = FfiServer::default(); } -pub struct FFIConfig { - callback_fn: CallbackFn, +pub struct FfiConfig { + callback_fn: FfiCallbackFn, } -/// To use the FFI, the foreign language and the FFI server must share -/// the same memory space -pub struct FFIServer { - // Object owned by the foreign language - // The foreign language is responsible for freeing this memory - // - // NOTE: For VideoBuffers, we always store the enum VideoFrameBuffer - ffi_owned: RwLock>, - next_handle_id: AtomicU64, // FFIHandleId - next_async_id: AtomicU64, - - rooms: RwLock, oneshot::Sender<()>)>>, +pub struct FfiServer { + rooms: Mutex>, + /// Store all FFI handles inside an HashMap, if this isn't efficient enough + /// We can still use Box::into_raw & Box::from_raw in the future (but keep it safe for now) + ffi_handles: DashMap, + next_id: AtomicUsize, async_runtime: tokio::runtime::Runtime, - initialized: AtomicBool, - config: Mutex>, + config: Mutex>, } -impl Default for FFIServer { +impl Default for FfiServer { fn default() -> Self { Self { - ffi_owned: RwLock::new(HashMap::new()), - next_handle_id: AtomicU64::new(1), // 0 is considered invalid - next_async_id: AtomicU64::new(1), - rooms: RwLock::new(HashMap::new()), + rooms: Default::default(), + ffi_handles: Default::default(), + next_id: AtomicUsize::new(1), // 0 is invalid async_runtime: tokio::runtime::Builder::new_multi_thread() .enable_all() .build() .unwrap(), - initialized: Default::default(), config: Default::default(), } } } -impl FFIServer { - pub fn initialize(&self, init: &proto::InitializeRequest) { - if self.initialized() { - self.dispose(); - } - - self.initialized.store(true, Ordering::SeqCst); - *self.config.lock() = Some(FFIConfig { - callback_fn: unsafe { std::mem::transmute(init.event_callback_ptr) }, - }); - } - - pub fn dispose(&self) { - self.initialized.store(false, Ordering::SeqCst); - *self.config.lock() = None; - self.async_runtime.block_on(self.close()); - } - - pub async fn close(&self) { +// Using &'static self inside the implementation, not sure if this is really idiomatic +// It simplifies the code a lot tho. In most cases the server is used until the end of the process +impl FfiServer { + pub async fn dispose(&'static self) { // Close all rooms - for (_, (handle, shutdown_tx)) in self.rooms.write().drain() { - let _ = shutdown_tx.send(()); - let _ = handle.await; + for (_, room_handle) in self.rooms.lock().drain() { + let room = self.ffi_handles.remove(&room_handle); + if let Some(room) = room { + let ffi_room = room.1.downcast::().unwrap(); + ffi_room.close().await; + } } + // Drop all handles + self.ffi_handles.clear(); + + // Invalidate the config + *self.config.lock() = None; } - pub fn add_room(&self, sid: RoomSid, handle: (JoinHandle<()>, oneshot::Sender<()>)) { - self.rooms.write().insert(sid, handle); + pub fn next_id(&'static self) -> usize { + self.next_id.fetch_add(1, Ordering::Relaxed) } - pub fn initialized(&self) -> bool { - self.initialized.load(Ordering::SeqCst) + pub fn ffi_handles(&'static self) -> &DashMap { + &self.ffi_handles } - pub fn next_handle_id(&self) -> FFIHandleId { - self.next_handle_id.fetch_add(1, Ordering::SeqCst) as FFIHandleId + pub fn rooms(&'static self) -> &Mutex> { + &self.rooms } - pub fn next_async_id(&self) -> u64 { - self.next_async_id.fetch_add(1, Ordering::SeqCst) - } - - pub fn insert_handle(&self, handle_id: FFIHandleId, handle: FFIHandle) { - self.ffi_owned.write().insert(handle_id, handle); - } - - pub fn release_handle(&self, handle_id: FFIHandleId) -> Option { - self.ffi_owned.write().remove(&handle_id) - } - - pub fn send_event( - &self, - message: proto::ffi_event::Message, - async_id: Option, - ) -> Result<(), FFIError> { - let config = self.config.lock(); - - if !self.initialized() { - Err(FFIError::NotConfigured)? - } + pub fn send_event(&'static self, message: proto::ffi_event::Message) -> FfiResult<()> { + let callback_fn = self + .config + .lock() + .as_ref() + .map_or_else(|| Err(FfiError::NotConfigured), |c| Ok(c.callback_fn))?; let message = proto::FfiEvent { - async_id, message: Some(message), } .encode_to_vec(); - let config = config.as_ref().unwrap(); - if let Err(err) = panic::catch_unwind(|| unsafe { - (config.callback_fn)(message.as_ptr(), message.len()); - }) { - eprintln!("panic when sending ffi event: {:?}", err); - Err(FFIError::CallbackFailed)? + unsafe { + callback_fn(message.as_ptr(), message.len()); } - Ok(()) } +} - pub fn handle_request(&self, message: proto::ffi_request::Message) -> proto::FfiResponse { - match message { - proto::ffi_request::Message::AsyncConnect(connect) => { - let async_id = self.next_async_id(); - self.async_runtime - .spawn(room::create_room(&FFI_SERVER, async_id, connect)); - - return proto::FfiResponse { - async_id: Some(async_id), - ..Default::default() - }; - } - proto::ffi_request::Message::ToI420(to_i420) => { - let mut buffer_info = None; - let buffer = self.release_handle(to_i420.buffer.unwrap().id as FFIHandleId); - - if let Some(buffer) = buffer { - if let Ok(buffer) = buffer.downcast::>() { - let handle_id = self.next_handle_id(); - let buffer = buffer.to_i420(); - buffer_info = Some(proto::VideoFrameBufferInfo::from(handle_id, &buffer)); - self.insert_handle(handle_id, Box::new(buffer)); - } - } - - return proto::FfiResponse { - message: Some(proto::ffi_response::Message::ToI420( - proto::ToI420Response { - new_buffer: buffer_info, - }, - )), - ..Default::default() - }; - } - proto::ffi_request::Message::ToArgb(to_argb) => { - let ffi_owned = self.ffi_owned.read(); - let buffer = ffi_owned.get(&(to_argb.buffer.unwrap().id as FFIHandleId)); - - if let Some(buffer) = buffer { - if let Some(buffer) = buffer.downcast_ref::>() { - let dst_buf = unsafe { - slice::from_raw_parts_mut( - to_argb.dst_ptr as *mut u8, - (to_argb.dst_stride * to_argb.dst_height) as usize, - ) - }; - - if let Err(err) = buffer.to_argb( - proto::VideoFormatType::from_i32(to_argb.dst_format) - .unwrap() - .into(), - dst_buf, - to_argb.dst_stride, - to_argb.dst_width, - to_argb.dst_height, - ) { - eprintln!("failed to convert videoframe to argb: {:?}", err); - } - } - } - } - _ => {} +impl FfiServer { + fn on_initialize( + &'static self, + init: proto::InitializeRequest, + ) -> FfiResult { + if self.config.lock().is_some() { + return Err(FfiError::AlreadyInitialized); } - proto::FfiResponse::default() + // # SAFETY: The foreign language is responsible for ensuring that the callback function is valid + unsafe { + *self.config.lock() = Some(FfiConfig { + callback_fn: std::mem::transmute(init.event_callback_ptr), + }); + } + + Ok(proto::InitializeResponse::default()) + } + + fn on_dispose( + &'static self, + dispose: proto::DisposeRequest, + ) -> FfiResult { + *self.config.lock() = None; + + let close = self.dispose(); + if !dispose.r#async { + self.async_runtime.block_on(close); + Ok(proto::DisposeResponse::default()) + } else { + let async_id = self.next_id(); + self.async_runtime.spawn(async move { + close.await; + }); + Ok(proto::DisposeResponse { + async_id: Some(proto::FfiAsyncId { + id: async_id as u64, + }), + }) + } + } + + // Room + + fn on_connect( + &'static self, + connect: proto::ConnectRequest, + ) -> FfiResult { + let async_id = self.next_id(); + self.async_runtime.spawn(async move { + // Try to connect to the Room + let res = room::FfiRoom::connect(&self, connect).await; + + // match res + match res { + Ok(room_info) => { + let _ = self.send_event(proto::ffi_event::Message::Connect( + proto::ConnectCallback { + async_id: Some(async_id.into()), + error: None, + room: Some(room_info), + }, + )); + } + Err(err) => { + let _ = self.send_event(proto::ffi_event::Message::Connect( + proto::ConnectCallback { + async_id: Some(async_id.into()), + error: Some(err.to_string()), + room: None, + }, + )); + } + } + }); + + Ok(proto::ConnectResponse { + async_id: Some(proto::FfiAsyncId { + id: async_id as u64, + }), + }) + } + + fn on_disconnect( + &'static self, + _disconnect: proto::DisconnectRequest, + ) -> FfiResult { + Ok(proto::DisconnectResponse::default()) + } + + fn on_publish_track( + &'static self, + publish: proto::PublishTrackRequest, + ) -> FfiResult { + let async_id = self.next_id() as FfiAsyncId; + tokio::spawn(async move { + let res = async { + let room_handle = publish + .room_handle + .as_ref() + .ok_or(FfiError::InvalidRequest("room_handle is empty"))? + .id as FfiHandleId; + + let room = self + .ffi_handles + .get(&room_handle) + .ok_or(FfiError::InvalidRequest("room not found"))?; + + let room = room + .downcast_ref::() + .ok_or(FfiError::InvalidRequest("room is not a FfiRoom"))?; + + let track_handle = publish + .track_handle + .as_ref() + .ok_or(FfiError::InvalidRequest("track_handle is empty"))? + .id as FfiHandleId; + + let track = self + .ffi_handles + .get(&track_handle) + .ok_or(FfiError::InvalidRequest("track not found"))?; + + let track = track + .downcast_ref::() + .ok_or(FfiError::InvalidRequest("track is not a LocalTrack"))?; + + let publication = room + .session() + .local_participant() + .publish_track( + track.clone(), + publish.options.map(Into::into).unwrap_or_default(), + ) + .await?; + + Ok::(publication) + } + .await; + + if let Err(err) = self.send_event(proto::ffi_event::Message::PublishTrack( + proto::PublishTrackCallback { + async_id: Some(async_id.into()), + error: res.as_ref().err().map(|e| e.to_string()), + publication: res.as_ref().ok().map(Into::into), + }, + )) { + log::warn!("error sending PublishTrack callback: {}", err); + } + }); + + Ok(proto::PublishTrackResponse { + async_id: Some(async_id.into()), + }) + } + + fn on_unpublish_track( + &'static self, + _unpublish: proto::UnpublishTrackRequest, + ) -> FfiResult { + Ok(proto::UnpublishTrackResponse::default()) + } + + // Track + fn on_create_video_track( + &'static self, + create: proto::CreateVideoTrackRequest, + ) -> FfiResult { + let handle_id = create + .source_handle + .as_ref() + .ok_or(FfiError::InvalidRequest("source_handle is empty"))? + .id as FfiHandleId; + + let source = self + .ffi_handles + .get(&handle_id) + .ok_or(FfiError::InvalidRequest("source not found"))?; + + let source = source + .downcast_ref::() + .ok_or(FfiError::InvalidRequest("handle is not a video source"))?; + + let source = source.inner_source().clone(); + let video_track = match source { + video_frame::VideoSource::Native(native_source) => LocalVideoTrack::create_video_track( + &create.name, + create.options.unwrap_or_default().into(), + native_source, + ), + }; + + let handle_id = self.next_id() as FfiHandleId; + let track_info = proto::TrackInfo::from_local_video_track(handle_id, &video_track); + + self.ffi_handles + .insert(handle_id, Box::new(LocalTrack::Video(video_track))); + + Ok(proto::CreateVideoTrackResponse { + track: Some(track_info), + }) + } + + fn on_create_audio_track( + &'static self, + create: proto::CreateAudioTrackRequest, + ) -> FfiResult { + let handle_id = create + .source_handle + .as_ref() + .ok_or(FfiError::InvalidRequest("source_handle is empty"))? + .id as FfiHandleId; + + let source = self + .ffi_handles + .get(&handle_id) + .ok_or(FfiError::InvalidRequest("source not found"))?; + + let source = source + .downcast_ref::() + .ok_or(FfiError::InvalidRequest("handle is not an audio source"))?; + + let source = source.inner_source().clone(); + let audio_track = match source { + audio_frame::AudioSource::Native(native_source) => LocalAudioTrack::create_audio_track( + &create.name, + create.options.unwrap_or_default().into(), + native_source, + ), + }; + + let handle_id = self.next_id() as FfiHandleId; + let track_info = proto::TrackInfo::from_local_audio_track(handle_id, &audio_track); + + self.ffi_handles + .insert(handle_id, Box::new(LocalTrack::Audio(audio_track))); + + Ok(proto::CreateAudioTrackResponse { + track: Some(track_info), + }) + } + + // Video + + fn on_alloc_video_buffer( + &'static self, + alloc: proto::AllocVideoBufferRequest, + ) -> FfiResult { + let frame_type = proto::VideoFrameBufferType::from_i32(alloc.r#type).unwrap(); + let buffer: BoxVideoFrameBuffer = match frame_type { + proto::VideoFrameBufferType::I420 => { + Box::new(I420Buffer::new(alloc.width, alloc.height)) + } + _ => return Err(FfiError::InvalidRequest("frame type is not supported")), + }; + + let handle_id = self.next_id(); + let buffer_info = proto::VideoFrameBufferInfo::from(handle_id, &buffer); + self.ffi_handles.insert(handle_id, Box::new(buffer)); + + Ok(proto::AllocVideoBufferResponse { + buffer: Some(buffer_info), + }) + } + + fn on_new_video_stream( + &'static self, + new_stream: proto::NewVideoStreamRequest, + ) -> FfiResult { + let stream_info = video_frame::FfiVideoStream::setup(&self, new_stream)?; + Ok(proto::NewVideoStreamResponse { + stream: Some(stream_info), + }) + } + + fn on_new_video_source( + &'static self, + new_source: proto::NewVideoSourceRequest, + ) -> FfiResult { + let source_info = video_frame::FfiVideoSource::setup(&self, new_source)?; + Ok(proto::NewVideoSourceResponse { + source: Some(source_info), + }) + } + + fn on_capture_video_frame( + &'static self, + push: proto::CaptureVideoFrameRequest, + ) -> FfiResult { + let handle_id = push + .source_handle + .as_ref() + .ok_or(FfiError::InvalidRequest("source_handle is empty"))? + .id as FfiHandleId; + + let video_source = self + .ffi_handles + .get(&handle_id) + .ok_or(FfiError::InvalidRequest("source not found"))?; + + let video_source = video_source + .downcast_ref::() + .ok_or(FfiError::InvalidRequest("handle is not a video source"))?; + + video_source.capture_frame(self, push)?; + Ok(proto::CaptureVideoFrameResponse::default()) + } + + fn on_to_i420( + &'static self, + to_i420: proto::ToI420Request, + ) -> FfiResult { + let from = to_i420 + .from + .ok_or(FfiError::InvalidRequest("from is empty"))?; + + let i420 = match from { + proto::to_i420_request::From::Argb(argb_info) => { + let mut i420 = I420Buffer::new(argb_info.width, argb_info.height); + let argb_format = proto::VideoFormatType::from_i32(argb_info.format).unwrap(); + let argb_ptr = argb_info.ptr as *const u8; + let argb_len = (argb_info.stride * argb_info.height) as usize; + let argb = unsafe { slice::from_raw_parts(argb_ptr, argb_len) }; + let argb_stride = argb_info.stride; + + let (stride_y, stride_u, stride_v) = i420.strides(); + let (data_y, data_u, data_v) = i420.data_mut(); + let width = argb_info.width as i32; + let mut height = argb_info.height as i32; + if to_i420.flip_y { + height = -height; + } + + match argb_format { + proto::VideoFormatType::FormatArgb => { + yuv_helper::argb_to_i420( + argb, + argb_stride, + data_y, + stride_y, + data_u, + stride_u, + data_v, + stride_v, + width, + height, + ) + .unwrap(); + } + proto::VideoFormatType::FormatAbgr => { + yuv_helper::abgr_to_i420( + argb, + argb_stride, + data_y, + stride_y, + data_u, + stride_u, + data_v, + stride_v, + width, + height, + ) + .unwrap(); + } + _ => return Err(FfiError::InvalidRequest("the format is not supported")), + } + + i420 + } + proto::to_i420_request::From::Buffer(handle) => { + let handle_id = handle.id as FfiHandleId; + let buffer = self + .ffi_handles + .get(&handle_id) + .ok_or(FfiError::InvalidRequest("handle not found"))?; + let i420 = buffer + .downcast_ref::() + .ok_or(FfiError::InvalidRequest("handle is not a video buffer"))? + .to_i420(); + + i420 + } + }; + + let i420: BoxVideoFrameBuffer = Box::new(i420); + let handle_id = self.next_id() as FfiHandleId; + let buffer_info = proto::VideoFrameBufferInfo::from(handle_id, &i420); + self.ffi_handles.insert(handle_id, Box::new(i420)); + Ok(proto::ToI420Response { + buffer: Some(buffer_info), + }) + } + + fn on_to_argb( + &'static self, + to_argb: proto::ToArgbRequest, + ) -> FfiResult { + let handle_id = to_argb + .buffer + .ok_or(FfiError::InvalidRequest("buffer is empty"))? + .id as FfiHandleId; + + let buffer = self + .ffi_handles + .get(&handle_id) + .ok_or(FfiError::InvalidRequest("buffer is not found"))?; + + let buffer = buffer + .downcast_ref::() + .ok_or(FfiError::InvalidRequest("handle is not a video buffer"))?; + + let flip_y = to_argb.flip_y; + let dst_format = proto::VideoFormatType::from_i32(to_argb.dst_format).unwrap(); + let dst_buf = unsafe { + slice::from_raw_parts_mut( + to_argb.dst_ptr as *mut u8, + (to_argb.dst_stride * to_argb.dst_height) as usize, + ) + }; + let dst_stride = to_argb.dst_stride; + let dst_width = to_argb.dst_width as i32; + let mut dst_height = to_argb.dst_height as i32; + if flip_y { + dst_height = -dst_height; + } + + buffer + .to_argb( + dst_format.into(), + dst_buf, + dst_stride, + dst_width, + dst_height, + ) + .unwrap(); + + Ok(proto::ToArgbResponse::default()) + } + + // Audio + + fn on_alloc_audio_buffer( + &'static self, + alloc: proto::AllocAudioBufferRequest, + ) -> FfiResult { + let frame = AudioFrame::new( + alloc.sample_rate, + alloc.num_channels, + alloc.samples_per_channel, + ); + + let handle_id = self.next_id() as FfiHandleId; + let frame_info = proto::AudioFrameBufferInfo::from(handle_id, &frame); + self.ffi_handles.insert(handle_id, Box::new(frame)); + + Ok(proto::AllocAudioBufferResponse { + buffer: Some(frame_info), + }) + } + + fn on_new_audio_stream( + &'static self, + new_stream: proto::NewAudioStreamRequest, + ) -> FfiResult { + let stream_info = audio_frame::FfiAudioSream::setup(self, new_stream)?; + Ok(proto::NewAudioStreamResponse { + stream: Some(stream_info), + }) + } + + fn on_new_audio_source( + &'static self, + new_source: proto::NewAudioSourceRequest, + ) -> FfiResult { + let source_info = audio_frame::FfiAudioSource::setup(self, new_source)?; + Ok(proto::NewAudioSourceResponse { + source: Some(source_info), + }) + } + + fn on_capture_audio_frame( + &'static self, + push: proto::CaptureAudioFrameRequest, + ) -> FfiResult { + let handle_id = push + .source_handle + .as_ref() + .ok_or(FfiError::InvalidRequest("handle is empty"))? + .id as FfiHandleId; + + let audio_source = self + .ffi_handles + .get(&handle_id) + .ok_or(FfiError::InvalidRequest("audio_source not found"))?; + + let audio_source = audio_source + .downcast_ref::() + .ok_or(FfiError::InvalidRequest("handle is not a video source"))?; + + audio_source.capture_frame(self, push)?; + Ok(proto::CaptureAudioFrameResponse::default()) + } + + pub fn handle_request( + &'static self, + request: proto::FfiRequest, + ) -> FfiResult { + let request = request + .message + .ok_or(FfiError::InvalidRequest("message is empty"))?; + + let mut res = proto::FfiResponse::default(); + res.message = Some(match request { + proto::ffi_request::Message::Initialize(init) => { + proto::ffi_response::Message::Initialize(self.on_initialize(init)?) + } + proto::ffi_request::Message::Dispose(dispose) => { + proto::ffi_response::Message::Dispose(self.on_dispose(dispose)?) + } + proto::ffi_request::Message::Connect(connect) => { + proto::ffi_response::Message::Connect(self.on_connect(connect)?) + } + proto::ffi_request::Message::Disconnect(disconnect) => { + proto::ffi_response::Message::Disconnect(self.on_disconnect(disconnect)?) + } + proto::ffi_request::Message::PublishTrack(publish) => { + proto::ffi_response::Message::PublishTrack(self.on_publish_track(publish)?) + } + proto::ffi_request::Message::UnpublishTrack(unpublish) => { + proto::ffi_response::Message::UnpublishTrack(self.on_unpublish_track(unpublish)?) + } + proto::ffi_request::Message::CreateVideoTrack(create) => { + proto::ffi_response::Message::CreateVideoTrack(self.on_create_video_track(create)?) + } + proto::ffi_request::Message::CreateAudioTrack(create) => { + proto::ffi_response::Message::CreateAudioTrack(self.on_create_audio_track(create)?) + } + proto::ffi_request::Message::AllocVideoBuffer(alloc) => { + proto::ffi_response::Message::AllocVideoBuffer(self.on_alloc_video_buffer(alloc)?) + } + proto::ffi_request::Message::NewVideoStream(new_stream) => { + proto::ffi_response::Message::NewVideoStream(self.on_new_video_stream(new_stream)?) + } + proto::ffi_request::Message::NewVideoSource(new_source) => { + proto::ffi_response::Message::NewVideoSource(self.on_new_video_source(new_source)?) + } + proto::ffi_request::Message::CaptureVideoFrame(push) => { + proto::ffi_response::Message::CaptureVideoFrame(self.on_capture_video_frame(push)?) + } + proto::ffi_request::Message::ToI420(to_i420) => { + proto::ffi_response::Message::ToI420(self.on_to_i420(to_i420)?) + } + proto::ffi_request::Message::ToArgb(to_argb) => { + proto::ffi_response::Message::ToArgb(self.on_to_argb(to_argb)?) + } + proto::ffi_request::Message::AllocAudioBuffer(alloc) => { + proto::ffi_response::Message::AllocAudioBuffer(self.on_alloc_audio_buffer(alloc)?) + } + proto::ffi_request::Message::NewAudioStream(new_stream) => { + proto::ffi_response::Message::NewAudioStream(self.on_new_audio_stream(new_stream)?) + } + proto::ffi_request::Message::NewAudioSource(new_source) => { + proto::ffi_response::Message::NewAudioSource(self.on_new_audio_source(new_source)?) + } + proto::ffi_request::Message::CaptureAudioFrame(push) => { + proto::ffi_response::Message::CaptureAudioFrame(self.on_capture_audio_frame(push)?) + } + }); + + Ok(res) } } - -/// This function is threadsafe, this is useful to run synchronous requests in another thread (e.g -/// color conversion) -#[no_mangle] -pub extern "C" fn livekit_ffi_request( - data: *const u8, - len: usize, - data_ptr: *mut *const u8, - data_len: *mut usize, -) -> FFIHandleId { - let data = unsafe { slice::from_raw_parts(data, len) }; - let res = proto::FfiRequest::decode(data); - if let Err(ref err) = res { - eprintln!("failed to decode FfiRequest: {:?}", err); - return 0; - } - - if res.as_ref().unwrap().message.is_none() { - eprintln!("request message is empty"); - return 0; - } - - let message = res.unwrap().message.unwrap(); - if let proto::ffi_request::Message::Initialize(ref init) = message { - FFI_SERVER.initialize(init); - } - - if let proto::ffi_request::Message::Dispose(_) = message { - FFI_SERVER.dispose(); - } - - if !FFI_SERVER.initialized() { - eprintln!("the FFIServer isn't initialized"); - return 0; - } - - let res = FFI_SERVER.handle_request(message); - let buf = res.encode_to_vec(); - - unsafe { - *data_ptr = buf.as_ptr(); - *data_len = buf.len(); - } - - let handle_id = FFI_SERVER.next_handle_id(); - FFI_SERVER.insert_handle(handle_id, Box::new(buf)); - handle_id -} - -#[no_mangle] -pub extern "C" fn livekit_ffi_drop_handle(handle_id: FFIHandleId) -> bool { - FFI_SERVER.release_handle(handle_id).is_some() // Free the memory -} diff --git a/livekit-ffi/src/server/room.rs b/livekit-ffi/src/server/room.rs index ed76b70..9909f92 100644 --- a/livekit-ffi/src/server/room.rs +++ b/livekit-ffi/src/server/room.rs @@ -1,54 +1,65 @@ -use crate::server::FFIServer; -use futures_util::stream::StreamExt; +use crate::server::FfiServer; +use crate::{proto, FfiHandleId, FfiResult}; use livekit::prelude::*; -use livekit::webrtc::video_stream::native::NativeVideoStream; use tokio::sync::{mpsc, oneshot}; -use crate::proto; +use tokio::task::JoinHandle; -pub async fn create_room( - server: &'static FFIServer, - async_id: u64, - connect: proto::ConnectRequest, -) { - let res = Room::connect(&connect.url, &connect.token).await; - if let Err(err) = &res { - // Failed to connect to the room - let _ = server.send_event( - proto::ffi_event::Message::ConnectEvent(proto::ConnectEvent { - success: false, - room: None, - }), - Some(async_id), - ); - return; +pub struct FfiRoom { + room: Room, + handle_id: FfiHandleId, + handle: JoinHandle<()>, + close_tx: oneshot::Sender<()>, +} + +impl FfiRoom { + pub async fn connect( + server: &'static FfiServer, + connect: proto::ConnectRequest, + ) -> FfiResult { + let (room, events) = Room::connect(&connect.url, &connect.token).await?; + let (close_tx, close_rx) = oneshot::channel(); + let session = room.session(); + let next_id = server.next_id() as FfiHandleId; + + let handle = tokio::spawn(room_task( + server, + session.clone(), + next_id, + events, + close_rx, + )); + let room_info = proto::RoomInfo::from_session(next_id, &session); + + let ffi_room = Self { + handle_id: next_id, + room, + handle, + close_tx, + }; + server.ffi_handles().insert(next_id, Box::new(ffi_room)); + server.rooms().lock().insert(session.sid(), next_id); + + Ok(room_info) } - let (room, events) = res.unwrap(); - let session = room.session(); + pub async fn close(self) { + self.room.close().await; + let _ = self.close_tx.send(()); + let _ = self.handle.await; + } - // Successfully connected to the room - let _ = server.send_event( - proto::ffi_event::Message::ConnectEvent(proto::ConnectEvent { - success: true, - room: Some((&session).into()), - }), - Some(async_id), - ); - - // Add the room to the server and listen to the incoming events - let (close_tx, close_rx) = oneshot::channel(); - let room_handle = tokio::spawn(room_task(server, room, events, close_rx)); - server.add_room(session.sid(), (room_handle, close_tx)); + pub fn session(&self) -> RoomSession { + self.room.session() + } } async fn room_task( - server: &'static FFIServer, - room: Room, + server: &'static FfiServer, + session: RoomSession, + room_handle: FfiHandleId, mut events: mpsc::UnboundedReceiver, mut close_rx: oneshot::Receiver<()>, ) { - let session = room.session(); - tokio::spawn(participant_task(Participant::Local( session.local_participant(), ))); @@ -56,24 +67,14 @@ async fn room_task( loop { tokio::select! { Some(event) = events.recv() => { - if let Some(event) = proto::RoomEvent::from(session.sid(), event.clone()) { - let _ = server.send_event(proto::ffi_event::Message::RoomEvent(event), None); + if let Some(event) = proto::RoomEvent::from(room_handle, event.clone()) { + let _ = server.send_event(proto::ffi_event::Message::RoomEvent(event)); } match event { RoomEvent::ParticipantConnected(p) => { tokio::spawn(participant_task(Participant::Remote(p))); } - RoomEvent::TrackSubscribed { - track, - publication: _, - participant: _, - } => { - if let RemoteTrack::Video(video_track) = track { - let video_stream = NativeVideoStream::new(video_track.rtc_track()); - tokio::spawn(video_frame_task(server, video_track.sid(), video_stream)); - } - } _ => {} } }, @@ -82,40 +83,11 @@ async fn room_task( } }; } - - room.close().await; } async fn participant_task(participant: Participant) { let mut participant_events = participant.register_observer(); - while let Some(event) = participant_events.recv().await { + while let Some(_event) = participant_events.recv().await { // TODO(theomonnom): convert event to proto } } - -async fn video_frame_task( - server: &'static FFIServer, - track_sid: TrackSid, - mut stream: NativeVideoStream, -) { - while let Some(frame) = stream.next().await { - let handle_id = server.next_handle_id(); - let frame_info = proto::VideoFrameInfo::from(&frame); - let buffer_info = proto::VideoFrameBufferInfo::from(handle_id, &frame.buffer); - server.insert_handle(handle_id, Box::new(frame.buffer)); - - // Send the received frame to the FFI language. - let _ = server.send_event( - proto::ffi_event::Message::TrackEvent(proto::TrackEvent { - track_sid: track_sid.to_string(), - message: Some(proto::track_event::Message::FrameReceived( - proto::FrameReceived { - frame: Some(frame_info), - buffer: Some(buffer_info), - }, - )), - }), - None, - ); - } -} diff --git a/livekit-ffi/src/server/tests.rs b/livekit-ffi/src/server/tests.rs new file mode 100644 index 0000000..5e32ee6 --- /dev/null +++ b/livekit-ffi/src/server/tests.rs @@ -0,0 +1,334 @@ +use std::time::Duration; + +use crate::FfiHandleId; +use crate::{proto, server}; +use livekit_api::access_token::{AccessToken, VideoGrants}; + +// Small FfiClient implementation used for testing +// This can be used as an example for a real implementation +mod client { + use crate::{ + livekit_ffi_drop_handle, livekit_ffi_request, proto, FfiCallbackFn, FfiHandleId, + INVALID_HANDLE, + }; + use lazy_static::lazy_static; + use prost::Message; + use std::sync::Mutex; + use tokio::sync::mpsc; + + lazy_static! { + static ref EVENT_TX: Mutex>> = + Default::default(); + pub static ref FFI_CLIENT: Mutex = Default::default(); + } + + pub struct FfiHandle(pub FfiHandleId); + + pub struct FfiClient { + event_rx: mpsc::UnboundedReceiver, + } + + impl Default for FfiClient { + fn default() -> Self { + let (event_tx, event_rx) = mpsc::unbounded_channel(); + *EVENT_TX.lock().unwrap() = Some(event_tx); + Self { event_rx } + } + } + + impl FfiClient { + pub async fn recv_event(&mut self) -> proto::ffi_event::Message { + self.event_rx.recv().await.unwrap() + } + + pub fn initialize(&self) { + self.send_request(proto::FfiRequest { + message: Some(proto::ffi_request::Message::Initialize( + proto::InitializeRequest { + event_callback_ptr: test_events_callback as FfiCallbackFn as u64, + }, + )), + }); + } + + pub fn send_request(&self, request: proto::FfiRequest) -> proto::FfiResponse { + let data = request.encode_to_vec(); + + let mut res_ptr: Box<*const u8> = Box::new(std::ptr::null()); + let mut res_len: Box = Box::new(0); + + let handle = livekit_ffi_request( + data.as_ptr(), + data.len(), + res_ptr.as_mut(), + res_len.as_mut(), + ); + let handle = FfiHandle(handle); // drop at end of scope + + let res = unsafe { + assert_ne!(handle.0, INVALID_HANDLE); + assert_ne!(*res_ptr, std::ptr::null()); + assert_ne!(*res_len, 0); + std::slice::from_raw_parts(*res_ptr, *res_len) + }; + + proto::FfiResponse::decode(res).unwrap() + } + } + + impl Drop for FfiHandle { + fn drop(&mut self) { + assert!(livekit_ffi_drop_handle(self.0)); + } + } + + #[no_mangle] + unsafe extern "C" fn test_events_callback(data_ptr: *const u8, len: usize) { + let data = unsafe { std::slice::from_raw_parts(data_ptr, len) }; + let event = proto::FfiEvent::decode(data).unwrap(); + EVENT_TX + .lock() + .unwrap() + .as_ref() + .unwrap() + .send(event.message.unwrap()) + .unwrap(); + } +} + +struct TestScope {} + +impl TestScope { + fn new() -> (Self, std::sync::MutexGuard<'static, client::FfiClient>) { + // Run one test at a time + let client = client::FFI_CLIENT.lock().unwrap(); + + (TestScope {}, client) + } +} + +impl Drop for TestScope { + fn drop(&mut self) { + // At the end of a test, no more handle should exist + assert!(server::FFI_SERVER.ffi_handles().is_empty()); + } +} + +fn test_env() -> (String, String, String) { + let lk_url = std::env::var("LK_TEST_URL").expect("LK_TEST_URL isn't set"); + let lk_api_key = std::env::var("LK_TEST_API_KEY").expect("LK_TEST_API_KEY isn't set"); + let lk_api_secret = std::env::var("LK_TEST_API_SECRET").expect("LK_TEST_API_SECRET isn't set"); + (lk_url, lk_api_key, lk_api_secret) +} + +macro_rules! wait_for_event { + ($client:ident, $variant:ident, $timeout:expr) => { + tokio::time::timeout(Duration::from_secs($timeout), async { + loop { + let event = $client.recv_event().await; + if let proto::ffi_event::Message::$variant(event) = event { + return event; + } + } + }) + }; +} + +#[test] +fn create_i420_buffer() { + let (_test, client) = TestScope::new(); + + // Create a new I420Buffer + let res = client.send_request(proto::FfiRequest { + message: Some(proto::ffi_request::Message::AllocVideoBuffer( + proto::AllocVideoBufferRequest { + r#type: proto::VideoFrameBufferType::I420 as i32, + width: 640, + height: 480, + }, + )), + }); + + let proto::ffi_response::Message::AllocVideoBuffer(alloc) = res.message.unwrap() else { + panic!("unexpected response"); + }; + + // Convert to I420 (copy/no-op) + let i420_handle = client::FfiHandle(alloc.buffer.unwrap().handle.unwrap().id as FfiHandleId); + + let res = client.send_request(proto::FfiRequest { + message: Some(proto::ffi_request::Message::ToI420(proto::ToI420Request { + flip_y: false, + from: Some(proto::to_i420_request::From::Buffer(proto::FfiHandleId { + id: i420_handle.0 as u64, + })), + })), + }); + + let proto::ffi_response::Message::ToI420(to_i420) = res.message.unwrap() else { + panic!("unexpected response"); + }; + + // Make sure to drop the handles + client::FfiHandle(to_i420.buffer.unwrap().handle.unwrap().id as FfiHandleId); +} + +#[test] +#[ignore] // Ignore for now ( need to setup GHA ) +fn publish_video_track() { + let (test, mut client) = TestScope::new(); + let (lk_url, lk_api_key, lk_api_secret) = test_env(); + + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap() + .block_on(async { + client.initialize(); + + let token = AccessToken::with_api_key(&lk_api_key, &lk_api_secret) + .with_grants(VideoGrants { + room: "livekit-ffi-test".to_string(), + ..Default::default() + }) + .with_identity("video_test") + .to_jwt() + .unwrap(); + + // Connect to the room + client.send_request(proto::FfiRequest { + message: Some(proto::ffi_request::Message::Connect( + proto::ConnectRequest { + url: lk_url.clone(), + token, + ..Default::default() + }, + )), + }); + + let connect = wait_for_event!(client, Connect, 5).await.unwrap(); + assert!(connect.error.is_none()); + + let room_handle = + client::FfiHandle(connect.room.unwrap().handle.unwrap().id as FfiHandleId); + + // Create a new VideoSource + let res = client.send_request(proto::FfiRequest { + message: Some(proto::ffi_request::Message::NewVideoSource( + proto::NewVideoSourceRequest { + r#type: proto::VideoSourceType::VideoSourceNative as i32, + }, + )), + }); + + const VIDEO_WIDTH: u32 = 640; + const VIDEO_HEIGHT: u32 = 480; + const VIDEO_FPS: f64 = 8.0; + + let proto::ffi_response::Message::NewVideoSource(new_video_source) = + res.message.unwrap() else { + panic!("unexpected response"); + }; + + let source_handle = client::FfiHandle( + new_video_source.source.unwrap().handle.unwrap().id as FfiHandleId, + ); + + // Create a new VideoTrack + let res = client.send_request(proto::FfiRequest { + message: Some(proto::ffi_request::Message::CreateVideoTrack( + proto::CreateVideoTrackRequest { + name: "video_test".to_string(), + source_handle: Some(proto::FfiHandleId { + id: source_handle.0 as u64, + }), + options: Some(proto::VideoCaptureOptions { + resolution: Some(proto::VideoResolution { + width: VIDEO_WIDTH, + height: VIDEO_HEIGHT, + frame_rate: VIDEO_FPS, + }), + }), + }, + )), + }); + + let proto::ffi_response::Message::CreateVideoTrack(create_video_track) = + res.message.unwrap() else { + panic!("unexpected response"); + }; + + let track_handle = client::FfiHandle( + create_video_track.track.unwrap().opt_handle.unwrap().id as FfiHandleId, + ); + + let publish_options = proto::TrackPublishOptions { + name: "video_test".to_string(), + video_codec: proto::VideoCodec::H264 as i32, + source: proto::TrackSource::SourceCamera as i32, + ..Default::default() + }; + + // Publish the VideoTrack + client.send_request(proto::FfiRequest { + message: Some(proto::ffi_request::Message::PublishTrack( + proto::PublishTrackRequest { + room_handle: Some(proto::FfiHandleId { + id: room_handle.0 as u64, + }), + track_handle: Some(proto::FfiHandleId { + id: track_handle.0 as u64, + }), + options: Some(publish_options), + }, + )), + }); + + let publish_track = wait_for_event!(client, PublishTrack, 5).await.unwrap(); + assert!(publish_track.error.is_none()); + + // Send red frames + let rgba: Vec = vec![0xff0000ff; (VIDEO_WIDTH * VIDEO_HEIGHT) as usize]; + let res = client.send_request(proto::FfiRequest { + message: Some(proto::ffi_request::Message::ToI420(proto::ToI420Request { + flip_y: false, + from: Some(proto::to_i420_request::From::Argb(proto::ArgbBufferInfo { + ptr: rgba.as_ptr() as u64, + format: proto::VideoFormatType::FormatAbgr as i32, + width: VIDEO_WIDTH, + height: VIDEO_HEIGHT, + stride: VIDEO_WIDTH * 4, + })), + })), + }); + + let proto::ffi_response::Message::ToI420(to_i420) = res.message.unwrap() else { + panic!("unexpected response"); + }; + + let buffer_handle = + client::FfiHandle(to_i420.buffer.unwrap().handle.unwrap().id as FfiHandleId); + + // 2 seconds + for _ in 0..16 { + client.send_request(proto::FfiRequest { + message: Some(proto::ffi_request::Message::CaptureVideoFrame( + proto::CaptureVideoFrameRequest { + source_handle: Some(proto::FfiHandleId { + id: source_handle.0 as u64, + }), + buffer_handle: Some(proto::FfiHandleId { + id: buffer_handle.0 as u64, + }), + frame: Some(proto::VideoFrameInfo { + timestamp: 0, // TODO + rotation: proto::VideoRotation::VideoRotation0 as i32, + }), + }, + )), + }); + + tokio::time::sleep(std::time::Duration::from_millis(1000 / VIDEO_FPS as u64)).await; + } + }) +} diff --git a/livekit-ffi/src/server/utils.rs b/livekit-ffi/src/server/utils.rs new file mode 100644 index 0000000..cb1a484 --- /dev/null +++ b/livekit-ffi/src/server/utils.rs @@ -0,0 +1,32 @@ +use crate::{server, FfiError, FfiHandleId, FfiResult}; +use livekit::prelude::*; + +pub fn find_remote_track( + server: &'static server::FfiServer, + track_sid: &TrackSid, + participant_sid: &ParticipantSid, + room_handle: FfiHandleId, +) -> FfiResult { + let room = server + .ffi_handles() + .get(&room_handle) + .ok_or(FfiError::InvalidRequest("room not found"))?; + + let room = room + .downcast_ref::() + .ok_or(FfiError::InvalidRequest("room is not ffi room"))?; + + let session = room.session(); + let participants = session.participants(); + let participant = participants + .get(participant_sid) + .ok_or(FfiError::InvalidRequest("participant not found"))?; + + let track = participant + .get_track_publication(track_sid) + .ok_or(FfiError::InvalidRequest("publication not found"))? + .track() + .ok_or(FfiError::InvalidRequest("track not found/subscribed"))?; + + Ok(track) +} diff --git a/livekit-ffi/src/server/video_frame.rs b/livekit-ffi/src/server/video_frame.rs new file mode 100644 index 0000000..73c1691 --- /dev/null +++ b/livekit-ffi/src/server/video_frame.rs @@ -0,0 +1,231 @@ +use crate::{proto, server, FfiError, FfiHandleId, FfiResult}; +use futures_util::StreamExt; +use livekit::prelude::*; +use livekit::webrtc::media_stream::MediaStreamTrack; +use livekit::webrtc::video_frame::{BoxVideoFrameBuffer, VideoFrame}; +use livekit::webrtc::video_source::native::NativeVideoSource; +use livekit::webrtc::video_stream::native::NativeVideoStream; +use log::warn; +use server::utils; +use tokio::sync::oneshot; + +// ===== FFIVideoStream ===== + +pub struct FfiVideoStream { + handle_id: FfiHandleId, + stream_type: proto::VideoStreamType, + track_sid: TrackSid, + + #[allow(dead_code)] + close_tx: oneshot::Sender<()>, // Close the stream on drop +} + +impl FfiVideoStream { + /// Setup a new VideoStream and forward the frame data to the client/the foreign + /// language. + /// + /// When FFIVideoStream is dropped (When the corresponding handle_id is dropped), the task + /// is being closed. + /// + /// It is possible that the client receives a VideoFrame after the task is closed. The client + /// musts ignore it. + pub fn setup( + server: &'static server::FfiServer, + new_stream: proto::NewVideoStreamRequest, + ) -> FfiResult { + let (close_tx, close_rx) = oneshot::channel(); + let stream_type = proto::VideoStreamType::from_i32(new_stream.r#type).unwrap(); + let track_sid: TrackSid = new_stream.track_sid.into(); + + let room_handle = new_stream + .room_handle + .ok_or(FfiError::InvalidRequest("room_handle is empty"))? + .id as FfiHandleId; + + let track = utils::find_remote_track( + server, + &track_sid, + &new_stream.participant_sid.into(), + room_handle, + )? + .rtc_track(); + + let MediaStreamTrack::Video(track) = track else { + return Err(FfiError::InvalidRequest("not a video track")); + }; + + let stream = match stream_type { + proto::VideoStreamType::VideoStreamNative => { + let video_stream = Self { + handle_id: server.next_id(), + close_tx, + stream_type, + track_sid, + }; + tokio::spawn(Self::native_video_stream_task( + server, + video_stream.handle_id, + NativeVideoStream::new(track), + close_rx, + )); + Ok::(video_stream) + } + // TODO(theomonnom): Support other stream types + _ => return Err(FfiError::InvalidRequest("unsupported video stream type")), + }?; + + // Store the new video stream and return the info + let info = proto::VideoStreamInfo::from(&stream); + server + .ffi_handles() + .insert(stream.handle_id, Box::new(stream)); + + Ok(info) + } + + pub fn handle_id(&self) -> FfiHandleId { + self.handle_id + } + + pub fn stream_type(&self) -> proto::VideoStreamType { + self.stream_type + } + + pub fn track_sid(&self) -> &TrackSid { + &self.track_sid + } + + async fn native_video_stream_task( + server: &'static server::FfiServer, + stream_handle_id: FfiHandleId, + mut native_stream: NativeVideoStream, + mut close_rx: oneshot::Receiver<()>, + ) { + loop { + tokio::select! { + _ = &mut close_rx => { + break; + } + frame = native_stream.next() => { + let Some(frame) = frame else { + break; + }; + + let handle_id = server.next_id(); + let frame_info = proto::VideoFrameInfo::from(&frame); + let buffer_info = proto::VideoFrameBufferInfo::from(handle_id, &frame.buffer); + + server + .ffi_handles() + .insert(handle_id, Box::new(frame.buffer)); + + if let Err(err) = server.send_event(proto::ffi_event::Message::VideoStreamEvent( + proto::VideoStreamEvent { + handle: Some(stream_handle_id.into()), + message: Some(proto::video_stream_event::Message::FrameReceived( + proto::VideoFrameReceived { + frame: Some(frame_info), + buffer: Some(buffer_info), + } + )), + } + )) { + warn!("failed to send video frame: {}", err); + } + } + } + } + } +} + +// ===== FFIVideoSource ===== + +pub struct FfiVideoSource { + handle_id: FfiHandleId, + source_type: proto::VideoSourceType, + source: VideoSource, +} + +#[derive(Clone)] +pub enum VideoSource { + Native(NativeVideoSource), +} + +impl FfiVideoSource { + pub fn setup( + server: &'static server::FfiServer, + new_source: proto::NewVideoSourceRequest, + ) -> FfiResult { + let source_type = proto::VideoSourceType::from_i32(new_source.r#type).unwrap(); + let source_inner = match source_type { + proto::VideoSourceType::VideoSourceNative => { + let video_source = NativeVideoSource::default(); + Ok(VideoSource::Native(video_source)) + } + _ => Err(FfiError::InvalidRequest("unsupported video source type")), + }?; + + let video_source = Self { + handle_id: server.next_id(), + source_type, + source: source_inner, + }; + let source_info = proto::VideoSourceInfo::from(&video_source); + + server + .ffi_handles() + .insert(video_source.handle_id, Box::new(video_source)); + + Ok(source_info) + } + + pub fn capture_frame( + &self, + server: &'static server::FfiServer, + capture: proto::CaptureVideoFrameRequest, + ) -> FfiResult<()> { + match self.source { + VideoSource::Native(ref source) => { + let frame_info = capture + .frame + .ok_or(FfiError::InvalidRequest("frame is empty"))?; + + let buffer_handle = capture + .buffer_handle + .ok_or(FfiError::InvalidRequest("buffer_handle is none"))? + .id as FfiHandleId; + + let buffer = server + .ffi_handles() + .get(&buffer_handle) + .ok_or(FfiError::InvalidRequest("handle not found"))?; + + let buffer = buffer + .downcast_ref::() + .ok_or(FfiError::InvalidRequest("handle is not video frame"))?; + + let rotation = proto::VideoRotation::from_i32(frame_info.rotation).unwrap(); + let frame = VideoFrame { + rotation: rotation.into(), + timestamp: frame_info.timestamp, + buffer, + }; + + source.capture_frame(&frame); + } + } + Ok(()) + } + + pub fn handle_id(&self) -> FfiHandleId { + self.handle_id + } + + pub fn source_type(&self) -> proto::VideoSourceType { + self.source_type + } + + pub fn inner_source(&self) -> &VideoSource { + &self.source + } +} diff --git a/livekit-webrtc/src/audio_frame.rs b/livekit-webrtc/src/audio_frame.rs index 1a86f21..2674c91 100644 --- a/livekit-webrtc/src/audio_frame.rs +++ b/livekit-webrtc/src/audio_frame.rs @@ -1,7 +1,18 @@ #[derive(Debug, Clone)] pub struct AudioFrame { pub data: Vec, - pub sample_rate_hz: u32, - pub num_channels: usize, - pub samples_per_channel: usize, + pub sample_rate: u32, + pub num_channels: u32, + pub samples_per_channel: u32, +} + +impl AudioFrame { + pub fn new(sample_rate: u32, num_channels: u32, samples_per_channel: u32) -> Self { + Self { + data: vec![0; (num_channels * samples_per_channel) as usize], + sample_rate, + num_channels, + samples_per_channel, + } + } } diff --git a/livekit-webrtc/src/audio_source.rs b/livekit-webrtc/src/audio_source.rs index a1ecec3..b13adcd 100644 --- a/livekit-webrtc/src/audio_source.rs +++ b/livekit-webrtc/src/audio_source.rs @@ -18,7 +18,7 @@ pub mod native { } impl NativeAudioSource { - pub fn capture_frame(&self, frame: AudioFrame) { + pub fn capture_frame(&self, frame: &AudioFrame) { self.handle.capture_frame(frame) } } diff --git a/livekit-webrtc/src/native/audio_source.rs b/livekit-webrtc/src/native/audio_source.rs index 37af08d..195078f 100644 --- a/livekit-webrtc/src/native/audio_source.rs +++ b/livekit-webrtc/src/native/audio_source.rs @@ -20,14 +20,14 @@ impl NativeAudioSource { self.sys_handle.clone() } - pub fn capture_frame(&self, frame: AudioFrame) { + pub fn capture_frame(&self, frame: &AudioFrame) { // TODO(theomonnom): Should we check for 10ms worth of data here? unsafe { self.sys_handle.on_captured_frame( frame.data.as_ptr(), - frame.sample_rate_hz as i32, - frame.num_channels, - frame.samples_per_channel, + frame.sample_rate as i32, + frame.num_channels as usize, + frame.samples_per_channel as usize, ) } } diff --git a/livekit-webrtc/src/native/audio_stream.rs b/livekit-webrtc/src/native/audio_stream.rs index 03503f2..4f21729 100644 --- a/livekit-webrtc/src/native/audio_stream.rs +++ b/livekit-webrtc/src/native/audio_stream.rs @@ -72,9 +72,9 @@ impl sys_ms::AudioSink for AudioTrackObserver { // TODO(theomonnom): Should we avoid copy here? let _ = self.frame_tx.send(AudioFrame { data: data.to_owned(), - sample_rate_hz: sample_rate as u32, - num_channels: nb_channels, - samples_per_channel: nb_frames, + sample_rate: sample_rate as u32, + num_channels: nb_channels as u32, + samples_per_channel: nb_frames as u32, }); } } diff --git a/livekit-webrtc/src/native/video_frame.rs b/livekit-webrtc/src/native/video_frame.rs index c4bbb92..1a7e335 100644 --- a/livekit-webrtc/src/native/video_frame.rs +++ b/livekit-webrtc/src/native/video_frame.rs @@ -149,11 +149,11 @@ impl NativeBuffer { &*self.sys_handle } - pub fn width(&self) -> i32 { + pub fn width(&self) -> u32 { self.sys_handle.width() } - pub fn height(&self) -> i32 { + pub fn height(&self) -> u32 { self.sys_handle.height() } @@ -167,7 +167,7 @@ impl NativeBuffer { &self, format: VideoFormatType, dst: &mut [u8], - dst_stride: i32, + dst_stride: u32, dst_width: i32, dst_height: i32, ) -> Result<(), ConvertError> { @@ -192,49 +192,49 @@ impl I420Buffer { unsafe { &*recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb) } } - pub fn width(&self) -> i32 { + pub fn width(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb); (*ptr).width() } } - pub fn height(&self) -> i32 { + pub fn height(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb); (*ptr).height() } } - pub fn chroma_width(&self) -> i32 { + pub fn chroma_width(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv); (*ptr).chroma_width() } } - pub fn chroma_height(&self) -> i32 { + pub fn chroma_height(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv); (*ptr).chroma_height() } } - pub fn stride_y(&self) -> i32 { + pub fn stride_y(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv); (*ptr).stride_y() } } - pub fn stride_u(&self) -> i32 { + pub fn stride_u(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv); (*ptr).stride_u() } } - pub fn stride_v(&self) -> i32 { + pub fn stride_v(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv); (*ptr).stride_v() @@ -258,7 +258,7 @@ impl I420Buffer { &self, format: VideoFormatType, dst: &mut [u8], - dst_stride: i32, + dst_stride: u32, dst_width: i32, dst_height: i32, ) -> Result<(), ConvertError> { @@ -292,56 +292,56 @@ impl I420ABuffer { unsafe { &*recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb) } } - pub fn width(&self) -> i32 { + pub fn width(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb); (*ptr).width() } } - pub fn height(&self) -> i32 { + pub fn height(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb); (*ptr).height() } } - pub fn chroma_width(&self) -> i32 { + pub fn chroma_width(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv); (*ptr).chroma_width() } } - pub fn chroma_height(&self) -> i32 { + pub fn chroma_height(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv); (*ptr).chroma_height() } } - pub fn stride_y(&self) -> i32 { + pub fn stride_y(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv); (*ptr).stride_y() } } - pub fn stride_u(&self) -> i32 { + pub fn stride_u(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv); (*ptr).stride_u() } } - pub fn stride_v(&self) -> i32 { + pub fn stride_v(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv); (*ptr).stride_v() } } - pub fn stride_a(&self) -> i32 { + pub fn stride_a(&self) -> u32 { self.sys_handle.stride_a() } @@ -359,7 +359,7 @@ impl I420ABuffer { &self, format: VideoFormatType, dst: &mut [u8], - dst_stride: i32, + dst_stride: u32, dst_width: i32, dst_height: i32, ) -> Result<(), ConvertError> { @@ -391,49 +391,49 @@ impl I422Buffer { unsafe { &*recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb) } } - pub fn width(&self) -> i32 { + pub fn width(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb); (*ptr).width() } } - pub fn height(&self) -> i32 { + pub fn height(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb); (*ptr).height() } } - pub fn chroma_width(&self) -> i32 { + pub fn chroma_width(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv); (*ptr).chroma_width() } } - pub fn chroma_height(&self) -> i32 { + pub fn chroma_height(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv); (*ptr).chroma_height() } } - pub fn stride_y(&self) -> i32 { + pub fn stride_y(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv); (*ptr).stride_y() } } - pub fn stride_u(&self) -> i32 { + pub fn stride_u(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv); (*ptr).stride_u() } } - pub fn stride_v(&self) -> i32 { + pub fn stride_v(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv); (*ptr).stride_v() @@ -453,7 +453,7 @@ impl I422Buffer { &self, format: VideoFormatType, dst: &mut [u8], - dst_stride: i32, + dst_stride: u32, dst_width: i32, dst_height: i32, ) -> Result<(), ConvertError> { @@ -477,49 +477,49 @@ impl I444Buffer { unsafe { &*recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb) } } - pub fn width(&self) -> i32 { + pub fn width(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb); (*ptr).width() } } - pub fn height(&self) -> i32 { + pub fn height(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb); (*ptr).height() } } - pub fn chroma_width(&self) -> i32 { + pub fn chroma_width(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv); (*ptr).chroma_width() } } - pub fn chroma_height(&self) -> i32 { + pub fn chroma_height(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv); (*ptr).chroma_height() } } - pub fn stride_y(&self) -> i32 { + pub fn stride_y(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv); (*ptr).stride_y() } } - pub fn stride_u(&self) -> i32 { + pub fn stride_u(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv); (*ptr).stride_u() } } - pub fn stride_v(&self) -> i32 { + pub fn stride_v(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv); (*ptr).stride_v() @@ -539,7 +539,7 @@ impl I444Buffer { &self, format: VideoFormatType, dst: &mut [u8], - dst_stride: i32, + dst_stride: u32, dst_width: i32, dst_height: i32, ) -> Result<(), ConvertError> { @@ -564,49 +564,49 @@ impl I010Buffer { unsafe { &*recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb) } } - pub fn width(&self) -> i32 { + pub fn width(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb); (*ptr).width() } } - pub fn height(&self) -> i32 { + pub fn height(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb); (*ptr).height() } } - pub fn chroma_width(&self) -> i32 { + pub fn chroma_width(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv); (*ptr).chroma_width() } } - pub fn chroma_height(&self) -> i32 { + pub fn chroma_height(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv); (*ptr).chroma_height() } } - pub fn stride_y(&self) -> i32 { + pub fn stride_y(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv); (*ptr).stride_y() } } - pub fn stride_u(&self) -> i32 { + pub fn stride_u(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv); (*ptr).stride_u() } } - pub fn stride_v(&self) -> i32 { + pub fn stride_v(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv); (*ptr).stride_v() @@ -627,7 +627,7 @@ impl I010Buffer { &self, format: VideoFormatType, dst: &mut [u8], - dst_stride: i32, + dst_stride: u32, dst_width: i32, dst_height: i32, ) -> Result<(), ConvertError> { @@ -669,7 +669,7 @@ impl NV12Buffer { } } - pub fn width(&self) -> i32 { + pub fn width(&self) -> u32 { unsafe { let ptr = recursive_cast!( &*self.sys_handle, @@ -681,7 +681,7 @@ impl NV12Buffer { } } - pub fn height(&self) -> i32 { + pub fn height(&self) -> u32 { unsafe { let ptr = recursive_cast!( &*self.sys_handle, @@ -693,28 +693,28 @@ impl NV12Buffer { } } - pub fn chroma_width(&self) -> i32 { + pub fn chroma_width(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv); (*ptr).chroma_width() } } - pub fn chroma_height(&self) -> i32 { + pub fn chroma_height(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv); (*ptr).chroma_height() } } - pub fn stride_y(&self) -> i32 { + pub fn stride_y(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv); (*ptr).stride_y() } } - pub fn stride_uv(&self) -> i32 { + pub fn stride_uv(&self) -> u32 { unsafe { let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv); (*ptr).stride_uv() @@ -739,7 +739,7 @@ impl NV12Buffer { &self, format: VideoFormatType, dst: &mut [u8], - dst_stride: i32, + dst_stride: u32, dst_width: i32, dst_height: i32, ) -> Result<(), ConvertError> { diff --git a/livekit-webrtc/src/native/video_source.rs b/livekit-webrtc/src/native/video_source.rs index a6c2d86..88db0b1 100644 --- a/livekit-webrtc/src/native/video_source.rs +++ b/livekit-webrtc/src/native/video_source.rs @@ -21,12 +21,12 @@ impl NativeVideoSource { self.sys_handle.clone() } - pub fn capture_frame(&self, frame: &VideoFrame) { + pub fn capture_frame>(&self, frame: &VideoFrame) { let mut builder = vf_sys::ffi::new_video_frame_builder(); builder.pin_mut().set_rotation(frame.rotation.into()); builder .pin_mut() - .set_video_frame_buffer(frame.buffer.sys_handle()); + .set_video_frame_buffer(frame.buffer.as_ref().sys_handle()); let frame = builder.pin_mut().build(); self.sys_handle.on_captured_frame(&frame); diff --git a/livekit-webrtc/src/native/yuv_helper.rs b/livekit-webrtc/src/native/yuv_helper.rs index df9f556..0a60696 100644 --- a/livekit-webrtc/src/native/yuv_helper.rs +++ b/livekit-webrtc/src/native/yuv_helper.rs @@ -10,11 +10,12 @@ pub enum ConvertError { #[inline] fn argb_assert_safety( src: &[u8], - src_stride: i32, + src_stride: u32, _width: i32, height: i32, ) -> Result<(), ConvertError> { - let min = (src_stride * height) as usize; + let height_abs = height.abs() as u32; + let min = (src_stride * height_abs) as usize; if src.len() < min { return Err(ConvertError::Convert("dst isn't large enough")); @@ -26,16 +27,17 @@ fn argb_assert_safety( #[inline] fn i420_assert_safety( src_y: &[u8], - src_stride_y: i32, + src_stride_y: u32, src_u: &[u8], - src_stride_u: i32, + src_stride_u: u32, src_v: &[u8], - src_stride_v: i32, + src_stride_v: u32, _width: i32, height: i32, ) -> Result<(), ConvertError> { - let chroma_height = (height + 1) / 2; - let min_y = (src_stride_y * height) as usize; + let height_abs = height.abs() as u32; + let chroma_height = (height_abs + 1) / 2; + let min_y = (src_stride_y * height_abs) as usize; let min_u = (src_stride_u * chroma_height) as usize; let min_v = (src_stride_v * chroma_height) as usize; @@ -58,13 +60,13 @@ macro_rules! i420_to_x { ($x:ident) => { pub fn $x( src_y: &[u8], - src_stride_y: i32, + src_stride_y: u32, src_u: &[u8], - src_stride_u: i32, + src_stride_u: u32, src_v: &[u8], - src_stride_v: i32, + src_stride_v: u32, dst: &mut [u8], - dst_stride: i32, + dst_stride: u32, width: i32, height: i32, ) -> Result<(), ConvertError> { @@ -83,13 +85,13 @@ macro_rules! i420_to_x { unsafe { yuv_sys::ffi::$x( src_y.as_ptr(), - src_stride_y, + src_stride_y as i32, src_u.as_ptr(), - src_stride_u, + src_stride_u as i32, src_v.as_ptr(), - src_stride_v, + src_stride_v as i32, dst.as_mut_ptr(), - dst_stride, + dst_stride as i32, width, height, ) @@ -105,13 +107,13 @@ macro_rules! x_to_i420 { ($x:ident) => { pub fn $x( src_argb: &[u8], - src_stride_argb: i32, + src_stride_argb: u32, dst_y: &mut [u8], - dst_stride_y: i32, + dst_stride_y: u32, dst_u: &mut [u8], - dst_stride_u: i32, + dst_stride_u: u32, dst_v: &mut [u8], - dst_stride_v: i32, + dst_stride_v: u32, width: i32, height: i32, ) -> Result<(), ConvertError> { @@ -130,13 +132,13 @@ macro_rules! x_to_i420 { unsafe { yuv_sys::ffi::$x( src_argb.as_ptr(), - src_stride_argb, + src_stride_argb as i32, dst_y.as_mut_ptr(), - dst_stride_y, + dst_stride_y as i32, dst_u.as_mut_ptr(), - dst_stride_u, + dst_stride_u as i32, dst_v.as_mut_ptr(), - dst_stride_v, + dst_stride_v as i32, width, height, ) @@ -150,9 +152,9 @@ macro_rules! x_to_i420 { pub fn argb_to_rgb24( src_argb: &[u8], - src_stride_argb: i32, + src_stride_argb: u32, dst_rgb24: &mut [u8], - dst_stride_rgb24: i32, + dst_stride_rgb24: u32, width: i32, height: i32, ) -> Result<(), ConvertError> { @@ -162,9 +164,9 @@ pub fn argb_to_rgb24( unsafe { yuv_sys::ffi::argb_to_rgb24( src_argb.as_ptr(), - src_stride_argb, + src_stride_argb as i32, dst_rgb24.as_mut_ptr(), - dst_stride_rgb24, + dst_stride_rgb24 as i32, width, height, ) diff --git a/livekit-webrtc/src/video_frame.rs b/livekit-webrtc/src/video_frame.rs index 8ef3a37..389b7a9 100644 --- a/livekit-webrtc/src/video_frame.rs +++ b/livekit-webrtc/src/video_frame.rs @@ -40,80 +40,20 @@ pub enum VideoFrameBufferType { #[derive(Debug)] pub struct VideoFrame where - T: VideoFrameBuffer, + T: AsRef, { pub rotation: VideoRotation, pub timestamp: i64, // When the frame was captured pub buffer: T, } -pub type BoxVideoFrame = VideoFrame>; - -macro_rules! new_buffer_type { - ($type:ident, $variant:ident, $as:ident) => { - pub struct $type { - pub(crate) handle: vf_imp::$type, - } - - impl $crate::video_frame::internal::BufferInternal for $type { - #[cfg(not(target_arch = "wasm32"))] - fn sys_handle(&self) -> &webrtc_sys::video_frame_buffer::ffi::VideoFrameBuffer { - self.handle.sys_handle() - } - - #[cfg(not(target_arch = "wasm32"))] - fn to_i420(&self) -> I420Buffer { - I420Buffer { - handle: self.handle.to_i420(), - } - } - - #[cfg(not(target_arch = "wasm32"))] - fn to_argb( - &self, - format: VideoFormatType, - dst: &mut [u8], - stride: i32, - width: i32, - height: i32, - ) -> Result<(), $crate::video_frame::native::ConvertError> { - self.handle.to_argb(format, dst, stride, width, height) - } - } - - impl VideoFrameBuffer for $type { - fn width(&self) -> i32 { - self.handle.width() - } - - fn height(&self) -> i32 { - self.handle.height() - } - - fn buffer_type(&self) -> VideoFrameBufferType { - VideoFrameBufferType::$variant - } - - fn $as(&self) -> Option<&$type> { - Some(self) - } - } - - impl Debug for $type { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct(stringify!($type)) - .field("width", &self.width()) - .field("height", &self.height()) - .finish() - } - } - }; -} +pub type BoxVideoFrameBuffer = Box; +pub type BoxVideoFrame = VideoFrame; pub(crate) mod internal { use super::{I420Buffer, VideoFormatType}; - pub trait BufferInternal { + pub trait BufferSealed: Send + Sync { #[cfg(not(target_arch = "wasm32"))] fn sys_handle(&self) -> &webrtc_sys::video_frame_buffer::ffi::VideoFrameBuffer; @@ -125,16 +65,16 @@ pub(crate) mod internal { &self, format: VideoFormatType, dst: &mut [u8], - dst_stride: i32, + dst_stride: u32, dst_width: i32, dst_height: i32, ) -> Result<(), super::native::ConvertError>; } } -pub trait VideoFrameBuffer: internal::BufferInternal + Debug { - fn width(&self) -> i32; - fn height(&self) -> i32; +pub trait VideoFrameBuffer: internal::BufferSealed + Debug { + fn width(&self) -> u32; + fn height(&self) -> u32; fn buffer_type(&self) -> VideoFrameBufferType; #[cfg(not(target_arch = "wasm32"))] @@ -167,6 +107,73 @@ pub trait VideoFrameBuffer: internal::BufferInternal + Debug { } } +macro_rules! new_buffer_type { + ($type:ident, $variant:ident, $as:ident) => { + pub struct $type { + pub(crate) handle: vf_imp::$type, + } + + impl $crate::video_frame::internal::BufferSealed for $type { + #[cfg(not(target_arch = "wasm32"))] + fn sys_handle(&self) -> &webrtc_sys::video_frame_buffer::ffi::VideoFrameBuffer { + self.handle.sys_handle() + } + + #[cfg(not(target_arch = "wasm32"))] + fn to_i420(&self) -> I420Buffer { + I420Buffer { + handle: self.handle.to_i420(), + } + } + + #[cfg(not(target_arch = "wasm32"))] + fn to_argb( + &self, + format: VideoFormatType, + dst: &mut [u8], + stride: u32, + width: i32, + height: i32, + ) -> Result<(), $crate::video_frame::native::ConvertError> { + self.handle.to_argb(format, dst, stride, width, height) + } + } + + impl VideoFrameBuffer for $type { + fn width(&self) -> u32 { + self.handle.width() + } + + fn height(&self) -> u32 { + self.handle.height() + } + + fn buffer_type(&self) -> VideoFrameBufferType { + VideoFrameBufferType::$variant + } + + fn $as(&self) -> Option<&$type> { + Some(self) + } + } + + impl Debug for $type { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct(stringify!($type)) + .field("width", &self.width()) + .field("height", &self.height()) + .finish() + } + } + + impl AsRef for $type { + fn as_ref(&self) -> &(dyn VideoFrameBuffer + 'static) { + self + } + } + }; +} + new_buffer_type!(I420Buffer, I420, as_i420); new_buffer_type!(I420ABuffer, I420A, as_i420a); new_buffer_type!(I422Buffer, I422, as_i422); @@ -175,24 +182,20 @@ new_buffer_type!(I010Buffer, I010, as_i010); new_buffer_type!(NV12Buffer, NV12, as_nv12); impl I420Buffer { - pub fn chroma_width(&self) -> i32 { + pub fn chroma_width(&self) -> u32 { self.handle.chroma_width() } - pub fn chroma_height(&self) -> i32 { + pub fn chroma_height(&self) -> u32 { self.handle.chroma_height() } - pub fn stride_y(&self) -> i32 { - self.handle.stride_y() - } - - pub fn stride_u(&self) -> i32 { - self.handle.stride_u() - } - - pub fn stride_v(&self) -> i32 { - self.handle.stride_v() + pub fn strides(&self) -> (u32, u32, u32) { + ( + self.handle.stride_y(), + self.handle.stride_u(), + self.handle.stride_v(), + ) } pub fn data(&self) -> (&[u8], &[u8], &[u8]) { @@ -212,28 +215,21 @@ impl I420Buffer { } impl I420ABuffer { - pub fn chroma_width(&self) -> i32 { + pub fn chroma_width(&self) -> u32 { self.handle.chroma_width() } - pub fn chroma_height(&self) -> i32 { + pub fn chroma_height(&self) -> u32 { self.handle.chroma_height() } - pub fn stride_y(&self) -> i32 { - self.handle.stride_y() - } - - pub fn stride_u(&self) -> i32 { - self.handle.stride_u() - } - - pub fn stride_v(&self) -> i32 { - self.handle.stride_v() - } - - pub fn stride_a(&self) -> i32 { - self.handle.stride_a() + pub fn strides(&self) -> (u32, u32, u32, u32) { + ( + self.handle.stride_y(), + self.handle.stride_u(), + self.handle.stride_v(), + self.handle.stride_a(), + ) } pub fn data(&self) -> (&[u8], &[u8], &[u8], Option<&[u8]>) { @@ -256,24 +252,20 @@ impl I420ABuffer { } impl I422Buffer { - pub fn chroma_width(&self) -> i32 { + pub fn chroma_width(&self) -> u32 { self.handle.chroma_width() } - pub fn chroma_height(&self) -> i32 { + pub fn chroma_height(&self) -> u32 { self.handle.chroma_height() } - pub fn stride_y(&self) -> i32 { - self.handle.stride_y() - } - - pub fn stride_u(&self) -> i32 { - self.handle.stride_u() - } - - pub fn stride_v(&self) -> i32 { - self.handle.stride_v() + pub fn strides(&self) -> (u32, u32, u32) { + ( + self.handle.stride_y(), + self.handle.stride_u(), + self.handle.stride_v(), + ) } pub fn data(&self) -> (&[u8], &[u8], &[u8]) { @@ -293,24 +285,20 @@ impl I422Buffer { } impl I444Buffer { - pub fn chroma_width(&self) -> i32 { + pub fn chroma_width(&self) -> u32 { self.handle.chroma_width() } - pub fn chroma_height(&self) -> i32 { + pub fn chroma_height(&self) -> u32 { self.handle.chroma_height() } - pub fn stride_y(&self) -> i32 { - self.handle.stride_y() - } - - pub fn stride_u(&self) -> i32 { - self.handle.stride_u() - } - - pub fn stride_v(&self) -> i32 { - self.handle.stride_v() + pub fn strides(&self) -> (u32, u32, u32) { + ( + self.handle.stride_y(), + self.handle.stride_u(), + self.handle.stride_v(), + ) } pub fn data(&self) -> (&[u8], &[u8], &[u8]) { @@ -330,24 +318,20 @@ impl I444Buffer { } impl I010Buffer { - pub fn chroma_width(&self) -> i32 { + pub fn chroma_width(&self) -> u32 { self.handle.chroma_width() } - pub fn chroma_height(&self) -> i32 { + pub fn chroma_height(&self) -> u32 { self.handle.chroma_height() } - pub fn stride_y(&self) -> i32 { - self.handle.stride_y() - } - - pub fn stride_u(&self) -> i32 { - self.handle.stride_u() - } - - pub fn stride_v(&self) -> i32 { - self.handle.stride_v() + pub fn strides(&self) -> (u32, u32, u32) { + ( + self.handle.stride_y(), + self.handle.stride_u(), + self.handle.stride_v(), + ) } pub fn data(&self) -> (&[u16], &[u16], &[u16]) { @@ -367,20 +351,16 @@ impl I010Buffer { } impl NV12Buffer { - pub fn chroma_width(&self) -> i32 { + pub fn chroma_width(&self) -> u32 { self.handle.chroma_width() } - pub fn chroma_height(&self) -> i32 { + pub fn chroma_height(&self) -> u32 { self.handle.chroma_height() } - pub fn stride_y(&self) -> i32 { - self.handle.stride_y() - } - - pub fn stride_uv(&self) -> i32 { - self.handle.stride_uv() + pub fn strides(&self) -> (u32, u32) { + (self.handle.stride_y(), self.handle.stride_uv()) } pub fn data(&self) -> (&[u8], &[u8]) { @@ -423,7 +403,7 @@ pub mod native { &self, format: VideoFormatType, dst: &mut [u8], - dst_stride: i32, + dst_stride: u32, dst_width: i32, dst_height: i32, ) -> Result<(), ConvertError>; @@ -438,7 +418,7 @@ pub mod native { &self, format: VideoFormatType, dst: &mut [u8], - dst_stride: i32, + dst_stride: u32, dst_width: i32, dst_height: i32, ) -> Result<(), ConvertError> { @@ -447,42 +427,6 @@ pub mod native { } } -impl internal::BufferInternal for Box { - fn sys_handle(&self) -> &webrtc_sys::video_frame_buffer::ffi::VideoFrameBuffer { - self.as_ref().sys_handle() - } - - fn to_i420(&self) -> I420Buffer { - self.as_ref().to_i420() - } - - fn to_argb( - &self, - format: VideoFormatType, - dst: &mut [u8], - dst_stride: i32, - dst_width: i32, - dst_height: i32, - ) -> Result<(), self::native::ConvertError> { - self.as_ref() - .to_argb(format, dst, dst_stride, dst_width, dst_height) - } -} - -impl VideoFrameBuffer for Box { - fn width(&self) -> i32 { - self.as_ref().width() - } - - fn height(&self) -> i32 { - self.as_ref().height() - } - - fn buffer_type(&self) -> VideoFrameBufferType { - self.as_ref().buffer_type() - } -} - #[cfg(target_arch = "wasm32")] pub mod web { use super::VideoFrameBuffer; diff --git a/livekit-webrtc/src/video_source.rs b/livekit-webrtc/src/video_source.rs index d6b3b6f..9ba3f90 100644 --- a/livekit-webrtc/src/video_source.rs +++ b/livekit-webrtc/src/video_source.rs @@ -18,7 +18,7 @@ pub mod native { } impl NativeVideoSource { - pub fn capture_frame(&self, frame: &VideoFrame) { + pub fn capture_frame>(&self, frame: &VideoFrame) { self.handle.capture_frame(frame) } } diff --git a/livekit/src/room/participant/local_participant.rs b/livekit/src/room/participant/local_participant.rs index 2999b2f..bd3085b 100644 --- a/livekit/src/room/participant/local_participant.rs +++ b/livekit/src/room/participant/local_participant.rs @@ -42,7 +42,7 @@ impl LocalParticipant { cid: track.rtc_track().id(), name: options.name.clone(), r#type: proto::TrackType::from(track.kind()) as i32, - muted: track.muted(), + muted: track.is_muted(), source: proto::TrackSource::from(options.source) as i32, disable_dtx: !options.dtx, disable_red: !options.red, diff --git a/livekit/src/room/participant/remote_participant.rs b/livekit/src/room/participant/remote_participant.rs index 4779006..9d4c820 100644 --- a/livekit/src/room/participant/remote_participant.rs +++ b/livekit/src/room/participant/remote_participant.rs @@ -105,7 +105,7 @@ impl RemoteParticipant { debug!("starting track: {:?}", sid); remote_publication.update_track(Some(track.clone().into())); - track.set_muted(remote_publication.muted()); + track.set_muted(remote_publication.is_muted()); track.update_info(proto::TrackInfo { sid: remote_publication.sid().to_string(), name: remote_publication.name().to_string(), diff --git a/livekit/src/room/publication/local.rs b/livekit/src/room/publication/local.rs index 48d204f..07986a3 100644 --- a/livekit/src/room/publication/local.rs +++ b/livekit/src/room/publication/local.rs @@ -75,8 +75,13 @@ impl LocalTrackPublication { } #[inline] - pub fn muted(&self) -> bool { - self.inner.publication_inner.muted() + pub fn is_muted(&self) -> bool { + self.inner.publication_inner.is_muted() + } + + #[inline] + pub fn is_remote(&self) -> bool { + false } #[inline] diff --git a/livekit/src/room/publication/mod.rs b/livekit/src/room/publication/mod.rs index 8400bab..d586625 100644 --- a/livekit/src/room/publication/mod.rs +++ b/livekit/src/room/publication/mod.rs @@ -132,7 +132,7 @@ impl TrackPublicationInner { self.track.lock().clone() } - pub fn muted(&self) -> bool { + pub fn is_muted(&self) -> bool { self.muted.load(Ordering::Relaxed) } } @@ -153,7 +153,8 @@ impl TrackPublication { pub fn simulcasted(self: &Self) -> bool; pub fn dimension(self: &Self) -> TrackDimension; pub fn mime_type(self: &Self) -> String; - pub fn muted(self: &Self) -> bool; + pub fn is_muted(self: &Self) -> bool; + pub fn is_remote(self: &Self) -> bool; ); pub fn track(&self) -> Option { diff --git a/livekit/src/room/publication/remote.rs b/livekit/src/room/publication/remote.rs index f699a59..5f43fcb 100644 --- a/livekit/src/room/publication/remote.rs +++ b/livekit/src/room/publication/remote.rs @@ -57,8 +57,13 @@ impl RemoteTrackPublication { } #[inline] - pub fn muted(&self) -> bool { - self.inner.muted() + pub fn is_muted(&self) -> bool { + self.inner.is_muted() + } + + #[inline] + pub fn is_remote(&self) -> bool { + true } #[inline] diff --git a/livekit/src/room/track/local_audio_track.rs b/livekit/src/room/track/local_audio_track.rs index 2a4aa14..f1575dc 100644 --- a/livekit/src/room/track/local_audio_track.rs +++ b/livekit/src/room/track/local_audio_track.rs @@ -81,8 +81,8 @@ impl LocalAudioTrack { } #[inline] - pub fn muted(&self) -> bool { - self.inner.track_inner.muted() + pub fn is_muted(&self) -> bool { + self.inner.track_inner.is_muted() } #[inline] @@ -106,6 +106,11 @@ impl LocalAudioTrack { self.inner.track_inner.register_observer() } + #[inline] + pub fn is_remote(&self) -> bool { + false + } + #[inline] pub(crate) fn transceiver(&self) -> Option { self.inner.track_inner.transceiver() diff --git a/livekit/src/room/track/local_video_track.rs b/livekit/src/room/track/local_video_track.rs index 349697d..23d3a45 100644 --- a/livekit/src/room/track/local_video_track.rs +++ b/livekit/src/room/track/local_video_track.rs @@ -80,8 +80,8 @@ impl LocalVideoTrack { } #[inline] - pub fn muted(&self) -> bool { - self.inner.track_inner.muted() + pub fn is_muted(&self) -> bool { + self.inner.track_inner.is_muted() } #[inline] @@ -106,7 +106,12 @@ impl LocalVideoTrack { } #[inline] - pub fn transceiver(&self) -> Option { + pub fn is_remote(&self) -> bool { + false + } + + #[inline] + pub(crate) fn transceiver(&self) -> Option { self.inner.track_inner.transceiver() } diff --git a/livekit/src/room/track/mod.rs b/livekit/src/room/track/mod.rs index f305692..70098dc 100644 --- a/livekit/src/room/track/mod.rs +++ b/livekit/src/room/track/mod.rs @@ -98,9 +98,10 @@ macro_rules! track_dispatch { pub fn stream_state(self: &Self) -> StreamState; pub fn start(self: &Self) -> (); pub fn stop(self: &Self) -> (); - pub fn muted(self: &Self) -> bool; + pub fn is_muted(self: &Self) -> bool; pub fn set_muted(self: &Self, muted: bool) -> (); pub fn register_observer(self: &Self) -> mpsc::UnboundedReceiver; + pub fn is_remote(self: &Self) -> bool; pub(crate) fn transceiver(self: &Self) -> Option; pub(crate) fn update_transceiver(self: &Self, transceiver: Option) -> (); @@ -224,7 +225,7 @@ impl TrackInner { self.stream_state.load(Ordering::SeqCst).try_into().unwrap() } - pub fn muted(&self) -> bool { + pub fn is_muted(&self) -> bool { self.muted.load(Ordering::SeqCst) } diff --git a/livekit/src/room/track/remote_audio_track.rs b/livekit/src/room/track/remote_audio_track.rs index b1d5715..8fd5042 100644 --- a/livekit/src/room/track/remote_audio_track.rs +++ b/livekit/src/room/track/remote_audio_track.rs @@ -62,8 +62,8 @@ impl RemoteAudioTrack { } #[inline] - pub fn muted(&self) -> bool { - self.inner.muted() + pub fn is_muted(&self) -> bool { + self.inner.is_muted() } #[inline] @@ -85,6 +85,11 @@ impl RemoteAudioTrack { self.inner.register_observer() } + #[inline] + pub fn is_remote(&self) -> bool { + true + } + #[inline] pub(crate) fn transceiver(&self) -> Option { self.inner.transceiver() diff --git a/livekit/src/room/track/remote_video_track.rs b/livekit/src/room/track/remote_video_track.rs index f9369b1..f7ed25b 100644 --- a/livekit/src/room/track/remote_video_track.rs +++ b/livekit/src/room/track/remote_video_track.rs @@ -62,8 +62,8 @@ impl RemoteVideoTrack { } #[inline] - pub fn muted(&self) -> bool { - self.inner.muted() + pub fn is_muted(&self) -> bool { + self.inner.is_muted() } #[inline] @@ -85,6 +85,11 @@ impl RemoteVideoTrack { self.inner.register_observer() } + #[inline] + pub fn is_remote(&self) -> bool { + true + } + #[inline] pub(crate) fn transceiver(&self) -> Option { self.inner.transceiver() diff --git a/webrtc-sys/build.rs b/webrtc-sys/build.rs index 90ce6fc..36bfd36 100644 --- a/webrtc-sys/build.rs +++ b/webrtc-sys/build.rs @@ -70,7 +70,6 @@ fn download_prebuilt_webrtc( for i in 0..archive.len() { let mut inner_file = archive.by_index(i)?; let relative_path = inner_file.mangled_name(); - if relative_path.to_string_lossy().is_empty() { continue; // Ignore root } @@ -133,7 +132,7 @@ fn main() { webrtc_include.join("sdk/objc/base"), ]; - let mut builder = cxx_build::bridges(&[ + let mut builder = cxx_build::bridges([ "src/peer_connection.rs", "src/peer_connection_factory.rs", "src/media_stream.rs", @@ -248,7 +247,7 @@ fn main() { println!("cargo:rustc-link-arg=-ObjC"); let sysroot = Command::new("xcrun") - .args(&["--sdk", "macosx", "--show-sdk-path"]) + .args(["--sdk", "macosx", "--show-sdk-path"]) .output() .unwrap(); diff --git a/webrtc-sys/include/livekit/video_frame.h b/webrtc-sys/include/livekit/video_frame.h index 1acfb5b..d831792 100644 --- a/webrtc-sys/include/livekit/video_frame.h +++ b/webrtc-sys/include/livekit/video_frame.h @@ -32,8 +32,8 @@ class VideoFrame { public: explicit VideoFrame(const webrtc::VideoFrame& frame); - int width() const; - int height() const; + unsigned int width() const; + unsigned int height() const; uint32_t size() const; uint16_t id() const; int64_t timestamp_us() const; diff --git a/webrtc-sys/include/livekit/video_frame_buffer.h b/webrtc-sys/include/livekit/video_frame_buffer.h index ca246d1..810898f 100644 --- a/webrtc-sys/include/livekit/video_frame_buffer.h +++ b/webrtc-sys/include/livekit/video_frame_buffer.h @@ -46,8 +46,8 @@ class VideoFrameBuffer { VideoFrameBufferType buffer_type() const; - int width() const; - int height() const; + unsigned int width() const; + unsigned int height() const; std::unique_ptr to_i420() const; @@ -68,12 +68,12 @@ class PlanarYuvBuffer : public VideoFrameBuffer { public: explicit PlanarYuvBuffer(rtc::scoped_refptr buffer); - int chroma_width() const; - int chroma_height() const; + unsigned int chroma_width() const; + unsigned int chroma_height() const; - int stride_y() const; - int stride_u() const; - int stride_v() const; + unsigned int stride_y() const; + unsigned int stride_u() const; + unsigned int stride_v() const; private: webrtc::PlanarYuvBuffer* buffer() const; @@ -110,11 +110,11 @@ class BiplanarYuvBuffer : public VideoFrameBuffer { explicit BiplanarYuvBuffer( rtc::scoped_refptr buffer); - int chroma_width() const; - int chroma_height() const; + unsigned int chroma_width() const; + unsigned int chroma_height() const; - int stride_y() const; - int stride_uv() const; + unsigned int stride_y() const; + unsigned int stride_uv() const; private: webrtc::BiplanarYuvBuffer* buffer() const; @@ -145,7 +145,7 @@ class I420ABuffer : public I420Buffer { public: explicit I420ABuffer(rtc::scoped_refptr buffer); - int stride_a() const; + unsigned int stride_a() const; const uint8_t* data_a() const; private: diff --git a/webrtc-sys/src/peer_connection_factory.rs b/webrtc-sys/src/peer_connection_factory.rs index 3561fde..79398f9 100644 --- a/webrtc-sys/src/peer_connection_factory.rs +++ b/webrtc-sys/src/peer_connection_factory.rs @@ -61,7 +61,7 @@ pub mod ffi { fn create_rtc_configuration(conf: RTCConfiguration) -> UniquePtr; /// # Safety - /// The observer must live as long as the PeerConnection + /// The observer must live as long as the PeerConnection does unsafe fn create_peer_connection( self: &PeerConnectionFactory, config: UniquePtr, diff --git a/webrtc-sys/src/video_frame.cpp b/webrtc-sys/src/video_frame.cpp index e80b254..fe8cbbf 100644 --- a/webrtc-sys/src/video_frame.cpp +++ b/webrtc-sys/src/video_frame.cpp @@ -24,10 +24,10 @@ namespace livekit { VideoFrame::VideoFrame(const webrtc::VideoFrame& frame) : frame_(std::move(frame)) {} -int VideoFrame::width() const { +unsigned int VideoFrame::width() const { return frame_.width(); } -int VideoFrame::height() const { +unsigned int VideoFrame::height() const { return frame_.height(); } uint32_t VideoFrame::size() const { diff --git a/webrtc-sys/src/video_frame.rs b/webrtc-sys/src/video_frame.rs index d18ff21..14018c8 100644 --- a/webrtc-sys/src/video_frame.rs +++ b/webrtc-sys/src/video_frame.rs @@ -22,8 +22,8 @@ pub mod ffi { type VideoFrame; - fn width(self: &VideoFrame) -> i32; - fn height(self: &VideoFrame) -> i32; + fn width(self: &VideoFrame) -> u32; + fn height(self: &VideoFrame) -> u32; fn size(self: &VideoFrame) -> u32; fn id(self: &VideoFrame) -> u16; fn timestamp_us(self: &VideoFrame) -> i64; diff --git a/webrtc-sys/src/video_frame_buffer.cpp b/webrtc-sys/src/video_frame_buffer.cpp index bb8ff62..410e267 100644 --- a/webrtc-sys/src/video_frame_buffer.cpp +++ b/webrtc-sys/src/video_frame_buffer.cpp @@ -26,11 +26,11 @@ VideoFrameBufferType VideoFrameBuffer::buffer_type() const { return static_cast(buffer_->type()); } -int VideoFrameBuffer::width() const { +unsigned int VideoFrameBuffer::width() const { return buffer_->width(); } -int VideoFrameBuffer::height() const { +unsigned int VideoFrameBuffer::height() const { return buffer_->height(); } @@ -83,23 +83,23 @@ PlanarYuvBuffer::PlanarYuvBuffer( rtc::scoped_refptr buffer) : VideoFrameBuffer(buffer) {} -int PlanarYuvBuffer::chroma_width() const { +unsigned int PlanarYuvBuffer::chroma_width() const { return buffer()->ChromaWidth(); } -int PlanarYuvBuffer::chroma_height() const { +unsigned int PlanarYuvBuffer::chroma_height() const { return buffer()->ChromaHeight(); } -int PlanarYuvBuffer::stride_y() const { +unsigned int PlanarYuvBuffer::stride_y() const { return buffer()->StrideY(); } -int PlanarYuvBuffer::stride_u() const { +unsigned int PlanarYuvBuffer::stride_u() const { return buffer()->StrideU(); } -int PlanarYuvBuffer::stride_v() const { +unsigned int PlanarYuvBuffer::stride_v() const { return buffer()->StrideV(); } @@ -151,19 +151,19 @@ BiplanarYuvBuffer::BiplanarYuvBuffer( rtc::scoped_refptr buffer) : VideoFrameBuffer(buffer) {} -int BiplanarYuvBuffer::chroma_width() const { +unsigned int BiplanarYuvBuffer::chroma_width() const { return buffer()->ChromaWidth(); } -int BiplanarYuvBuffer::chroma_height() const { +unsigned int BiplanarYuvBuffer::chroma_height() const { return buffer()->ChromaHeight(); } -int BiplanarYuvBuffer::stride_y() const { +unsigned int BiplanarYuvBuffer::stride_y() const { return buffer()->StrideY(); } -int BiplanarYuvBuffer::stride_uv() const { +unsigned int BiplanarYuvBuffer::stride_uv() const { return buffer()->StrideUV(); } @@ -204,7 +204,7 @@ I420ABuffer::I420ABuffer( rtc::scoped_refptr buffer) : I420Buffer(buffer) {} -int I420ABuffer::stride_a() const { +unsigned int I420ABuffer::stride_a() const { return buffer()->StrideA(); } diff --git a/webrtc-sys/src/video_frame_buffer.rs b/webrtc-sys/src/video_frame_buffer.rs index 7bd67dd..fb5096a 100644 --- a/webrtc-sys/src/video_frame_buffer.rs +++ b/webrtc-sys/src/video_frame_buffer.rs @@ -31,8 +31,8 @@ pub mod ffi { type NV12Buffer; fn buffer_type(self: &VideoFrameBuffer) -> VideoFrameBufferType; - fn width(self: &VideoFrameBuffer) -> i32; - fn height(self: &VideoFrameBuffer) -> i32; + fn width(self: &VideoFrameBuffer) -> u32; + fn height(self: &VideoFrameBuffer) -> u32; /// # SAFETY /// If the buffer type is I420, the buffer must be cloned before @@ -47,11 +47,11 @@ pub mod ffi { unsafe fn get_i010(self: Pin<&mut VideoFrameBuffer>) -> UniquePtr; unsafe fn get_nv12(self: Pin<&mut VideoFrameBuffer>) -> UniquePtr; - fn chroma_width(self: &PlanarYuvBuffer) -> i32; - fn chroma_height(self: &PlanarYuvBuffer) -> i32; - fn stride_y(self: &PlanarYuvBuffer) -> i32; - fn stride_u(self: &PlanarYuvBuffer) -> i32; - fn stride_v(self: &PlanarYuvBuffer) -> i32; + fn chroma_width(self: &PlanarYuvBuffer) -> u32; + fn chroma_height(self: &PlanarYuvBuffer) -> u32; + fn stride_y(self: &PlanarYuvBuffer) -> u32; + fn stride_u(self: &PlanarYuvBuffer) -> u32; + fn stride_v(self: &PlanarYuvBuffer) -> u32; fn data_y(self: &PlanarYuv8Buffer) -> *const u8; fn data_u(self: &PlanarYuv8Buffer) -> *const u8; @@ -61,15 +61,15 @@ pub mod ffi { fn data_u(self: &PlanarYuv16BBuffer) -> *const u16; fn data_v(self: &PlanarYuv16BBuffer) -> *const u16; - fn chroma_width(self: &BiplanarYuvBuffer) -> i32; - fn chroma_height(self: &BiplanarYuvBuffer) -> i32; - fn stride_y(self: &BiplanarYuvBuffer) -> i32; - fn stride_uv(self: &BiplanarYuvBuffer) -> i32; + fn chroma_width(self: &BiplanarYuvBuffer) -> u32; + fn chroma_height(self: &BiplanarYuvBuffer) -> u32; + fn stride_y(self: &BiplanarYuvBuffer) -> u32; + fn stride_uv(self: &BiplanarYuvBuffer) -> u32; fn data_y(self: &BiplanarYuv8Buffer) -> *const u8; fn data_uv(self: &BiplanarYuv8Buffer) -> *const u8; - fn stride_a(self: &I420ABuffer) -> i32; + fn stride_a(self: &I420ABuffer) -> u32; fn data_a(self: &I420ABuffer) -> *const u8; fn new_i420_buffer(width: i32, height: i32) -> UniquePtr;