feat: add publishing & audio to ffi (#46)
- Divide FFI protocol into multiple files
- Audio support
- AV Streams/Sources
- Create tracks
- Use Dashmap to store handles
- Add a capture test
- Ignored by GHA atm
This commit is contained in:
Théo Monnom
@@ -0,0 +1,86 @@
|
||||
syntax = "proto3";
|
||||
|
||||
package livekit;
|
||||
option csharp_namespace = "LiveKit.Proto";
|
||||
|
||||
import "handle.proto";
|
||||
|
||||
// Allocate a new AudioFrameBuffer
|
||||
// This is not necessary required because the data structure is fairly simple
|
||||
// But keep the API consistent with VideoFrame
|
||||
message AllocAudioBufferRequest {
|
||||
uint32 sample_rate = 1;
|
||||
uint32 num_channels = 2;
|
||||
uint32 samples_per_channel = 3;
|
||||
}
|
||||
message AllocAudioBufferResponse { AudioFrameBufferInfo buffer = 1; }
|
||||
|
||||
// Create a new AudioStream
|
||||
// AudioStream is used to receive audio frames from a track
|
||||
message NewAudioStreamRequest {
|
||||
FFIHandleId room_handle = 1;
|
||||
string participant_sid = 2;
|
||||
string track_sid = 3;
|
||||
AudioStreamType type = 4;
|
||||
}
|
||||
message NewAudioStreamResponse { AudioStreamInfo stream = 1; }
|
||||
|
||||
// Create a new AudioSource
|
||||
message NewAudioSourceRequest { AudioSourceType type = 1; }
|
||||
message NewAudioSourceResponse { AudioSourceInfo source = 1; }
|
||||
|
||||
// Push a frame to an AudioSource
|
||||
message CaptureAudioFrameRequest {
|
||||
FFIHandleId source_handle = 1;
|
||||
FFIHandleId buffer_handle = 2;
|
||||
}
|
||||
message CaptureAudioFrameResponse {}
|
||||
|
||||
///
|
||||
/// AudioFrame buffer ///
|
||||
///
|
||||
|
||||
message AudioFrameBufferInfo {
|
||||
FFIHandleId handle = 1;
|
||||
uint64 data_ptr = 2; // *const i16
|
||||
uint32 num_channels = 3;
|
||||
uint32 sample_rate = 4;
|
||||
uint32 samples_per_channel = 5;
|
||||
}
|
||||
|
||||
///
|
||||
/// AudioStream ///
|
||||
///
|
||||
|
||||
enum AudioStreamType {
|
||||
AUDIO_STREAM_NATIVE = 0;
|
||||
AUDIO_STREAM_HTML = 1;
|
||||
}
|
||||
|
||||
message AudioStreamInfo {
|
||||
FFIHandleId handle = 1;
|
||||
AudioStreamType type = 2;
|
||||
string track_sid = 3;
|
||||
}
|
||||
|
||||
message AudioStreamEvent {
|
||||
FFIHandleId handle = 1;
|
||||
oneof message { AudioFrameReceived frame_received = 2; }
|
||||
}
|
||||
|
||||
message AudioFrameReceived {
|
||||
AudioFrameBufferInfo frame = 1;
|
||||
}
|
||||
|
||||
///
|
||||
/// AudioSource ///
|
||||
///
|
||||
|
||||
enum AudioSourceType {
|
||||
AUDIO_SOURCE_NATIVE = 0;
|
||||
}
|
||||
|
||||
message AudioSourceInfo {
|
||||
FFIHandleId handle = 1;
|
||||
AudioSourceType type = 2;
|
||||
}
|
||||
+77
-315
@@ -3,347 +3,109 @@ syntax = "proto3";
|
||||
package livekit;
|
||||
option csharp_namespace = "LiveKit.Proto";
|
||||
|
||||
/// IPC
|
||||
|
||||
/// # Safety
|
||||
/// The foreign language is responsable for disposing an handle
|
||||
/// Forgetting to dispose the handle may lead to memory leaks
|
||||
/// Messages in this file can contain an FFIHandle
|
||||
message FFIHandleId { uint64 id = 1; }
|
||||
import "handle.proto";
|
||||
import "track.proto";
|
||||
import "room.proto";
|
||||
import "participant.proto";
|
||||
import "video_frame.proto";
|
||||
import "audio_frame.proto";
|
||||
|
||||
/// This is the input of livekit_ffi_request function
|
||||
/// We always expect a response (FFIResponse)
|
||||
message FFIRequest {
|
||||
oneof message {
|
||||
InitializeRequest initialize = 1;
|
||||
// Stop all rooms synchronously (Do we need async here?).
|
||||
// e.g: This is used for the Unity Editor after each assemblies reload.
|
||||
DisposeRequest dispose = 2;
|
||||
ConnectRequest async_connect = 3;
|
||||
DisconnectRequest async_disconnect = 4;
|
||||
ToI420Request to_i420 = 5;
|
||||
ToARGBRequest to_argb = 6;
|
||||
|
||||
// Room
|
||||
ConnectRequest connect = 3;
|
||||
DisconnectRequest disconnect = 4;
|
||||
PublishTrackRequest publish_track = 5;
|
||||
UnpublishTrackRequest unpublish_track = 6;
|
||||
|
||||
// Track
|
||||
CreateVideoTrackRequest create_video_track = 7;
|
||||
CreateAudioTrackRequest create_audio_track = 8;
|
||||
|
||||
// Video
|
||||
AllocVideoBufferRequest alloc_video_buffer = 9;
|
||||
NewVideoStreamRequest new_video_stream = 10;
|
||||
NewVideoSourceRequest new_video_source = 11;
|
||||
CaptureVideoFrameRequest capture_video_frame = 12;
|
||||
ToI420Request to_i420 = 13;
|
||||
ToARGBRequest to_argb = 14;
|
||||
|
||||
// Audio
|
||||
AllocAudioBufferRequest alloc_audio_buffer = 15;
|
||||
NewAudioStreamRequest new_audio_stream = 16;
|
||||
NewAudioSourceRequest new_audio_source = 17;
|
||||
CaptureAudioFrameRequest capture_audio_frame = 18;
|
||||
}
|
||||
}
|
||||
|
||||
/// This is the output of livekit_ffi_request function.
|
||||
/// The message field is mostly used to send result of a synchronous operation
|
||||
/// to the foreign language.
|
||||
message FFIResponse {
|
||||
optional uint64 async_id = 1;
|
||||
oneof message { ToI420Response to_i420 = 2; }
|
||||
oneof message {
|
||||
InitializeResponse initialize = 1;
|
||||
DisposeResponse dispose = 2;
|
||||
|
||||
// Room
|
||||
ConnectResponse connect = 3;
|
||||
DisconnectResponse disconnect = 4;
|
||||
PublishTrackResponse publish_track = 5;
|
||||
UnpublishTrackResponse unpublish_track = 6;
|
||||
|
||||
// Track
|
||||
CreateVideoTrackResponse create_video_track = 7;
|
||||
CreateAudioTrackResponse create_audio_track = 8;
|
||||
|
||||
// Video
|
||||
AllocVideoBufferResponse alloc_video_buffer = 9;
|
||||
NewVideoStreamResponse new_video_stream = 10;
|
||||
NewVideoSourceResponse new_video_source = 11;
|
||||
CaptureVideoFrameResponse capture_video_frame = 12;
|
||||
ToI420Response to_i420 = 13;
|
||||
ToARGBResponse to_argb = 14;
|
||||
|
||||
// Audio
|
||||
AllocAudioBufferResponse alloc_audio_buffer = 15;
|
||||
NewAudioStreamResponse new_audio_stream = 16;
|
||||
NewAudioSourceResponse new_audio_source = 17;
|
||||
CaptureAudioFrameResponse capture_audio_frame = 18;
|
||||
}
|
||||
}
|
||||
|
||||
/// This message is used to receive the result of asynchronous requests
|
||||
/// Or to receive events of a Room
|
||||
message FFIEvent {
|
||||
// Used if the message is used to send the async result to the foreign
|
||||
// language
|
||||
optional uint64 async_id = 1;
|
||||
oneof message {
|
||||
ConnectEvent connect_event = 2;
|
||||
RoomEvent room_event = 3;
|
||||
TrackEvent track_event = 4;
|
||||
ParticipantEvent participant_event = 5;
|
||||
RoomEvent room_event = 1;
|
||||
TrackEvent track_event = 2;
|
||||
ParticipantEvent participant_event = 3;
|
||||
VideoStreamEvent video_stream_event = 4;
|
||||
AudioStreamEvent audio_stream_event = 5;
|
||||
ConnectCallback connect = 6;
|
||||
DisposeCallback dispose = 7;
|
||||
PublishTrackCallback publish_track = 8;
|
||||
}
|
||||
}
|
||||
|
||||
// Setup the callback where the foreign language can receive events
|
||||
// and responses to asynchronous requests
|
||||
message InitializeRequest { uint64 event_callback_ptr = 1; }
|
||||
message InitializeResponse {}
|
||||
|
||||
message DisposeRequest {}
|
||||
|
||||
message ConnectRequest {
|
||||
string url = 1;
|
||||
string token = 2;
|
||||
RoomOptions options = 3;
|
||||
// Stop all rooms synchronously (Do we need async here?).
|
||||
// e.g: This is used for the Unity Editor after each assemblies reload.
|
||||
message DisposeRequest {
|
||||
bool async = 1;
|
||||
}
|
||||
|
||||
message DisconnectRequest { string room_sid = 1; }
|
||||
|
||||
/// Convert a VideoFrameBuffer to a I420Buffer
|
||||
message ToI420Request {
|
||||
FFIHandleId buffer = 1; // NOTE: This buffer will be dropped!
|
||||
message DisposeResponse {
|
||||
optional FFIAsyncId async_id = 1; // None if sync
|
||||
}
|
||||
|
||||
message ToARGBRequest {
|
||||
FFIHandleId buffer = 1;
|
||||
uint64 dst_ptr = 2;
|
||||
VideoFormatType dst_format = 3;
|
||||
int32 dst_stride = 4;
|
||||
int32 dst_width = 5;
|
||||
int32 dst_height = 6;
|
||||
message DisposeCallback {
|
||||
FFIAsyncId async_id = 1;
|
||||
}
|
||||
|
||||
message ConnectEvent {
|
||||
bool success = 1;
|
||||
optional RoomInfo room = 2;
|
||||
}
|
||||
// TODO(theomonnom): Debug messages (Print handles, forward logs).
|
||||
|
||||
message ToI420Response { optional VideoFrameBufferInfo new_buffer = 1; }
|
||||
|
||||
/// Models
|
||||
|
||||
message Dimension {
|
||||
uint32 width = 1;
|
||||
uint32 height = 2;
|
||||
}
|
||||
|
||||
message RoomOptions {
|
||||
bool auto_subscribe = 1;
|
||||
bool adaptive_stream = 2;
|
||||
}
|
||||
|
||||
message RoomInfo {
|
||||
string sid = 1;
|
||||
string name = 2;
|
||||
string metadata = 3;
|
||||
ParticipantInfo local_participant = 4;
|
||||
repeated ParticipantInfo participants = 5;
|
||||
}
|
||||
|
||||
message ParticipantInfo {
|
||||
string sid = 1;
|
||||
string name = 2;
|
||||
string identity = 3;
|
||||
string metadata = 4;
|
||||
repeated TrackPublicationInfo publications = 5;
|
||||
}
|
||||
|
||||
message TrackPublicationInfo {
|
||||
string sid = 1;
|
||||
string name = 2;
|
||||
TrackKind kind = 3;
|
||||
TrackSource source = 4;
|
||||
bool simulcasted = 5;
|
||||
Dimension dimension = 6;
|
||||
string mime_type = 7;
|
||||
bool muted = 8;
|
||||
}
|
||||
|
||||
message TrackInfo {
|
||||
string sid = 1;
|
||||
string name = 2;
|
||||
TrackKind kind = 3;
|
||||
StreamState stream_state = 4;
|
||||
bool muted = 5;
|
||||
}
|
||||
|
||||
message VideoSinkInfo {
|
||||
string track_sid = 1;
|
||||
// More info?
|
||||
}
|
||||
|
||||
enum TrackKind {
|
||||
KIND_UNKNOWN = 0;
|
||||
KIND_AUDIO = 1;
|
||||
KIND_VIDEO = 2;
|
||||
}
|
||||
|
||||
enum TrackSource {
|
||||
SOURCE_UNKNOWN = 0;
|
||||
SOURCE_CAMERA = 1;
|
||||
SOURCE_MICROPHONE = 2;
|
||||
SOURCE_SCREENSHARE = 3;
|
||||
SOURCE_SCREENSHARE_AUDIO = 4;
|
||||
}
|
||||
|
||||
enum ConnectionQuality {
|
||||
QUALITY_POOR = 0;
|
||||
QUALITY_GOOD = 1;
|
||||
QUALITY_EXCELLENT = 2;
|
||||
}
|
||||
|
||||
enum ConnectionState {
|
||||
CONN_DISCONNECTED = 0;
|
||||
CONN_CONNECTED = 1;
|
||||
CONN_RECONNECTING = 2;
|
||||
CONN_UNKNOWN = 3;
|
||||
}
|
||||
|
||||
enum StreamState {
|
||||
STATE_UNKNOWN = 0;
|
||||
STATE_ACTIVE = 1;
|
||||
STATE_PAUSED = 2;
|
||||
}
|
||||
|
||||
enum VideoRotation {
|
||||
VIDEO_ROTATION_0 = 0;
|
||||
VIDEO_ROTATION_90 = 1;
|
||||
VIDEO_ROTATION_180 = 2;
|
||||
VIDEO_ROTATION_270 = 3;
|
||||
}
|
||||
|
||||
enum DataPacketKind {
|
||||
KIND_UNRELIABLE = 0;
|
||||
KIND_RELIABLE = 1;
|
||||
}
|
||||
|
||||
enum VideoFormatType {
|
||||
FORMAT_ARGB = 0;
|
||||
FORMAT_BGRA = 1;
|
||||
FORMAT_ABGR = 2;
|
||||
FORMAT_RGBA = 3;
|
||||
}
|
||||
|
||||
/// Room Events
|
||||
|
||||
message RoomEvent {
|
||||
string room_sid = 1;
|
||||
oneof message {
|
||||
ParticipantConnected participant_connected = 2;
|
||||
ParticipantDisconnected participant_disconnected = 3;
|
||||
TrackPublished track_published = 4;
|
||||
TrackUnpublished track_unpublished = 5;
|
||||
TrackSubscribed track_subscribed = 6;
|
||||
TrackUnsubscribed track_unsubscribed = 7;
|
||||
TrackMuted track_muted = 8;
|
||||
TrackUnmuted track_unmuted = 9;
|
||||
ActiveSpeakersChanged speakers_changed = 10;
|
||||
ConnectionQualityChanged connection_quality_changed = 11;
|
||||
DataReceived data_received = 12;
|
||||
ConnectionStateChanged connection_state_changed = 13;
|
||||
Connected connected = 14;
|
||||
Disconnected disconnected = 15;
|
||||
Reconnecting reconnecting = 16;
|
||||
Reconnected reconnected = 17;
|
||||
}
|
||||
}
|
||||
|
||||
message ParticipantConnected { ParticipantInfo info = 1; }
|
||||
|
||||
message ParticipantDisconnected { ParticipantInfo info = 1; }
|
||||
|
||||
message TrackPublished {
|
||||
string participant_sid = 1;
|
||||
TrackPublicationInfo publication = 2;
|
||||
}
|
||||
|
||||
message TrackUnpublished {
|
||||
string participant_sid = 1;
|
||||
string publication_sid = 2;
|
||||
}
|
||||
|
||||
// Publication isn't needed for subscription events on the FFI
|
||||
// The FFI will retrieve the publication using the Track sid
|
||||
message TrackSubscribed {
|
||||
string participant_sid = 1;
|
||||
TrackInfo track = 2;
|
||||
VideoSinkInfo sink = 3;
|
||||
}
|
||||
|
||||
message TrackUnsubscribed {
|
||||
// The FFI language can dispose/remove the VideoSink here
|
||||
string participant_sid = 1;
|
||||
string track_sid = 2;
|
||||
}
|
||||
|
||||
message TrackMuted {
|
||||
string participant_sid = 1;
|
||||
string track_sid = 2;
|
||||
}
|
||||
|
||||
message TrackUnmuted {
|
||||
string participant_sid = 1;
|
||||
string track_sid = 2;
|
||||
}
|
||||
|
||||
message ActiveSpeakersChanged { repeated string participant_sids = 1; }
|
||||
|
||||
message ConnectionQualityChanged {
|
||||
string participant_sid = 1;
|
||||
ConnectionQuality quality = 2;
|
||||
}
|
||||
|
||||
message DataReceived {
|
||||
FFIHandleId handle = 1;
|
||||
optional string participant_sid = 2;
|
||||
uint64 data_ptr = 3;
|
||||
uint64 data_size = 4;
|
||||
DataPacketKind kind = 5;
|
||||
}
|
||||
|
||||
message ConnectionStateChanged { ConnectionState state = 1; }
|
||||
|
||||
message Connected {}
|
||||
message Disconnected {}
|
||||
message Reconnecting {}
|
||||
message Reconnected {}
|
||||
|
||||
/// Track Events
|
||||
|
||||
message TrackEvent {
|
||||
string track_sid = 1;
|
||||
oneof message { FrameReceived frame_received = 2; }
|
||||
}
|
||||
|
||||
message FrameReceived {
|
||||
VideoFrameInfo frame = 1;
|
||||
VideoFrameBufferInfo buffer = 2;
|
||||
}
|
||||
|
||||
message VideoFrameInfo {
|
||||
int64 timestamp = 1;
|
||||
VideoRotation rotation = 2;
|
||||
}
|
||||
|
||||
message VideoFrameBufferInfo {
|
||||
FFIHandleId handle = 1;
|
||||
VideoFrameBufferType buffer_type = 2;
|
||||
int32 width = 3;
|
||||
int32 height = 4;
|
||||
oneof buffer {
|
||||
PlanarYuvBufferInfo yuv = 5;
|
||||
BiplanarYuvBufferInfo bi_yuv = 6;
|
||||
NativeBufferInfo native = 7;
|
||||
}
|
||||
}
|
||||
|
||||
message PlanarYuvBufferInfo {
|
||||
int32 chroma_width = 1;
|
||||
int32 chroma_height = 2;
|
||||
int32 stride_y = 3;
|
||||
int32 stride_u = 4;
|
||||
int32 stride_v = 5;
|
||||
int32 stride_a = 6;
|
||||
|
||||
// *const u8 or *const u16
|
||||
uint64 data_y_ptr = 7;
|
||||
uint64 data_u_ptr = 8;
|
||||
uint64 data_v_ptr = 9;
|
||||
uint64 data_a_ptr = 10; // nullptr = no alpha
|
||||
}
|
||||
|
||||
message BiplanarYuvBufferInfo {
|
||||
int32 chroma_width = 1;
|
||||
int32 chroma_height = 2;
|
||||
int32 stride_y = 3;
|
||||
int32 stride_uv = 4;
|
||||
|
||||
uint64 data_y_ptr = 5;
|
||||
uint64 data_uv_ptr = 6;
|
||||
}
|
||||
|
||||
message NativeBufferInfo {
|
||||
// TODO(theomonnom): Expose graphic context?
|
||||
}
|
||||
|
||||
enum VideoFrameBufferType {
|
||||
NATIVE = 0;
|
||||
I420 = 1;
|
||||
I420A = 2;
|
||||
I422 = 3;
|
||||
I444 = 4;
|
||||
I010 = 5;
|
||||
NV12 = 6;
|
||||
WEBGL = 7;
|
||||
}
|
||||
|
||||
/// Participant Events
|
||||
|
||||
message ParticipantEvent {
|
||||
string participant_sid = 1;
|
||||
oneof message { IsSpeakingChanged speaking_changed = 2; }
|
||||
}
|
||||
|
||||
message IsSpeakingChanged { bool speaking = 1; }
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
syntax = "proto3";
|
||||
|
||||
package livekit;
|
||||
option csharp_namespace = "LiveKit.Proto";
|
||||
|
||||
/// # Safety
|
||||
/// The foreign language is responsable for disposing handles
|
||||
/// Forgetting to dispose the handle may lead to memory leaks
|
||||
/// Messages in this file can contain an FFIHandle
|
||||
message FFIHandleId {
|
||||
uint64 id = 1;
|
||||
}
|
||||
|
||||
/// Link the request/response of an asynchronous call
|
||||
message FFIAsyncId {
|
||||
uint64 id = 1;
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
syntax = "proto3";
|
||||
|
||||
package livekit;
|
||||
option csharp_namespace = "LiveKit.Proto";
|
||||
|
||||
//import "handle.proto";
|
||||
import "track.proto";
|
||||
|
||||
message ParticipantInfo {
|
||||
string sid = 1;
|
||||
string name = 2;
|
||||
string identity = 3;
|
||||
string metadata = 4;
|
||||
repeated TrackPublicationInfo publications = 5;
|
||||
}
|
||||
|
||||
message ParticipantEvent {
|
||||
string participant_sid = 1;
|
||||
oneof message { IsSpeakingChanged speaking_changed = 2; }
|
||||
}
|
||||
|
||||
message IsSpeakingChanged { bool speaking = 1; }
|
||||
@@ -0,0 +1,199 @@
|
||||
syntax = "proto3";
|
||||
|
||||
package livekit;
|
||||
option csharp_namespace = "LiveKit.Proto";
|
||||
|
||||
import "handle.proto";
|
||||
import "participant.proto";
|
||||
import "track.proto";
|
||||
import "video_frame.proto";
|
||||
|
||||
// Connect to a new LiveKit room
|
||||
message ConnectRequest {
|
||||
string url = 1;
|
||||
string token = 2;
|
||||
RoomOptions options = 3;
|
||||
}
|
||||
message ConnectResponse {
|
||||
FFIAsyncId async_id = 1;
|
||||
}
|
||||
message ConnectCallback {
|
||||
FFIAsyncId async_id = 1;
|
||||
optional string error = 2;
|
||||
RoomInfo room = 3;
|
||||
}
|
||||
|
||||
// Disconnect from the a room
|
||||
message DisconnectRequest { FFIHandleId room_handle = 1; }
|
||||
message DisconnectResponse { FFIAsyncId async_id = 1; }
|
||||
message DisconnectCallback { }
|
||||
|
||||
// Publish a track to the room
|
||||
message PublishTrackRequest {
|
||||
FFIHandleId room_handle = 1;
|
||||
FFIHandleId track_handle = 2;
|
||||
TrackPublishOptions options = 3;
|
||||
}
|
||||
message PublishTrackResponse {
|
||||
FFIAsyncId async_id = 1;
|
||||
}
|
||||
message PublishTrackCallback {
|
||||
FFIAsyncId async_id = 1;
|
||||
optional string error = 2;
|
||||
TrackPublicationInfo publication = 3;
|
||||
}
|
||||
|
||||
// Unpublish a track from the room
|
||||
message UnpublishTrackRequest {
|
||||
FFIHandleId room_handle = 1;
|
||||
string track_sid = 2;
|
||||
bool stop_on_unpublish = 3;
|
||||
}
|
||||
message UnpublishTrackResponse {
|
||||
FFIAsyncId async_id = 1;
|
||||
}
|
||||
message UnpublishTrackCallback {
|
||||
optional string error = 1;
|
||||
}
|
||||
|
||||
///
|
||||
/// Options
|
||||
///
|
||||
|
||||
message VideoEncoding {
|
||||
uint64 max_bitrate = 1;
|
||||
double max_framerate = 2;
|
||||
}
|
||||
|
||||
message AudioEncoding {
|
||||
uint64 max_bitrate = 1;
|
||||
}
|
||||
|
||||
message TrackPublishOptions {
|
||||
// encodings are optional
|
||||
VideoEncoding video_encoding = 1;
|
||||
AudioEncoding audio_encoding = 2;
|
||||
VideoCodec video_codec = 3;
|
||||
bool dtx = 4;
|
||||
bool red = 5;
|
||||
bool simulcast = 6;
|
||||
string name = 7;
|
||||
TrackSource source = 8;
|
||||
}
|
||||
|
||||
message RoomOptions {
|
||||
bool auto_subscribe = 1;
|
||||
bool adaptive_stream = 2;
|
||||
}
|
||||
|
||||
///
|
||||
/// Room
|
||||
///
|
||||
|
||||
enum ConnectionQuality {
|
||||
QUALITY_POOR = 0;
|
||||
QUALITY_GOOD = 1;
|
||||
QUALITY_EXCELLENT = 2;
|
||||
}
|
||||
|
||||
enum ConnectionState {
|
||||
CONN_DISCONNECTED = 0;
|
||||
CONN_CONNECTED = 1;
|
||||
CONN_RECONNECTING = 2;
|
||||
CONN_UNKNOWN = 3;
|
||||
}
|
||||
|
||||
enum DataPacketKind {
|
||||
KIND_UNRELIABLE = 0;
|
||||
KIND_RELIABLE = 1;
|
||||
}
|
||||
|
||||
message RoomEvent {
|
||||
FFIHandleId room_handle = 1;
|
||||
oneof message {
|
||||
ParticipantConnected participant_connected = 2;
|
||||
ParticipantDisconnected participant_disconnected = 3;
|
||||
TrackPublished track_published = 4;
|
||||
TrackUnpublished track_unpublished = 5;
|
||||
TrackSubscribed track_subscribed = 6;
|
||||
TrackUnsubscribed track_unsubscribed = 7;
|
||||
TrackMuted track_muted = 8;
|
||||
TrackUnmuted track_unmuted = 9;
|
||||
ActiveSpeakersChanged speakers_changed = 10;
|
||||
ConnectionQualityChanged connection_quality_changed = 11;
|
||||
DataReceived data_received = 12;
|
||||
ConnectionStateChanged connection_state_changed = 13;
|
||||
Connected connected = 14;
|
||||
Disconnected disconnected = 15;
|
||||
Reconnecting reconnecting = 16;
|
||||
Reconnected reconnected = 17;
|
||||
}
|
||||
}
|
||||
|
||||
message RoomInfo {
|
||||
FFIHandleId handle = 1;
|
||||
string sid = 2;
|
||||
string name = 3;
|
||||
string metadata = 4;
|
||||
ParticipantInfo local_participant = 5;
|
||||
repeated ParticipantInfo participants = 6;
|
||||
}
|
||||
|
||||
message DataReceived {
|
||||
FFIHandleId handle = 1;
|
||||
optional string participant_sid = 2;
|
||||
uint64 data_ptr = 3;
|
||||
uint64 data_size = 4;
|
||||
DataPacketKind kind = 5;
|
||||
}
|
||||
|
||||
// Publication isn't needed for subscription events on the FFI
|
||||
// The FFI will retrieve the publication using the Track sid
|
||||
message TrackSubscribed {
|
||||
string participant_sid = 1;
|
||||
TrackInfo track = 2;
|
||||
}
|
||||
|
||||
message TrackUnsubscribed {
|
||||
// The FFI language can dispose/remove the VideoSink here
|
||||
string participant_sid = 1;
|
||||
string track_sid = 2;
|
||||
}
|
||||
|
||||
message TrackMuted {
|
||||
string participant_sid = 1;
|
||||
string track_sid = 2;
|
||||
}
|
||||
|
||||
message TrackUnmuted {
|
||||
string participant_sid = 1;
|
||||
string track_sid = 2;
|
||||
}
|
||||
|
||||
message ParticipantConnected { ParticipantInfo info = 1; }
|
||||
|
||||
message ParticipantDisconnected { ParticipantInfo info = 1; }
|
||||
|
||||
message TrackPublished {
|
||||
string participant_sid = 1;
|
||||
TrackPublicationInfo publication = 2;
|
||||
}
|
||||
|
||||
message TrackUnpublished {
|
||||
string participant_sid = 1;
|
||||
string publication_sid = 2;
|
||||
}
|
||||
|
||||
message ActiveSpeakersChanged { repeated string participant_sids = 1; }
|
||||
|
||||
message ConnectionQualityChanged {
|
||||
string participant_sid = 1;
|
||||
ConnectionQuality quality = 2;
|
||||
}
|
||||
|
||||
message ConnectionStateChanged { ConnectionState state = 1; }
|
||||
|
||||
message Connected {}
|
||||
message Disconnected {}
|
||||
message Reconnecting {}
|
||||
message Reconnected {}
|
||||
@@ -0,0 +1,91 @@
|
||||
syntax = "proto3";
|
||||
|
||||
package livekit;
|
||||
option csharp_namespace = "LiveKit.Proto";
|
||||
|
||||
import "handle.proto";
|
||||
import "video_frame.proto";
|
||||
import "audio_frame.proto";
|
||||
|
||||
message VideoCaptureOptions {
|
||||
VideoResolution resolution = 1;
|
||||
}
|
||||
|
||||
message AudioCaptureOptions {
|
||||
bool echo_cancellation = 1;
|
||||
bool noise_suppression = 2;
|
||||
bool auto_gain_control = 3;
|
||||
}
|
||||
|
||||
// Create a new VideoTrack from a VideoSource
|
||||
message CreateVideoTrackRequest {
|
||||
string name = 1;
|
||||
VideoCaptureOptions options = 2;
|
||||
FFIHandleId source_handle = 3;
|
||||
}
|
||||
message CreateVideoTrackResponse {
|
||||
TrackInfo track = 1;
|
||||
}
|
||||
|
||||
// Create a new AudioTrack from a AudioSource
|
||||
message CreateAudioTrackRequest {
|
||||
string name = 1;
|
||||
AudioCaptureOptions options = 2;
|
||||
FFIHandleId source_handle = 3;
|
||||
}
|
||||
message CreateAudioTrackResponse {
|
||||
TrackInfo track = 1;
|
||||
}
|
||||
|
||||
///
|
||||
/// Track
|
||||
///
|
||||
|
||||
message TrackEvent {}
|
||||
|
||||
enum TrackKind {
|
||||
KIND_UNKNOWN = 0;
|
||||
KIND_AUDIO = 1;
|
||||
KIND_VIDEO = 2;
|
||||
}
|
||||
|
||||
enum TrackSource {
|
||||
SOURCE_UNKNOWN = 0;
|
||||
SOURCE_CAMERA = 1;
|
||||
SOURCE_MICROPHONE = 2;
|
||||
SOURCE_SCREENSHARE = 3;
|
||||
SOURCE_SCREENSHARE_AUDIO = 4;
|
||||
}
|
||||
|
||||
enum StreamState {
|
||||
STATE_UNKNOWN = 0;
|
||||
STATE_ACTIVE = 1;
|
||||
STATE_PAUSED = 2;
|
||||
}
|
||||
|
||||
// TODO(theomonnom): Should we have a separate message whether the track is local or remote?
|
||||
|
||||
message TrackPublicationInfo {
|
||||
string sid = 1;
|
||||
string name = 2;
|
||||
TrackKind kind = 3;
|
||||
TrackSource source = 4;
|
||||
bool simulcasted = 5;
|
||||
uint32 width = 6;
|
||||
uint32 height = 7;
|
||||
string mime_type = 8;
|
||||
bool muted = 9;
|
||||
bool remote = 10;
|
||||
}
|
||||
|
||||
message TrackInfo {
|
||||
// Tracks created/owned by the client will have a handle
|
||||
FFIHandleId opt_handle = 1;
|
||||
string sid = 2;
|
||||
string name = 3;
|
||||
TrackKind kind = 4;
|
||||
StreamState stream_state = 5;
|
||||
bool muted = 6;
|
||||
bool remote = 7;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
syntax = "proto3";
|
||||
|
||||
package livekit;
|
||||
option csharp_namespace = "LiveKit.Proto";
|
||||
|
||||
import "handle.proto";
|
||||
|
||||
// Allocate a new VideoFrameBuffer
|
||||
message AllocVideoBufferRequest {
|
||||
VideoFrameBufferType type = 1; // Only I420 is supported atm
|
||||
uint32 width = 2;
|
||||
uint32 height = 3;
|
||||
}
|
||||
message AllocVideoBufferResponse { VideoFrameBufferInfo buffer = 1; }
|
||||
|
||||
// Create a new VideoStream
|
||||
// VideoStream is used to receive video frames from a track
|
||||
message NewVideoStreamRequest {
|
||||
FFIHandleId room_handle = 1;
|
||||
string participant_sid = 2;
|
||||
string track_sid = 3;
|
||||
VideoStreamType type = 4;
|
||||
}
|
||||
message NewVideoStreamResponse { VideoStreamInfo stream = 1; }
|
||||
|
||||
// Create a new VideoSource
|
||||
// VideoSource is used to send video frame to a track
|
||||
message NewVideoSourceRequest { VideoSourceType type = 1; }
|
||||
message NewVideoSourceResponse { VideoSourceInfo source = 1; }
|
||||
|
||||
// Push a frame to a VideoSource
|
||||
message CaptureVideoFrameRequest {
|
||||
FFIHandleId source_handle = 1;
|
||||
VideoFrameInfo frame = 2;
|
||||
FFIHandleId buffer_handle = 3;
|
||||
}
|
||||
message CaptureVideoFrameResponse {}
|
||||
|
||||
// Convert a RGBA frame to a I420 YUV frame
|
||||
// Or convert another YUV frame format to I420
|
||||
message ToI420Request {
|
||||
bool flip_y = 1;
|
||||
oneof from {
|
||||
ARGBBufferInfo argb = 2;
|
||||
FFIHandleId buffer = 3;
|
||||
}
|
||||
}
|
||||
message ToI420Response { VideoFrameBufferInfo buffer = 1; }
|
||||
|
||||
// Convert a YUV frame to a RGBA frame
|
||||
// Only I420 is supported atm
|
||||
message ToARGBRequest {
|
||||
FFIHandleId buffer = 1;
|
||||
uint64 dst_ptr = 2;
|
||||
VideoFormatType dst_format = 3;
|
||||
uint32 dst_stride = 4;
|
||||
uint32 dst_width = 5;
|
||||
uint32 dst_height = 6;
|
||||
bool flip_y = 7;
|
||||
}
|
||||
message ToARGBResponse {}
|
||||
|
||||
///
|
||||
/// VideoFrame buffers ///
|
||||
///
|
||||
|
||||
message VideoResolution {
|
||||
uint32 width = 1;
|
||||
uint32 height = 2;
|
||||
double frame_rate = 3;
|
||||
}
|
||||
|
||||
enum VideoCodec {
|
||||
VP8 = 0;
|
||||
H264 = 1;
|
||||
AV1 = 2;
|
||||
}
|
||||
|
||||
enum VideoRotation {
|
||||
VIDEO_ROTATION_0 = 0;
|
||||
VIDEO_ROTATION_90 = 1;
|
||||
VIDEO_ROTATION_180 = 2;
|
||||
VIDEO_ROTATION_270 = 3;
|
||||
}
|
||||
|
||||
enum VideoFormatType {
|
||||
FORMAT_ARGB = 0;
|
||||
FORMAT_BGRA = 1;
|
||||
FORMAT_ABGR = 2;
|
||||
FORMAT_RGBA = 3;
|
||||
}
|
||||
|
||||
enum VideoFrameBufferType {
|
||||
NATIVE = 0;
|
||||
I420 = 1;
|
||||
I420A = 2;
|
||||
I422 = 3;
|
||||
I444 = 4;
|
||||
I010 = 5;
|
||||
NV12 = 6;
|
||||
WEBGL = 7;
|
||||
}
|
||||
|
||||
message ARGBBufferInfo {
|
||||
uint64 ptr = 1;
|
||||
VideoFormatType format = 2;
|
||||
uint32 stride = 3;
|
||||
uint32 width = 4;
|
||||
uint32 height = 5;
|
||||
}
|
||||
|
||||
message VideoFrameInfo {
|
||||
int64 timestamp = 1;
|
||||
VideoRotation rotation = 2;
|
||||
}
|
||||
|
||||
message VideoFrameBufferInfo {
|
||||
FFIHandleId handle = 1;
|
||||
VideoFrameBufferType buffer_type = 2;
|
||||
uint32 width = 3;
|
||||
uint32 height = 4;
|
||||
oneof buffer {
|
||||
PlanarYuvBufferInfo yuv = 5;
|
||||
BiplanarYuvBufferInfo bi_yuv = 6;
|
||||
NativeBufferInfo native = 7;
|
||||
}
|
||||
}
|
||||
|
||||
message PlanarYuvBufferInfo {
|
||||
uint32 chroma_width = 1;
|
||||
uint32 chroma_height = 2;
|
||||
uint32 stride_y = 3;
|
||||
uint32 stride_u = 4;
|
||||
uint32 stride_v = 5;
|
||||
uint32 stride_a = 6;
|
||||
|
||||
// *const u8 or *const u16
|
||||
uint64 data_y_ptr = 7;
|
||||
uint64 data_u_ptr = 8;
|
||||
uint64 data_v_ptr = 9;
|
||||
uint64 data_a_ptr = 10; // nullptr = no alpha
|
||||
}
|
||||
|
||||
message BiplanarYuvBufferInfo {
|
||||
uint32 chroma_width = 1;
|
||||
uint32 chroma_height = 2;
|
||||
uint32 stride_y = 3;
|
||||
uint32 stride_uv = 4;
|
||||
|
||||
uint64 data_y_ptr = 5;
|
||||
uint64 data_uv_ptr = 6;
|
||||
}
|
||||
|
||||
message NativeBufferInfo {
|
||||
// TODO(theomonnom): Expose graphic context?
|
||||
}
|
||||
|
||||
///
|
||||
/// VideoStream ///
|
||||
///
|
||||
|
||||
enum VideoStreamType {
|
||||
VIDEO_STREAM_NATIVE = 0;
|
||||
VIDEO_STREAM_WEBGL = 1;
|
||||
VIDEO_STREAM_HTML = 2;
|
||||
}
|
||||
|
||||
message VideoStreamInfo {
|
||||
FFIHandleId handle = 1;
|
||||
VideoStreamType type = 2;
|
||||
string track_sid = 3;
|
||||
}
|
||||
|
||||
message VideoStreamEvent {
|
||||
FFIHandleId handle = 1;
|
||||
oneof message { VideoFrameReceived frame_received = 2; }
|
||||
}
|
||||
|
||||
message VideoFrameReceived {
|
||||
VideoFrameInfo frame = 1;
|
||||
VideoFrameBufferInfo buffer = 2;
|
||||
}
|
||||
|
||||
///
|
||||
/// VideoSource ///
|
||||
///
|
||||
|
||||
enum VideoSourceType {
|
||||
VIDEO_SOURCE_NATIVE = 0;
|
||||
}
|
||||
|
||||
message VideoSourceInfo {
|
||||
// # SAFETY
|
||||
// This handle must not be dropped if a track is currently using it
|
||||
FFIHandleId handle = 1;
|
||||
VideoSourceType type = 2;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user