feat: add publishing & audio to ffi (#46)

- Divide FFI protocol into multiple files - Audio support - AV Streams/Sources - Create tracks - Use Dashmap to store handles - Add a capture test - Ignored by GHA atm
2023-05-08 23:12:37 +02:00
parent a5caf12902
commit 1c12e749f2
58 changed files with 3046 additions and 1188 deletions
@@ -0,0 +1,199 @@
+syntax = "proto3";
+
+package livekit;
+option csharp_namespace = "LiveKit.Proto";
+
+import "handle.proto";
+import "participant.proto";
+import "track.proto";
+import "video_frame.proto";
+
+// Connect to a new LiveKit room
+message ConnectRequest {
+  string url = 1;
+  string token = 2;
+  RoomOptions options = 3;
+}
+message ConnectResponse {
+  FFIAsyncId async_id = 1;
+}
+message ConnectCallback {
+  FFIAsyncId async_id = 1;
+  optional string error = 2;
+  RoomInfo room = 3;
+}
+
+// Disconnect from the a room
+message DisconnectRequest { FFIHandleId room_handle = 1; }
+message DisconnectResponse { FFIAsyncId async_id = 1; }
+message DisconnectCallback { }
+
+// Publish a track to the room
+message PublishTrackRequest {
+  FFIHandleId room_handle = 1;
+  FFIHandleId track_handle = 2;
+  TrackPublishOptions options = 3;
+}
+message PublishTrackResponse {
+  FFIAsyncId async_id = 1;
+}
+message PublishTrackCallback {
+  FFIAsyncId async_id = 1;
+  optional string error = 2;
+  TrackPublicationInfo publication = 3;
+}
+
+// Unpublish a track from the room
+message UnpublishTrackRequest {
+  FFIHandleId room_handle = 1;
+  string track_sid = 2;
+  bool stop_on_unpublish = 3;
+}
+message UnpublishTrackResponse {
+  FFIAsyncId async_id = 1;
+}
+message UnpublishTrackCallback {
+  optional string error = 1;
+}
+
+///
+/// Options
+///
+
+message VideoEncoding {
+  uint64 max_bitrate = 1;
+  double max_framerate = 2;
+}
+
+message AudioEncoding {
+  uint64 max_bitrate = 1;
+}
+
+message TrackPublishOptions {
+  // encodings are optional
+  VideoEncoding video_encoding = 1;
+  AudioEncoding audio_encoding = 2;
+  VideoCodec video_codec = 3;
+  bool dtx = 4;
+  bool red = 5;
+  bool simulcast = 6;
+  string name = 7;
+  TrackSource source = 8;
+}
+
+message RoomOptions {
+  bool auto_subscribe = 1;
+  bool adaptive_stream = 2;
+}
+
+///
+/// Room
+///
+
+enum ConnectionQuality {
+  QUALITY_POOR = 0;
+  QUALITY_GOOD = 1;
+  QUALITY_EXCELLENT = 2;
+}
+
+enum ConnectionState {
+  CONN_DISCONNECTED = 0;
+  CONN_CONNECTED = 1;
+  CONN_RECONNECTING = 2;
+  CONN_UNKNOWN = 3;
+}
+
+enum DataPacketKind {
+  KIND_UNRELIABLE = 0;
+  KIND_RELIABLE = 1;
+}
+
+message RoomEvent {
+  FFIHandleId room_handle = 1;
+  oneof message {
+    ParticipantConnected participant_connected = 2;
+    ParticipantDisconnected participant_disconnected = 3;
+    TrackPublished track_published = 4;
+    TrackUnpublished track_unpublished = 5;
+    TrackSubscribed track_subscribed = 6;
+    TrackUnsubscribed track_unsubscribed = 7;
+    TrackMuted track_muted = 8;
+    TrackUnmuted track_unmuted = 9;
+    ActiveSpeakersChanged speakers_changed = 10;
+    ConnectionQualityChanged connection_quality_changed = 11;
+    DataReceived data_received = 12;
+    ConnectionStateChanged connection_state_changed = 13;
+    Connected connected = 14;
+    Disconnected disconnected = 15;
+    Reconnecting reconnecting = 16;
+    Reconnected reconnected = 17;
+  }
+}
+
+message RoomInfo {
+  FFIHandleId handle = 1;
+  string sid = 2;
+  string name = 3;
+  string metadata = 4;
+  ParticipantInfo local_participant = 5;
+  repeated ParticipantInfo participants = 6;
+}
+
+message DataReceived {
+  FFIHandleId handle = 1;
+  optional string participant_sid = 2;
+  uint64 data_ptr = 3;
+  uint64 data_size = 4;
+  DataPacketKind kind = 5;
+}
+
+// Publication isn't needed for subscription events on the FFI
+// The FFI will retrieve the publication using the Track sid
+message TrackSubscribed {
+  string participant_sid = 1;
+  TrackInfo track = 2;
+}
+
+message TrackUnsubscribed {
+  // The FFI language can dispose/remove the VideoSink here
+  string participant_sid = 1;
+  string track_sid = 2;
+}
+
+message TrackMuted {
+  string participant_sid = 1;
+  string track_sid = 2;
+}
+
+message TrackUnmuted {
+  string participant_sid = 1;
+  string track_sid = 2;
+}
+
+message ParticipantConnected { ParticipantInfo info = 1; }
+
+message ParticipantDisconnected { ParticipantInfo info = 1; }
+
+message TrackPublished {
+  string participant_sid = 1;
+  TrackPublicationInfo publication = 2;
+}
+
+message TrackUnpublished {
+  string participant_sid = 1;
+  string publication_sid = 2;
+}
+
+message ActiveSpeakersChanged { repeated string participant_sids = 1; }
+
+message ConnectionQualityChanged {
+  string participant_sid = 1;
+  ConnectionQuality quality = 2;
+}
+
+message ConnectionStateChanged { ConnectionState state = 1; }
+
+message Connected {}
+message Disconnected {}
+message Reconnecting {}
+message Reconnected {}