feat: add publishing & audio to ffi (#46)

- Divide FFI protocol into multiple files - Audio support - AV Streams/Sources - Create tracks - Use Dashmap to store handles - Add a capture test - Ignored by GHA atm
2023-05-08 23:12:37 +02:00
parent a5caf12902
commit 1c12e749f2
58 changed files with 3046 additions and 1188 deletions
@@ -0,0 +1,198 @@
+syntax = "proto3";
+
+package livekit;
+option csharp_namespace = "LiveKit.Proto";
+
+import "handle.proto";
+
+// Allocate a new VideoFrameBuffer
+message AllocVideoBufferRequest {
+  VideoFrameBufferType type = 1; // Only I420 is supported atm
+  uint32 width = 2;
+  uint32 height = 3;
+}
+message AllocVideoBufferResponse { VideoFrameBufferInfo buffer = 1; }
+
+// Create a new VideoStream
+// VideoStream is used to receive video frames from a track
+message NewVideoStreamRequest {
+  FFIHandleId room_handle = 1;
+  string participant_sid = 2;
+  string track_sid = 3;
+  VideoStreamType type = 4;
+}
+message NewVideoStreamResponse { VideoStreamInfo stream = 1; }
+
+// Create a new VideoSource
+// VideoSource is used to send video frame to a track
+message NewVideoSourceRequest { VideoSourceType type = 1; }
+message NewVideoSourceResponse { VideoSourceInfo source = 1; }
+
+// Push a frame to a VideoSource
+message CaptureVideoFrameRequest {
+  FFIHandleId source_handle = 1;
+  VideoFrameInfo frame = 2;
+  FFIHandleId buffer_handle = 3;
+}
+message CaptureVideoFrameResponse {}
+
+// Convert a RGBA frame to a I420 YUV frame
+// Or convert another YUV frame format to I420
+message ToI420Request {
+  bool flip_y = 1;
+  oneof from {
+    ARGBBufferInfo argb = 2;
+    FFIHandleId buffer = 3;
+  }
+}
+message ToI420Response { VideoFrameBufferInfo buffer = 1; }
+
+// Convert a YUV frame to a RGBA frame
+// Only I420 is supported atm
+message ToARGBRequest {
+  FFIHandleId buffer = 1;
+  uint64 dst_ptr = 2;
+  VideoFormatType dst_format = 3;
+  uint32 dst_stride = 4;
+  uint32 dst_width = 5;
+  uint32 dst_height = 6;
+  bool flip_y = 7;
+}
+message ToARGBResponse {}
+
+///
+/// VideoFrame buffers ///
+///
+
+message VideoResolution {
+  uint32 width = 1;
+  uint32 height = 2;
+  double frame_rate = 3;
+}
+
+enum VideoCodec {
+  VP8 = 0;
+  H264 = 1;
+  AV1 = 2;
+}
+
+enum VideoRotation {
+  VIDEO_ROTATION_0 = 0;
+  VIDEO_ROTATION_90 = 1;
+  VIDEO_ROTATION_180 = 2;
+  VIDEO_ROTATION_270 = 3;
+}
+
+enum VideoFormatType {
+  FORMAT_ARGB = 0;
+  FORMAT_BGRA = 1;
+  FORMAT_ABGR = 2;
+  FORMAT_RGBA = 3;
+}
+
+enum VideoFrameBufferType {
+  NATIVE = 0;
+  I420 = 1;
+  I420A = 2;
+  I422 = 3;
+  I444 = 4;
+  I010 = 5;
+  NV12 = 6;
+  WEBGL = 7;
+}
+
+message ARGBBufferInfo {
+  uint64 ptr = 1;
+  VideoFormatType format = 2;
+  uint32 stride = 3;
+  uint32 width = 4;
+  uint32 height = 5;
+}
+
+message VideoFrameInfo {
+  int64 timestamp = 1;
+  VideoRotation rotation = 2;
+}
+
+message VideoFrameBufferInfo {
+  FFIHandleId handle = 1;
+  VideoFrameBufferType buffer_type = 2;
+  uint32 width = 3;
+  uint32 height = 4;
+  oneof buffer {
+    PlanarYuvBufferInfo yuv = 5;
+    BiplanarYuvBufferInfo bi_yuv = 6;
+    NativeBufferInfo native = 7;
+  }
+}
+
+message PlanarYuvBufferInfo {
+  uint32 chroma_width = 1;
+  uint32 chroma_height = 2;
+  uint32 stride_y = 3;
+  uint32 stride_u = 4;
+  uint32 stride_v = 5;
+  uint32 stride_a = 6;
+
+  // *const u8 or *const u16
+  uint64 data_y_ptr = 7;
+  uint64 data_u_ptr = 8;
+  uint64 data_v_ptr = 9;
+  uint64 data_a_ptr = 10; // nullptr = no alpha
+}
+
+message BiplanarYuvBufferInfo {
+  uint32 chroma_width = 1;
+  uint32 chroma_height = 2;
+  uint32 stride_y = 3;
+  uint32 stride_uv = 4;
+
+  uint64 data_y_ptr = 5;
+  uint64 data_uv_ptr = 6;
+}
+
+message NativeBufferInfo {
+  // TODO(theomonnom): Expose graphic context?
+}
+
+///
+/// VideoStream ///
+///
+
+enum VideoStreamType {
+  VIDEO_STREAM_NATIVE = 0;
+  VIDEO_STREAM_WEBGL = 1;
+  VIDEO_STREAM_HTML = 2;
+}
+
+message VideoStreamInfo {
+  FFIHandleId handle = 1;
+  VideoStreamType type = 2;
+  string track_sid = 3;
+}
+
+message VideoStreamEvent {
+  FFIHandleId handle = 1;
+  oneof message { VideoFrameReceived frame_received = 2; }
+}
+
+message VideoFrameReceived {
+  VideoFrameInfo frame = 1;
+  VideoFrameBufferInfo buffer = 2;
+}
+
+///
+/// VideoSource ///
+///
+
+enum VideoSourceType {
+    VIDEO_SOURCE_NATIVE = 0;
+}
+
+message VideoSourceInfo {
+  // # SAFETY
+  // This handle must not be dropped if a track is currently using it
+  FFIHandleId handle = 1;
+  VideoSourceType type = 2;
+}
+