client-sdk-rust/livekit-ffi/protocol/video_frame.proto

syntax = "proto3";

package livekit;
option csharp_namespace = "LiveKit.Proto";

import "handle.proto";

// Allocate a new VideoFrameBuffer
message AllocVideoBufferRequest {
  VideoFrameBufferType type = 1; // Only I420 is supported atm
  uint32 width = 2;
  uint32 height = 3;
}
message AllocVideoBufferResponse { VideoFrameBufferInfo buffer = 1; }

// Create a new VideoStream
// VideoStream is used to receive video frames from a track
message NewVideoStreamRequest {
  FfiHandleId track_handle = 1;
  VideoStreamType type = 2;
}
message NewVideoStreamResponse { VideoStreamInfo stream = 1; }

// Create a new VideoSource
// VideoSource is used to send video frame to a track
message NewVideoSourceRequest { 
  VideoSourceType type = 1; 
  // Used to determine which encodings to use + simulcast layers
  // Most of the time it corresponds to the source resolution 
  optional VideoSourceResolution resolution = 2;
}
message NewVideoSourceResponse { VideoSourceInfo source = 1; }

// Push a frame to a VideoSource
message CaptureVideoFrameRequest {
  FfiHandleId source_handle = 1;
  VideoFrameInfo frame = 2;
  FfiHandleId buffer_handle = 3;
}
message CaptureVideoFrameResponse {}

// Convert a RGBA frame to a I420 YUV frame
// Or convert another YUV frame format to I420
message ToI420Request {
  bool flip_y = 1;
  oneof from {
    ARGBBufferInfo argb = 2;
    FfiHandleId buffer = 3;
  }
}
message ToI420Response { VideoFrameBufferInfo buffer = 1; }

// Convert a YUV frame to a RGBA frame
// Only I420 is supported atm
message ToArgbRequest {
  FfiHandleId buffer = 1;
  uint64 dst_ptr = 2;
  VideoFormatType dst_format = 3;
  uint32 dst_stride = 4;
  uint32 dst_width = 5;
  uint32 dst_height = 6;
  bool flip_y = 7;
}
message ToArgbResponse {}

///
/// VideoFrame buffers ///
///

message VideoResolution {
  uint32 width = 1;
  uint32 height = 2;
  double frame_rate = 3;
}

enum VideoCodec {
  VP8 = 0;
  H264 = 1;
  AV1 = 2;
}

enum VideoRotation {
  VIDEO_ROTATION_0 = 0;
  VIDEO_ROTATION_90 = 1;
  VIDEO_ROTATION_180 = 2;
  VIDEO_ROTATION_270 = 3;
}

enum VideoFormatType {
  FORMAT_ARGB = 0;
  FORMAT_BGRA = 1;
  FORMAT_ABGR = 2;
  FORMAT_RGBA = 3;
}

enum VideoFrameBufferType {
  NATIVE = 0;
  I420 = 1;
  I420A = 2;
  I422 = 3;
  I444 = 4;
  I010 = 5;
  NV12 = 6;
  WEBGL = 7;
}

message ARGBBufferInfo {
  uint64 ptr = 1;
  VideoFormatType format = 2;
  uint32 stride = 3;
  uint32 width = 4;
  uint32 height = 5;
}

message VideoFrameInfo {
  int64 timestamp_us = 1; // In microseconds
  VideoRotation rotation = 2;
}

message VideoFrameBufferInfo {
  FfiHandleId handle = 1;
  VideoFrameBufferType buffer_type = 2;
  uint32 width = 3;
  uint32 height = 4;
  oneof buffer {
    PlanarYuvBufferInfo yuv = 5;
    BiplanarYuvBufferInfo bi_yuv = 6;
    NativeBufferInfo native = 7;
  }
}

message PlanarYuvBufferInfo {
  uint32 chroma_width = 1;
  uint32 chroma_height = 2;
  uint32 stride_y = 3;
  uint32 stride_u = 4;
  uint32 stride_v = 5;
  uint32 stride_a = 6;

  // *const u8 or *const u16
  uint64 data_y_ptr = 7;
  uint64 data_u_ptr = 8;
  uint64 data_v_ptr = 9;
  uint64 data_a_ptr = 10; // nullptr = no alpha
}

message BiplanarYuvBufferInfo {
  uint32 chroma_width = 1;
  uint32 chroma_height = 2;
  uint32 stride_y = 3;
  uint32 stride_uv = 4;

  uint64 data_y_ptr = 5;
  uint64 data_uv_ptr = 6;
}

message NativeBufferInfo {
  // TODO(theomonnom): Expose graphic context?
}

///
/// VideoStream ///
///

enum VideoStreamType {
  VIDEO_STREAM_NATIVE = 0;
  VIDEO_STREAM_WEBGL = 1;
  VIDEO_STREAM_HTML = 2;
}

message VideoStreamInfo {
  FfiHandleId handle = 1;
  VideoStreamType type = 2;
}

message VideoStreamEvent {
  FfiHandleId handle = 1;
  oneof message { VideoFrameReceived frame_received = 2; }
}

message VideoFrameReceived {
  VideoFrameInfo frame = 1;
  VideoFrameBufferInfo buffer = 2;
}

///
/// VideoSource ///
///

message VideoSourceResolution {
  uint32 width = 1;
  uint32 height = 2;
}

enum VideoSourceType {
    VIDEO_SOURCE_NATIVE = 0;
}

message VideoSourceInfo {
  // # SAFETY
  // This handle must not be dropped if a track is currently using it
  FfiHandleId handle = 1;
  VideoSourceType type = 2;
}