Initial downstream tracks

2022-11-22 17:59:55 +01:00
parent 82ec808dee
commit ae776f04d0
49 changed files with 1910 additions and 406 deletions
@@ -13,26 +13,7 @@

 namespace livekit {

-class MediaStreamTrack {
- public:
-  explicit MediaStreamTrack(
-      rtc::scoped_refptr<webrtc::MediaStreamTrackInterface> track);
-
-  rust::String kind() const;
-  rust::String id() const;
-
-  bool enabled() const;
-  bool set_enabled(bool enable);
-
-  TrackState state() const;
-
- private:
-  rtc::scoped_refptr<webrtc::MediaStreamTrackInterface> track_;
-};
-
-static std::unique_ptr<MediaStreamTrack> _unique_media_stream_track() {
-  return nullptr;  // Ignore
-}
+class NativeVideoFrameSink;

 class MediaStream {
 public:
@@ -47,6 +28,96 @@ class MediaStream {
 static std::unique_ptr<MediaStream> _unique_media_stream() {
  return nullptr;  // Ignore
 }
+
+class MediaStreamTrack {
+ protected:
+  explicit MediaStreamTrack(
+      rtc::scoped_refptr<webrtc::MediaStreamTrackInterface> track);
+
+ public:
+  static std::unique_ptr<MediaStreamTrack> from(
+      rtc::scoped_refptr<webrtc::MediaStreamTrackInterface> track);
+
+  rust::String kind() const;
+  rust::String id() const;
+
+  bool enabled() const;
+  bool set_enabled(bool enable);
+
+  TrackState state() const;
+
+ protected:
+  rtc::scoped_refptr<webrtc::MediaStreamTrackInterface> track_;
+};
+
+static std::unique_ptr<MediaStreamTrack> _unique_media_stream_track() {
+  return nullptr;  // Ignore
+}
+
+class AudioTrack : public MediaStreamTrack {
+ public:
+  explicit AudioTrack(rtc::scoped_refptr<webrtc::AudioTrackInterface> track);
+};
+
+static std::unique_ptr<AudioTrack> _unique_audio_track() {
+  return nullptr;  // Ignore
+}
+
+class VideoTrack : public MediaStreamTrack {
+ public:
+  explicit VideoTrack(rtc::scoped_refptr<webrtc::VideoTrackInterface> track);
+
+  void add_sink(NativeVideoFrameSink& sink);
+  void remove_sink(NativeVideoFrameSink& sink);
+
+  void set_should_receive(bool should_receive);
+  bool should_receive() const;
+  ContentHint content_hint() const;
+  void set_content_hint(ContentHint hint);
+
+ private:
+  webrtc::VideoTrackInterface* track() const {
+    return static_cast<webrtc::VideoTrackInterface*>(track_.get());
+  }
+};
+
+static std::unique_ptr<VideoTrack> _unique_video_track() {
+  return nullptr;  // Ignore
+}
+
+class NativeVideoFrameSink
+    : public rtc::VideoSinkInterface<webrtc::VideoFrame> {
+ public:
+  explicit NativeVideoFrameSink(rust::Box<VideoFrameSinkWrapper> observer);
+
+  void OnFrame(const webrtc::VideoFrame& frame) override;
+  void OnDiscardedFrame() override;
+  void OnConstraintsChanged(
+      const webrtc::VideoTrackSourceConstraints& constraints) override;
+
+ private:
+  rust::Box<VideoFrameSinkWrapper> observer_;
+};
+
+std::unique_ptr<NativeVideoFrameSink> create_native_video_frame_sink(
+    rust::Box<VideoFrameSinkWrapper> observer);
+
+const MediaStreamTrack* video_to_media(const VideoTrack* track) {
+  return track;
+}
+
+const MediaStreamTrack* audio_to_media(const AudioTrack* track) {
+  return track;
+}
+
+const VideoTrack* media_to_video(const MediaStreamTrack* track) {
+  return static_cast<const VideoTrack*>(track);
+}
+
+const AudioTrack* media_to_audio(const MediaStreamTrack* track) {
+  return static_cast<const AudioTrack*>(track); 
+}
+
 }  // namespace livekit

 #endif  // CLIENT_SDK_NATIVE_MEDIA_STREAM_INTERFACE_H
@@ -15,6 +15,7 @@ struct SetLocalSdpObserverWrapper;
 struct SetRemoteSdpObserverWrapper;
 struct DataChannelObserverWrapper;
 struct AddIceCandidateObserverWrapper;
+struct VideoFrameSinkWrapper;

 // Shared types
 enum class PeerConnectionState;
@@ -24,6 +25,9 @@ enum class IceGatheringState;
 enum class SdpType;
 enum class DataState;
 enum class TrackState;
+enum class ContentHint;
+enum class VideoRotation;
+enum class VideoFrameBufferType;
 struct SdpParseError;
 struct RTCOfferAnswerOptions;
 struct RTCError;
@@ -0,0 +1,48 @@
+//
+// Created by theom on 14/11/2022.
+//
+
+#ifndef LIVEKIT_WEBRTC_VIDEO_FRAME_H
+#define LIVEKIT_WEBRTC_VIDEO_FRAME_H
+
+#include "api/video/video_frame.h"
+#include "livekit/rust_types.h"
+#include "livekit/video_frame_buffer.h"
+
+namespace livekit {
+
+class VideoFrame {
+ public:
+  explicit VideoFrame(const webrtc::VideoFrame& frame)
+      : frame_(std::move(frame)) {}
+
+  int width() const { return frame_.width(); }
+  int height() const { return frame_.height(); }
+  uint32_t size() const { return frame_.size(); }
+  uint16_t id() const { return frame_.id(); }
+  int64_t timestamp_us() const { return frame_.timestamp_us(); }
+  int64_t ntp_time_ms() const { return frame_.ntp_time_ms(); }
+  uint32_t transport_frame_id() const { return frame_.transport_frame_id(); }
+  uint32_t timestamp() const { return frame_.timestamp(); }
+
+  VideoRotation rotation() const {
+    return static_cast<VideoRotation>(frame_.rotation());
+  }
+
+  // TODO(theomonnom) This shouldn't create a new shared_ptr at each call
+  std::shared_ptr<VideoFrameBuffer> video_frame_buffer() const {
+    return std::make_shared<VideoFrameBuffer>(frame_.video_frame_buffer());
+  }
+
+ private:
+  webrtc::VideoFrame frame_;
+};
+
+static std::unique_ptr<VideoFrame> _unique_video_frame() {
+  return nullptr;  // Ignore
+}
+
+
+}  // namespace livekit
+
+#endif  // LIVEKIT_WEBRTC_VIDEO_FRAME_H
@@ -0,0 +1,93 @@
+//
+// Created by theom on 14/11/2022.
+//
+
+#ifndef LIVEKIT_WEBRTC_VIDEO_FRAME_BUFFER_H
+#define LIVEKIT_WEBRTC_VIDEO_FRAME_BUFFER_H
+
+#include "api/video/video_frame_buffer.h"
+#include "rust_types.h"
+
+namespace livekit {
+
+class PlanarYuvBuffer;
+class PlanarYuv8Buffer;
+class I420Buffer;
+
+class VideoFrameBuffer {
+ public:
+  explicit VideoFrameBuffer(rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer)
+      : buffer_(std::move(buffer)) {}
+
+  VideoFrameBufferType buffer_type() const {
+    return static_cast<VideoFrameBufferType>(buffer_->type());
+  }
+
+  int width() const { return buffer_->width(); }
+  int height() const { return buffer_->height(); }
+
+  std::shared_ptr<I420Buffer> to_i420() {
+    return std::make_shared<I420Buffer>(buffer_->ToI420());
+  }
+
+ protected:
+  rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer_;
+};
+
+class PlanarYuvBuffer : public VideoFrameBuffer {
+ public:
+  explicit PlanarYuvBuffer(rtc::scoped_refptr<webrtc::PlanarYuvBuffer> buffer)
+      : VideoFrameBuffer(buffer) {}
+
+  int chroma_width() const { return buffer()->ChromaWidth(); }
+  int chroma_height() const { return buffer()->ChromaHeight(); }
+
+  int stride_y() const { return buffer()->StrideY(); }
+  int stride_u() const { return buffer()->StrideU(); }
+  int stride_v() const { return buffer()->StrideV(); }
+
+ private:
+  webrtc::PlanarYuvBuffer* buffer() const {
+    return static_cast<webrtc::PlanarYuvBuffer*>(buffer_.get());
+  }
+};
+
+class PlanarYuv8Buffer : public PlanarYuvBuffer {
+ public:
+  explicit PlanarYuv8Buffer(rtc::scoped_refptr<webrtc::PlanarYuv8Buffer> buffer)
+      : PlanarYuvBuffer(buffer) {}
+
+  const uint8_t* data_y() const { return buffer()->DataY(); }
+  const uint8_t* data_u() const { return buffer()->DataU(); }
+  const uint8_t* data_v() const { return buffer()->DataV(); }
+
+ private:
+  webrtc::PlanarYuv8Buffer* buffer() const {
+    return static_cast<webrtc::PlanarYuv8Buffer*>(buffer_.get());
+  }
+};
+
+class I420Buffer : public PlanarYuv8Buffer {
+ public:
+  explicit I420Buffer(rtc::scoped_refptr<webrtc::I420BufferInterface> buffer)
+      : PlanarYuv8Buffer(buffer) {}
+};
+
+std::shared_ptr<VideoFrameBuffer> to_video_frame_buffer(
+    std::shared_ptr<PlanarYuvBuffer> buffer) {
+  return buffer;
+}
+
+std::shared_ptr<PlanarYuvBuffer> to_yuv_buffer(
+    std::shared_ptr<PlanarYuv8Buffer> buffer) {
+  return buffer;
+}
+
+std::shared_ptr<PlanarYuv8Buffer> to_yuv8_buffer(
+    std::shared_ptr<I420Buffer> buffer) {
+  return buffer;
+}
+
+}  // namespace livekit
+
+#endif  // LIVEKIT_WEBRTC_VIDEO_FRAME_BUFFER_H