feat: add ffi datachannel & mute events (#88)

webrtc-sys/include/livekit/audio_track.h

@@ -18,6 +18,7 @@
 
 #include <memory>
 
+#include "api/audio_options.h"
 #include "livekit/helper.h"
 #include "livekit/media_stream_track.h"
 #include "livekit/webrtc.h"
@@ -78,7 +79,7 @@ std::shared_ptr<NativeAudioSink> new_native_audio_sink(
 class AudioTrackSource {
   class InternalSource : public webrtc::LocalAudioSource {
    public:
-    InternalSource();
+    InternalSource(const cricket::AudioOptions& options);
 
     SourceState state() const override;
     bool remote() const override;
@@ -88,6 +89,8 @@ class AudioTrackSource {
     void AddSink(webrtc::AudioTrackSinkInterface* sink) override;
     void RemoveSink(webrtc::AudioTrackSinkInterface* sink) override;
 
+    void set_options(const cricket::AudioOptions& options);
+
     // AudioFrame should always contain 10 ms worth of data (see index.md of
     // acm)
     void on_captured_frame(rust::Slice<const int16_t> audio_data,
@@ -96,13 +99,17 @@ class AudioTrackSource {
                            size_t number_of_frames);
 
    private:
-    webrtc::Mutex mutex_;
+    mutable webrtc::Mutex mutex_;
     std::vector<webrtc::AudioTrackSinkInterface*> sinks_;
     cricket::AudioOptions options_{};
   };
 
  public:
-  AudioTrackSource();
+  AudioTrackSource(AudioSourceOptions options);
+
+  AudioSourceOptions audio_options() const;
+
+  void set_audio_options(const AudioSourceOptions& options) const;
 
   void on_captured_frame(rust::Slice<const int16_t> audio_data,
                          int sample_rate,
@@ -115,7 +122,8 @@ class AudioTrackSource {
   rtc::scoped_refptr<InternalSource> source_;
 };
 
-std::shared_ptr<AudioTrackSource> new_audio_track_source();
+std::shared_ptr<AudioTrackSource> new_audio_track_source(
+    AudioSourceOptions options);
 
 static std::shared_ptr<MediaStreamTrack> audio_to_media(
     std::shared_ptr<AudioTrack> track) {

webrtc-sys/include/livekit/video_track.h

@@ -87,23 +87,29 @@ std::shared_ptr<NativeVideoSink> new_native_video_sink(
 class VideoTrackSource {
   class InternalSource : public rtc::AdaptedVideoTrackSource {
    public:
-    InternalSource();
+    InternalSource(const VideoResolution&
+                       resolution);  // (0, 0) means no resolution/optional, the
+                                     // source will guess the resolution at the
+                                     // first captured frame
     ~InternalSource() override;
 
     bool is_screencast() const override;
     absl::optional<bool> needs_denoising() const override;
     SourceState state() const override;
     bool remote() const override;
-
+    VideoResolution video_resolution() const;
     bool on_captured_frame(const webrtc::VideoFrame& frame);
 
    private:
-    webrtc::Mutex mutex_;
+    mutable webrtc::Mutex mutex_;
     rtc::TimestampAligner timestamp_aligner_;
+    VideoResolution resolution_;
   };
 
  public:
-  VideoTrackSource();
+  VideoTrackSource(const VideoResolution& resolution);
+
+  VideoResolution video_resolution() const;
 
   bool on_captured_frame(const std::unique_ptr<VideoFrame>& frame)
       const;  // frames pushed from Rust (+interior mutability)
@@ -114,7 +120,8 @@ class VideoTrackSource {
   rtc::scoped_refptr<InternalSource> source_;
 };
 
-std::shared_ptr<VideoTrackSource> new_video_track_source();
+std::shared_ptr<VideoTrackSource> new_video_track_source(
+    const VideoResolution& resolution);
 
 static std::shared_ptr<MediaStreamTrack> video_to_media(
     std::shared_ptr<VideoTrack> track) {

webrtc-sys/src/audio_track.cpp

@@ -20,6 +20,7 @@
 #include <iostream>
 #include <memory>
 
+#include "api/audio_options.h"
 #include "api/media_stream_interface.h"
 #include "audio/remix_resample.h"
 #include "common_audio/include/audio_util.h"
@@ -28,9 +29,28 @@
 #include "rtc_base/synchronization/mutex.h"
 #include "rtc_base/time_utils.h"
 #include "rust/cxx.h"
+#include "webrtc-sys/src/audio_track.rs.h"
 
 namespace livekit {
 
+inline cricket::AudioOptions to_native_audio_options(
+    const AudioSourceOptions& options) {
+  cricket::AudioOptions rtc_options{};
+  rtc_options.echo_cancellation = options.echo_cancellation;
+  rtc_options.noise_suppression = options.noise_suppression;
+  rtc_options.auto_gain_control = options.auto_gain_control;
+  return rtc_options;
+}
+
+inline AudioSourceOptions to_rust_audio_options(
+    const cricket::AudioOptions& rtc_options) {
+  AudioSourceOptions options{};
+  options.echo_cancellation = rtc_options.echo_cancellation.value_or(false);
+  options.noise_suppression = rtc_options.noise_suppression.value_or(false);
+  options.auto_gain_control = rtc_options.auto_gain_control.value_or(false);
+  return options;
+}
+
 AudioTrack::AudioTrack(std::shared_ptr<RtcRuntime> rtc_runtime,
                        rtc::scoped_refptr<webrtc::AudioTrackInterface> track)
     : MediaStreamTrack(rtc_runtime, std::move(track)) {}
@@ -74,11 +94,8 @@ std::shared_ptr<NativeAudioSink> new_native_audio_sink(
   return std::make_shared<NativeAudioSink>(std::move(observer));
 }
 
-AudioTrackSource::InternalSource::InternalSource() {
-  options_.echo_cancellation = false;
-  options_.auto_gain_control = false;
-  options_.noise_suppression = false;
-}
+AudioTrackSource::InternalSource::InternalSource(
+    const cricket::AudioOptions& options) {}
 
 webrtc::MediaSourceInterface::SourceState
 AudioTrackSource::InternalSource::state() const {
@@ -90,9 +107,16 @@ bool AudioTrackSource::InternalSource::remote() const {
 }
 
 const cricket::AudioOptions AudioTrackSource::InternalSource::options() const {
+  webrtc::MutexLock lock(&mutex_);
   return options_;
 }
 
+void AudioTrackSource::InternalSource::set_options(
+    const cricket::AudioOptions& options) {
+  webrtc::MutexLock lock(&mutex_);
+  options_ = options;
+}
+
 void AudioTrackSource::InternalSource::AddSink(
     webrtc::AudioTrackSinkInterface* sink) {
   webrtc::MutexLock lock(&mutex_);
@@ -117,8 +141,18 @@ void AudioTrackSource::InternalSource::on_captured_frame(
   }
 }
 
-AudioTrackSource::AudioTrackSource() {
-  source_ = rtc::make_ref_counted<InternalSource>();
+AudioTrackSource::AudioTrackSource(AudioSourceOptions options) {
+  source_ =
+      rtc::make_ref_counted<InternalSource>(to_native_audio_options(options));
+}
+
+AudioSourceOptions AudioTrackSource::audio_options() const {
+  return to_rust_audio_options(source_->options());
+}
+
+void AudioTrackSource::set_audio_options(
+    const AudioSourceOptions& options) const {
+  source_->set_options(to_native_audio_options(options));
 }
 
 void AudioTrackSource::on_captured_frame(rust::Slice<const int16_t> audio_data,
@@ -134,8 +168,9 @@ rtc::scoped_refptr<AudioTrackSource::InternalSource> AudioTrackSource::get()
   return source_;
 }
 
-std::shared_ptr<AudioTrackSource> new_audio_track_source() {
-  return std::make_shared<AudioTrackSource>();
+std::shared_ptr<AudioTrackSource> new_audio_track_source(
+    AudioSourceOptions options) {
+  return std::make_shared<AudioTrackSource>(options);
 }
 
 }  // namespace livekit

webrtc-sys/src/audio_track.rs

@@ -3,6 +3,13 @@ use std::sync::Arc;
 
 #[cxx::bridge(namespace = "livekit")]
 pub mod ffi {
+
+    pub struct AudioSourceOptions {
+        pub echo_cancellation: bool,
+        pub noise_suppression: bool,
+        pub auto_gain_control: bool,
+    }
+
     extern "C++" {
         include!("livekit/media_stream_track.h");
 
@@ -27,7 +34,9 @@ pub mod ffi {
             nb_channels: usize,
             nb_frames: usize,
         );
-        fn new_audio_track_source() -> SharedPtr<AudioTrackSource>;
+        fn audio_options(self: &AudioTrackSource) -> AudioSourceOptions;
+        fn set_audio_options(self: &AudioTrackSource, options: &AudioSourceOptions);
+        fn new_audio_track_source(options: AudioSourceOptions) -> SharedPtr<AudioTrackSource>;
 
         fn audio_to_media(track: SharedPtr<AudioTrack>) -> SharedPtr<MediaStreamTrack>;
         unsafe fn media_to_audio(track: SharedPtr<MediaStreamTrack>) -> SharedPtr<AudioTrack>;

webrtc-sys/src/video_track.cpp

@@ -31,6 +31,7 @@
 #include "rtc_base/ref_counted_object.h"
 #include "rtc_base/synchronization/mutex.h"
 #include "rtc_base/time_utils.h"
+#include "webrtc-sys/src/video_track.rs.h"
 
 namespace livekit {
 
@@ -103,8 +104,9 @@ std::shared_ptr<NativeVideoSink> new_native_video_sink(
   return std::make_shared<NativeVideoSink>(std::move(observer));
 }
 
-VideoTrackSource::InternalSource::InternalSource()
-    : rtc::AdaptedVideoTrackSource(4) {}
+VideoTrackSource::InternalSource::InternalSource(
+    const VideoResolution& resolution)
+    : rtc::AdaptedVideoTrackSource(4), resolution_(resolution) {}
 
 VideoTrackSource::InternalSource::~InternalSource() {}
 
@@ -125,6 +127,11 @@ bool VideoTrackSource::InternalSource::remote() const {
   return false;
 }
 
+VideoResolution VideoTrackSource::InternalSource::video_resolution() const {
+  webrtc::MutexLock lock(&mutex_);
+  return resolution_;
+}
+
 bool VideoTrackSource::InternalSource::on_captured_frame(
     const webrtc::VideoFrame& frame) {
   webrtc::MutexLock lock(&mutex_);
@@ -135,6 +142,11 @@ bool VideoTrackSource::InternalSource::on_captured_frame(
   rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer =
       frame.video_frame_buffer();
 
+  if (resolution_.height == 0 || resolution_.width == 0) {
+    resolution_ = VideoResolution{static_cast<uint32_t>(buffer->width()),
+                                  static_cast<uint32_t>(buffer->height())};
+  }
+
   int adapted_width, adapted_height, crop_width, crop_height, crop_x, crop_y;
   if (!AdaptFrame(buffer->width(), buffer->height(), aligned_timestamp_us,
                   &adapted_width, &adapted_height, &crop_width, &crop_height,
@@ -163,16 +175,17 @@ bool VideoTrackSource::InternalSource::on_captured_frame(
   return true;
 }
 
-VideoTrackSource::VideoTrackSource() {
-  source_ = rtc::make_ref_counted<InternalSource>();
+VideoTrackSource::VideoTrackSource(const VideoResolution& resolution) {
+  source_ = rtc::make_ref_counted<InternalSource>(resolution);
+}
+
+VideoResolution VideoTrackSource::video_resolution() const {
+  return source_->video_resolution();
 }
 
 bool VideoTrackSource::on_captured_frame(
     const std::unique_ptr<VideoFrame>& frame) const {
   auto rtc_frame = frame->get();
-  rtc_frame.set_timestamp_us(
-      rtc::TimeMicros());  // TODO(theomonnom): Expore capture ts to Rust
-
   return source_->on_captured_frame(rtc_frame);
 }
 
@@ -181,8 +194,9 @@ rtc::scoped_refptr<VideoTrackSource::InternalSource> VideoTrackSource::get()
   return source_;
 }
 
-std::shared_ptr<VideoTrackSource> new_video_track_source() {
-  return std::make_shared<VideoTrackSource>();
+std::shared_ptr<VideoTrackSource> new_video_track_source(
+    const VideoResolution& resolution) {
+  return std::make_shared<VideoTrackSource>(resolution);
 }
 
 }  // namespace livekit

webrtc-sys/src/video_track.rs

@@ -21,6 +21,12 @@ pub mod ffi {
         pub max_fps: f64,
     }
 
+    #[derive(Debug)]
+    pub struct VideoResolution {
+        pub width: u32,
+        pub height: u32,
+    }
+
     extern "C++" {
         include!("livekit/video_frame.h");
         include!("livekit/media_stream_track.h");
@@ -44,9 +50,9 @@ pub mod ffi {
         fn set_content_hint(self: &VideoTrack, hint: ContentHint);
         fn new_native_video_sink(observer: Box<VideoSinkWrapper>) -> SharedPtr<NativeVideoSink>;
 
+        fn video_resolution(self: &VideoTrackSource) -> VideoResolution;
         fn on_captured_frame(self: &VideoTrackSource, frame: &UniquePtr<VideoFrame>) -> bool;
-        fn new_video_track_source() -> SharedPtr<VideoTrackSource>;
-
+        fn new_video_track_source(resolution: &VideoResolution) -> SharedPtr<VideoTrackSource>;
         fn video_to_media(track: SharedPtr<VideoTrack>) -> SharedPtr<MediaStreamTrack>;
         unsafe fn media_to_video(track: SharedPtr<MediaStreamTrack>) -> SharedPtr<VideoTrack>;
         fn _shared_video_track() -> SharedPtr<VideoTrack>;