feat: audio support (#45)

- Receive/Send audio frames
- LocalAudioTrack

webrtc-sys/build.rs

@@ -153,6 +153,7 @@ fn main() {
     builder.file("src/video_frame_buffer.cpp");
     builder.file("src/video_encoder_factory.cpp");
     builder.file("src/video_decoder_factory.cpp");
+    builder.file("src/audio_device.cpp");
 
     for include in includes {
         builder.include(include);

webrtc-sys/include/livekit/audio_device.h

@@ -0,0 +1,129 @@
+/*
+ * Copyright 2023 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the “License”);
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an “AS IS” BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <atomic>
+
+#include "api/task_queue/task_queue_factory.h"
+#include "modules/audio_device/include/audio_device.h"
+#include "rtc_base/synchronization/mutex.h"
+#include "rtc_base/task_queue.h"
+#include "rtc_base/task_utils/repeating_task.h"
+
+namespace livekit {
+
+class AudioDevice : public webrtc::AudioDeviceModule {
+ public:
+  AudioDevice(webrtc::TaskQueueFactory* task_queue_factory);
+  ~AudioDevice() override;
+
+  int32_t ActiveAudioLayer(AudioLayer* audioLayer) const override;
+  int32_t RegisterAudioCallback(webrtc::AudioTransport* transport) override;
+
+  int32_t Init() override;
+  int32_t Terminate() override;
+  bool Initialized() const override;
+
+  int16_t PlayoutDevices() override;
+  int16_t RecordingDevices() override;
+  int32_t PlayoutDeviceName(uint16_t index,
+                            char name[webrtc::kAdmMaxDeviceNameSize],
+                            char guid[webrtc::kAdmMaxGuidSize]) override;
+
+  int32_t RecordingDeviceName(uint16_t index,
+                              char name[webrtc::kAdmMaxDeviceNameSize],
+                              char guid[webrtc::kAdmMaxGuidSize]) override;
+
+  int32_t SetPlayoutDevice(uint16_t index) override;
+  int32_t SetPlayoutDevice(WindowsDeviceType device) override;
+  int32_t SetRecordingDevice(uint16_t index) override;
+  int32_t SetRecordingDevice(WindowsDeviceType device) override;
+
+  int32_t PlayoutIsAvailable(bool* available) override;
+  int32_t InitPlayout() override;
+  bool PlayoutIsInitialized() const override;
+  int32_t RecordingIsAvailable(bool* available) override;
+  int32_t InitRecording() override;
+  bool RecordingIsInitialized() const override;
+
+  int32_t StartPlayout() override;
+  int32_t StopPlayout() override;
+  bool Playing() const override;
+  int32_t StartRecording() override;
+  int32_t StopRecording() override;
+  bool Recording() const override;
+
+  int32_t InitSpeaker() override;
+  bool SpeakerIsInitialized() const override;
+  int32_t InitMicrophone() override;
+  bool MicrophoneIsInitialized() const override;
+
+  int32_t SpeakerVolumeIsAvailable(bool* available) override;
+  int32_t SetSpeakerVolume(uint32_t volume) override;
+  int32_t SpeakerVolume(uint32_t* volume) const override;
+  int32_t MaxSpeakerVolume(uint32_t* maxVolume) const override;
+  int32_t MinSpeakerVolume(uint32_t* minVolume) const override;
+
+  int32_t MicrophoneVolumeIsAvailable(bool* available) override;
+  int32_t SetMicrophoneVolume(uint32_t volume) override;
+  int32_t MicrophoneVolume(uint32_t* volume) const override;
+  int32_t MaxMicrophoneVolume(uint32_t* maxVolume) const override;
+  int32_t MinMicrophoneVolume(uint32_t* minVolume) const override;
+
+  int32_t SpeakerMuteIsAvailable(bool* available) override;
+  int32_t SetSpeakerMute(bool enable) override;
+  int32_t SpeakerMute(bool* enabled) const override;
+
+  int32_t MicrophoneMuteIsAvailable(bool* available) override;
+  int32_t SetMicrophoneMute(bool enable) override;
+  int32_t MicrophoneMute(bool* enabled) const override;
+
+  int32_t StereoPlayoutIsAvailable(bool* available) const override;
+  int32_t SetStereoPlayout(bool enable) override;
+  int32_t StereoPlayout(bool* enabled) const override;
+  int32_t StereoRecordingIsAvailable(bool* available) const override;
+  int32_t SetStereoRecording(bool enable) override;
+  int32_t StereoRecording(bool* enabled) const override;
+
+  int32_t PlayoutDelay(uint16_t* delayMS) const override;
+
+  bool BuiltInAECIsAvailable() const override;
+  bool BuiltInAGCIsAvailable() const override;
+  bool BuiltInNSIsAvailable() const override;
+
+  int32_t EnableBuiltInAEC(bool enable) override;
+  int32_t EnableBuiltInAGC(bool enable) override;
+  int32_t EnableBuiltInNS(bool enable) override;
+
+#if defined(WEBRTC_IOS)
+  int GetPlayoutAudioParameters(AudioParameters* params) const override;
+  int GetRecordAudioParameters(AudioParameters* params) const override;
+#endif  // WEBRTC_IOS
+
+  int32_t SetAudioDeviceSink(webrtc::AudioDeviceSink* sink) const override;
+
+ private:
+  mutable webrtc::Mutex mutex_;
+  webrtc::TaskQueueFactory* task_queue_factory_;
+  std::unique_ptr<rtc::TaskQueue> audio_queue_;
+  webrtc::RepeatingTaskHandle audio_task_;
+  std::vector<int16_t> data_;
+  webrtc::AudioTransport* audio_transport_;
+  std::atomic<bool> playing_{false};
+  std::atomic<bool> initialized_{false};
+};
+}  // namespace livekit

webrtc-sys/include/livekit/media_stream.h

@@ -20,9 +20,12 @@
 
 #include "api/media_stream_interface.h"
 #include "api/video/video_frame.h"
+#include "common_audio/resampler/include/push_resampler.h"
+#include "common_audio/ring_buffer.h"
 #include "livekit/helper.h"
 #include "livekit/video_frame.h"
 #include "media/base/adapted_video_track_source.h"
+#include "pc/local_audio_source.h"
 #include "rtc_base/synchronization/mutex.h"
 #include "rtc_base/timestamp_aligner.h"
 #include "rust/cxx.h"
@@ -34,6 +37,8 @@ class MediaStreamTrack;
 class VideoTrack;
 class AudioTrack;
 class NativeVideoFrameSink;
+class NativeAudioSink;
+class AudioTrackSource;
 class AdaptedVideoTrackSource;
 }  // namespace livekit
 #include "webrtc-sys/src/media_stream.rs.h"
@@ -86,8 +91,73 @@ class MediaStreamTrack {
 class AudioTrack : public MediaStreamTrack {
  public:
   explicit AudioTrack(rtc::scoped_refptr<webrtc::AudioTrackInterface> track);
+
+  void add_sink(NativeAudioSink& sink) const;
+  void remove_sink(NativeAudioSink& sink) const;
+
+ private:
+  webrtc::AudioTrackInterface* track() const {
+    return static_cast<webrtc::AudioTrackInterface*>(track_.get());
+  }
 };
 
+class NativeAudioSink : public webrtc::AudioTrackSinkInterface {
+ public:
+  explicit NativeAudioSink(rust::Box<AudioSinkWrapper> observer);
+  void OnData(const void* audio_data,
+              int bits_per_sample,
+              int sample_rate,
+              size_t number_of_channels,
+              size_t number_of_frames) override;
+
+ private:
+  rust::Box<AudioSinkWrapper> observer_;
+};
+
+std::unique_ptr<NativeAudioSink> new_native_audio_sink(
+    rust::Box<AudioSinkWrapper> observer);
+
+class NativeAudioTrackSource : public webrtc::LocalAudioSource {
+ public:
+  NativeAudioTrackSource();
+
+  SourceState state() const override;
+  bool remote() const override;
+
+  const cricket::AudioOptions options() const override;
+
+  void AddSink(webrtc::AudioTrackSinkInterface* sink) override;
+  void RemoveSink(webrtc::AudioTrackSinkInterface* sink) override;
+
+  // AudioFrame should always contain 10 ms worth of data (see index.md of acm)
+  void on_captured_frame(const int16_t* audio_data,
+                         int sample_rate,
+                         size_t number_of_channels,
+                         size_t number_of_frames);
+
+ private:
+  webrtc::Mutex mutex_;
+  std::vector<webrtc::AudioTrackSinkInterface*> sinks_;
+  cricket::AudioOptions options_{};
+};
+
+class AudioTrackSource {
+ public:
+  AudioTrackSource(rtc::scoped_refptr<NativeAudioTrackSource> source);
+
+  void on_captured_frame(const int16_t* audio_data,
+                         int sample_rate,
+                         size_t number_of_channels,
+                         size_t number_of_frames) const;
+
+  rtc::scoped_refptr<NativeAudioTrackSource> get() const;
+
+ private:
+  rtc::scoped_refptr<NativeAudioTrackSource> source_;
+};
+
+std::shared_ptr<AudioTrackSource> new_audio_track_source();
+
 class VideoTrack : public MediaStreamTrack {
  public:
   explicit VideoTrack(rtc::scoped_refptr<webrtc::VideoTrackInterface> track);
@@ -131,7 +201,7 @@ class NativeVideoTrackSource : public rtc::AdaptedVideoTrackSource {
 
   bool is_screencast() const override;
   absl::optional<bool> needs_denoising() const override;
-  webrtc::MediaSourceInterface::SourceState state() const override;
+  SourceState state() const override;
   bool remote() const override;
 
   bool on_captured_frame(const webrtc::VideoFrame& frame);

webrtc-sys/include/livekit/peer_connection_factory.h

@@ -46,6 +46,10 @@ class PeerConnectionFactory {
       rust::String label,
       std::shared_ptr<AdaptedVideoTrackSource> source) const;
 
+  std::shared_ptr<AudioTrack> create_audio_track(
+      rust::String label,
+      std::shared_ptr<AudioTrackSource> source) const;
+
   RtpCapabilities get_rtp_sender_capabilities(MediaType type) const;
 
   RtpCapabilities get_rtp_receiver_capabilities(MediaType type) const;

webrtc-sys/src/audio_device.cpp

@@ -0,0 +1,323 @@
+/*
+ * Copyright 2023 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the “License”);
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an “AS IS” BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "livekit/audio_device.h"
+
+const int kBitsPerSample = 16;
+const int kSampleRate = 48000;
+const int kChannels = 2;
+const int kSamplesPer10Ms = kSampleRate / 100;
+
+namespace livekit {
+
+AudioDevice::AudioDevice(webrtc::TaskQueueFactory* task_queue_factory)
+    : task_queue_factory_(task_queue_factory),
+      data_(kSamplesPer10Ms * kChannels) {}
+
+AudioDevice::~AudioDevice() {
+  Terminate();
+}
+
+int32_t AudioDevice::ActiveAudioLayer(AudioLayer* audioLayer) const {
+  *audioLayer = AudioLayer::kDummyAudio;
+  return 0;
+}
+
+int32_t AudioDevice::RegisterAudioCallback(webrtc::AudioTransport* transport) {
+  webrtc::MutexLock lock(&mutex_);
+  audio_transport_ = transport;
+  return 0;
+}
+
+int32_t AudioDevice::Init() {
+  audio_queue_ =
+      std::make_unique<rtc::TaskQueue>(task_queue_factory_->CreateTaskQueue(
+          "AudioDevice", webrtc::TaskQueueFactory::Priority::NORMAL));
+
+  audio_task_ =
+      webrtc::RepeatingTaskHandle::Start(audio_queue_->Get(), [this]() {
+        webrtc::MutexLock lock(&mutex_);
+
+        if (playing_) {
+          int64_t elapsed_time_ms = -1;
+          int64_t ntp_time_ms = -1;
+          void* data = data_.data();
+
+          // Request the AudioData, otherwise WebRTC will ignore the packets.
+          // 10ms of audio data.
+          audio_transport_->PullRenderData(kBitsPerSample, kSampleRate,
+                                           kChannels, kSamplesPer10Ms, data,
+                                           &elapsed_time_ms, &ntp_time_ms);
+        }
+
+        return webrtc::TimeDelta::Millis(10);
+      });
+
+  initialized_ = true;
+  return 0;
+}
+
+int32_t AudioDevice::Terminate() {
+  if (!initialized_)
+    return 0;
+
+  initialized_ = false;
+
+  audio_queue_->PostTask([this] { audio_task_.Stop(); });
+
+  StopRecording();
+  StopPlayout();
+  return 0;
+}
+
+bool AudioDevice::Initialized() const {
+  return initialized_;
+}
+
+int16_t AudioDevice::PlayoutDevices() {
+  return 0;
+}
+
+int16_t AudioDevice::RecordingDevices() {
+  return 0;
+}
+
+int32_t AudioDevice::PlayoutDeviceName(uint16_t index,
+                                       char name[webrtc::kAdmMaxDeviceNameSize],
+                                       char guid[webrtc::kAdmMaxGuidSize]) {
+  return 0;
+}
+
+int32_t AudioDevice::RecordingDeviceName(
+    uint16_t index,
+    char name[webrtc::kAdmMaxDeviceNameSize],
+    char guid[webrtc::kAdmMaxGuidSize]) {
+  return 0;
+}
+
+int32_t AudioDevice::SetPlayoutDevice(uint16_t index) {
+  return 0;
+}
+
+int32_t AudioDevice::SetPlayoutDevice(WindowsDeviceType device) {
+  return 0;
+}
+
+int32_t AudioDevice::SetRecordingDevice(uint16_t index) {
+  return 0;
+}
+
+int32_t AudioDevice::SetRecordingDevice(WindowsDeviceType device) {
+  return 0;
+}
+
+int32_t AudioDevice::PlayoutIsAvailable(bool* available) {
+  return 0;
+}
+
+int32_t AudioDevice::InitPlayout() {
+  return 0;
+}
+
+bool AudioDevice::PlayoutIsInitialized() const {
+  return false;
+}
+
+int32_t AudioDevice::RecordingIsAvailable(bool* available) {
+  return 0;
+}
+
+int32_t AudioDevice::InitRecording() {
+  return 0;
+}
+
+bool AudioDevice::RecordingIsInitialized() const {
+  return false;
+}
+
+int32_t AudioDevice::StartPlayout() {
+  playing_ = true;
+  return 0;
+}
+
+int32_t AudioDevice::StopPlayout() {
+  playing_ = false;
+  return 0;
+}
+
+bool AudioDevice::Playing() const {
+  return false;
+}
+
+int32_t AudioDevice::StartRecording() {
+  return 0;
+}
+
+int32_t AudioDevice::StopRecording() {
+  return 0;
+}
+
+bool AudioDevice::Recording() const {
+  return false;
+}
+
+int32_t AudioDevice::InitSpeaker() {
+  return 0;
+}
+
+bool AudioDevice::SpeakerIsInitialized() const {
+  return false;
+}
+
+int32_t AudioDevice::InitMicrophone() {
+  return 0;
+}
+
+bool AudioDevice::MicrophoneIsInitialized() const {
+  return false;
+}
+
+int32_t AudioDevice::SpeakerVolumeIsAvailable(bool* available) {
+  return 0;
+}
+
+int32_t AudioDevice::SetSpeakerVolume(uint32_t volume) {
+  return 0;
+}
+
+int32_t AudioDevice::SpeakerVolume(uint32_t* volume) const {
+  return 0;
+}
+
+int32_t AudioDevice::MaxSpeakerVolume(uint32_t* maxVolume) const {
+  return 0;
+}
+
+int32_t AudioDevice::MinSpeakerVolume(uint32_t* minVolume) const {
+  return 0;
+}
+
+int32_t AudioDevice::MicrophoneVolumeIsAvailable(bool* available) {
+  return 0;
+}
+
+int32_t AudioDevice::SetMicrophoneVolume(uint32_t volume) {
+  return 0;
+}
+
+int32_t AudioDevice::MicrophoneVolume(uint32_t* volume) const {
+  return 0;
+}
+
+int32_t AudioDevice::MaxMicrophoneVolume(uint32_t* maxVolume) const {
+  return 0;
+}
+
+int32_t AudioDevice::MinMicrophoneVolume(uint32_t* minVolume) const {
+  return 0;
+}
+
+int32_t AudioDevice::SpeakerMuteIsAvailable(bool* available) {
+  return 0;
+}
+
+int32_t AudioDevice::SetSpeakerMute(bool enable) {
+  return 0;
+}
+
+int32_t AudioDevice::SpeakerMute(bool* enabled) const {
+  return 0;
+}
+
+int32_t AudioDevice::MicrophoneMuteIsAvailable(bool* available) {
+  return 0;
+}
+
+int32_t AudioDevice::SetMicrophoneMute(bool enable) {
+  return 0;
+}
+
+int32_t AudioDevice::MicrophoneMute(bool* enabled) const {
+  return 0;
+}
+
+int32_t AudioDevice::StereoPlayoutIsAvailable(bool* available) const {
+  return 0;
+}
+
+int32_t AudioDevice::SetStereoPlayout(bool enable) {
+  return 0;
+}
+
+int32_t AudioDevice::StereoPlayout(bool* enabled) const {
+  return 0;
+}
+
+int32_t AudioDevice::StereoRecordingIsAvailable(bool* available) const {
+  return 0;
+}
+
+int32_t AudioDevice::SetStereoRecording(bool enable) {
+  return 0;
+}
+
+int32_t AudioDevice::StereoRecording(bool* enabled) const {
+  return 0;
+}
+
+int32_t AudioDevice::PlayoutDelay(uint16_t* delayMS) const {
+  return 0;
+}
+
+bool AudioDevice::BuiltInAECIsAvailable() const {
+  return false;
+}
+
+bool AudioDevice::BuiltInAGCIsAvailable() const {
+  return false;
+}
+
+bool AudioDevice::BuiltInNSIsAvailable() const {
+  return false;
+}
+
+int32_t AudioDevice::EnableBuiltInAEC(bool enable) {
+  return 0;
+}
+
+int32_t AudioDevice::EnableBuiltInAGC(bool enable) {
+  return 0;
+}
+
+int32_t AudioDevice::EnableBuiltInNS(bool enable) {
+  return 0;
+}
+
+#if defined(WEBRTC_IOS)
+int AudioDevice::GetPlayoutAudioParameters(AudioParameters* params) const {
+  return 0;
+}
+
+int AudioDevice::GetRecordAudioParameters(AudioParameters* params) const {
+  return 0;
+}
+#endif  // WEBRTC_IOS
+
+int32_t AudioDevice::SetAudioDeviceSink(webrtc::AudioDeviceSink* sink) const {
+  return 0;
+}
+
+}  // namespace livekit

webrtc-sys/src/media_stream.cpp

@@ -23,6 +23,8 @@
 #include "api/media_stream_interface.h"
 #include "api/video/video_frame.h"
 #include "api/video/video_rotation.h"
+#include "audio/remix_resample.h"
+#include "common_audio/include/audio_util.h"
 #include "rtc_base/logging.h"
 #include "rtc_base/ref_counted_object.h"
 #include "rtc_base/time_utils.h"
@@ -129,6 +131,93 @@ TrackState MediaStreamTrack::state() const {
 AudioTrack::AudioTrack(rtc::scoped_refptr<webrtc::AudioTrackInterface> track)
     : MediaStreamTrack(std::move(track)) {}
 
+void AudioTrack::add_sink(NativeAudioSink& sink) const {
+  track()->AddSink(&sink);
+}
+
+void AudioTrack::remove_sink(NativeAudioSink& sink) const {
+  track()->RemoveSink(&sink);
+}
+
+NativeAudioSink::NativeAudioSink(rust::Box<AudioSinkWrapper> observer)
+    : observer_(std::move(observer)) {}
+
+void NativeAudioSink::OnData(const void* audio_data,
+                             int bits_per_sample,
+                             int sample_rate,
+                             size_t number_of_channels,
+                             size_t number_of_frames) {
+  RTC_CHECK_EQ(16, bits_per_sample);
+
+  observer_->on_data(static_cast<const int16_t*>(audio_data), sample_rate,
+                     number_of_channels, number_of_frames);
+}
+
+std::unique_ptr<NativeAudioSink> new_native_audio_sink(
+    rust::Box<AudioSinkWrapper> observer) {
+  return std::make_unique<NativeAudioSink>(std::move(observer));
+}
+
+NativeAudioTrackSource::NativeAudioTrackSource() {
+  options_.echo_cancellation = false;
+  options_.auto_gain_control = false;
+  options_.noise_suppression = false;
+}
+
+webrtc::MediaSourceInterface::SourceState NativeAudioTrackSource::state()
+    const {
+  return webrtc::MediaSourceInterface::SourceState::kLive;
+}
+
+bool NativeAudioTrackSource::remote() const {
+  return false;
+}
+
+const cricket::AudioOptions NativeAudioTrackSource::options() const {
+  return options_;
+}
+
+void NativeAudioTrackSource::AddSink(webrtc::AudioTrackSinkInterface* sink) {
+  webrtc::MutexLock lock(&mutex_);
+  sinks_.push_back(sink);
+}
+
+void NativeAudioTrackSource::RemoveSink(webrtc::AudioTrackSinkInterface* sink) {
+  webrtc::MutexLock lock(&mutex_);
+  sinks_.erase(std::remove(sinks_.begin(), sinks_.end(), sink), sinks_.end());
+}
+
+void NativeAudioTrackSource::on_captured_frame(const int16_t* data,
+                                               int sample_rate,
+                                               size_t number_of_channels,
+                                               size_t number_of_frames) {
+  webrtc::MutexLock lock(&mutex_);
+  for (auto sink : sinks_) {
+    sink->OnData(data, 16, sample_rate, number_of_channels, number_of_frames);
+  }
+}
+
+AudioTrackSource::AudioTrackSource(
+    rtc::scoped_refptr<NativeAudioTrackSource> source)
+    : source_(std::move(source)) {}
+
+void AudioTrackSource::on_captured_frame(const int16_t* audio_data,
+                                         int sample_rate,
+                                         size_t number_of_channels,
+                                         size_t number_of_frames) const {
+  source_->on_captured_frame(audio_data, sample_rate, number_of_channels,
+                             number_of_frames);
+}
+
+rtc::scoped_refptr<NativeAudioTrackSource> AudioTrackSource::get() const {
+  return source_;
+}
+
+std::shared_ptr<AudioTrackSource> new_audio_track_source() {
+  return std::make_shared<AudioTrackSource>(
+      rtc::make_ref_counted<NativeAudioTrackSource>());
+}
+
 VideoTrack::VideoTrack(rtc::scoped_refptr<webrtc::VideoTrackInterface> track)
     : MediaStreamTrack(std::move(track)) {}
 

webrtc-sys/src/media_stream.rs

@@ -40,11 +40,13 @@ pub mod ffi {
     unsafe extern "C++" {
         include!("livekit/media_stream.h");
 
+        type NativeAudioSink;
         type NativeVideoFrameSink;
         type MediaStreamTrack;
         type MediaStream;
         type AudioTrack;
         type VideoTrack;
+        type AudioTrackSource;
         type AdaptedVideoTrackSource;
 
         fn id(self: &MediaStream) -> String;
@@ -61,6 +63,21 @@ pub mod ffi {
         fn set_enabled(self: &MediaStreamTrack, enable: bool) -> bool;
         fn state(self: &MediaStreamTrack) -> TrackState;
 
+        unsafe fn add_sink(self: &AudioTrack, sink: Pin<&mut NativeAudioSink>);
+        unsafe fn remove_sink(self: &AudioTrack, sink: Pin<&mut NativeAudioSink>);
+
+        fn new_native_audio_sink(observer: Box<AudioSinkWrapper>) -> UniquePtr<NativeAudioSink>;
+
+        unsafe fn on_captured_frame(
+            self: &AudioTrackSource,
+            data: *const i16,
+            sample_rate: i32,
+            nb_channels: usize,
+            nb_frames: usize,
+        );
+
+        fn new_audio_track_source() -> SharedPtr<AudioTrackSource>;
+
         unsafe fn add_sink(self: &VideoTrack, sink: Pin<&mut NativeVideoFrameSink>);
         unsafe fn remove_sink(self: &VideoTrack, sink: Pin<&mut NativeVideoFrameSink>);
 
@@ -90,8 +107,17 @@ pub mod ffi {
     }
 
     extern "Rust" {
+        type AudioSinkWrapper;
         type VideoFrameSinkWrapper;
 
+        unsafe fn on_data(
+            self: &AudioSinkWrapper,
+            data: *const i16,
+            sample_rate: i32,
+            nb_channels: usize,
+            nb_frames: usize,
+        );
+
         fn on_frame(self: &VideoFrameSinkWrapper, frame: UniquePtr<VideoFrame>);
         fn on_discarded_frame(self: &VideoFrameSinkWrapper);
         fn on_constraints_changed(
@@ -106,8 +132,33 @@ impl_thread_safety!(ffi::MediaStream, Send + Sync);
 impl_thread_safety!(ffi::AudioTrack, Send + Sync);
 impl_thread_safety!(ffi::VideoTrack, Send + Sync);
 impl_thread_safety!(ffi::NativeVideoFrameSink, Send + Sync);
+impl_thread_safety!(ffi::NativeAudioSink, Send + Sync);
+impl_thread_safety!(ffi::AudioTrackSource, Send + Sync);
 impl_thread_safety!(ffi::AdaptedVideoTrackSource, Send + Sync);
 
+pub trait AudioSink: Send {
+    fn on_data(&self, data: &[i16], sample_rate: i32, nb_channels: usize, nb_frames: usize);
+}
+
+pub struct AudioSinkWrapper {
+    observer: *mut dyn AudioSink,
+}
+
+impl AudioSinkWrapper {
+    /// # Safety
+    /// AudioSink must lives as long as AudioSinkWrapper does
+    pub unsafe fn new(observer: *mut dyn AudioSink) -> Self {
+        Self { observer }
+    }
+
+    fn on_data(&self, data: *const i16, sample_rate: i32, nb_channels: usize, nb_frames: usize) {
+        unsafe {
+            let data = std::slice::from_raw_parts(data, nb_channels * nb_frames);
+            (*self.observer).on_data(data, sample_rate, nb_channels, nb_frames);
+        }
+    }
+}
+
 pub trait VideoFrameSink: Send {
     fn on_frame(&self, frame: UniquePtr<VideoFrame>);
     fn on_discarded_frame(&self);

webrtc-sys/src/peer_connection_factory.cpp

@@ -25,11 +25,14 @@
 #include "api/task_queue/default_task_queue_factory.h"
 #include "api/video_codecs/builtin_video_decoder_factory.h"
 #include "api/video_codecs/builtin_video_encoder_factory.h"
+#include "livekit/audio_device.h"
 #include "livekit/rtc_error.h"
 #include "livekit/rtp_parameters.h"
 #include "livekit/video_decoder_factory.h"
 #include "livekit/video_encoder_factory.h"
 #include "media/engine/webrtc_media_engine.h"
+#include "rtc_base/location.h"
+#include "rtc_base/thread.h"
 
 namespace livekit {
 
@@ -51,6 +54,14 @@ PeerConnectionFactory::PeerConnectionFactory(
 
   cricket::MediaEngineDependencies media_deps;
   media_deps.task_queue_factory = dependencies.task_queue_factory.get();
+
+  media_deps.adm = rtc_runtime_->worker_thread()
+                       ->Invoke<rtc::scoped_refptr<livekit::AudioDevice>>(
+                           RTC_FROM_HERE, [&] {
+                             return rtc::make_ref_counted<livekit::AudioDevice>(
+                                 media_deps.task_queue_factory);
+                           });
+
   media_deps.video_encoder_factory =
       std::move(std::make_unique<livekit::VideoEncoderFactory>());
   media_deps.video_decoder_factory =
@@ -96,6 +107,13 @@ std::shared_ptr<VideoTrack> PeerConnectionFactory::create_video_track(
       peer_factory_->CreateVideoTrack(label.c_str(), source->get().get()));
 }
 
+std::shared_ptr<AudioTrack> PeerConnectionFactory::create_audio_track(
+    rust::String label,
+    std::shared_ptr<AudioTrackSource> source) const {
+  return std::make_shared<AudioTrack>(
+      peer_factory_->CreateAudioTrack(label.c_str(), source->get().get()));
+}
+
 RtpCapabilities PeerConnectionFactory::get_rtp_sender_capabilities(
     MediaType type) const {
   return to_rust_rtp_capabilities(peer_factory_->GetRtpSenderCapabilities(

webrtc-sys/src/peer_connection_factory.rs

@@ -37,7 +37,9 @@ pub mod ffi {
         include!("livekit/webrtc.h");
         include!("livekit/rtp_parameters.h");
 
+        type AudioTrackSource = crate::media_stream::ffi::AudioTrackSource;
         type AdaptedVideoTrackSource = crate::media_stream::ffi::AdaptedVideoTrackSource;
+        type AudioTrack = crate::media_stream::ffi::AudioTrack;
         type VideoTrack = crate::media_stream::ffi::VideoTrack;
         type RtpCapabilities = crate::rtp_parameters::ffi::RtpCapabilities;
         type MediaType = crate::webrtc::ffi::MediaType;
@@ -72,6 +74,12 @@ pub mod ffi {
             source: SharedPtr<AdaptedVideoTrackSource>,
         ) -> SharedPtr<VideoTrack>;
 
+        fn create_audio_track(
+            self: &PeerConnectionFactory,
+            label: String,
+            source: SharedPtr<AudioTrackSource>,
+        ) -> SharedPtr<AudioTrack>;
+
         fn get_rtp_sender_capabilities(
             self: &PeerConnectionFactory,
             kind: MediaType,