feat: audio support (#45)

- Receive/Send audio frames - LocalAudioTrack
2023-03-19 21:58:20 +01:00
parent cad6d36201
commit 64c0705916
30 changed files with 1207 additions and 69 deletions
@@ -0,0 +1,323 @@
+/*
+ * Copyright 2023 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the “License”);
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an “AS IS” BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "livekit/audio_device.h"
+
+const int kBitsPerSample = 16;
+const int kSampleRate = 48000;
+const int kChannels = 2;
+const int kSamplesPer10Ms = kSampleRate / 100;
+
+namespace livekit {
+
+AudioDevice::AudioDevice(webrtc::TaskQueueFactory* task_queue_factory)
+    : task_queue_factory_(task_queue_factory),
+      data_(kSamplesPer10Ms * kChannels) {}
+
+AudioDevice::~AudioDevice() {
+  Terminate();
+}
+
+int32_t AudioDevice::ActiveAudioLayer(AudioLayer* audioLayer) const {
+  *audioLayer = AudioLayer::kDummyAudio;
+  return 0;
+}
+
+int32_t AudioDevice::RegisterAudioCallback(webrtc::AudioTransport* transport) {
+  webrtc::MutexLock lock(&mutex_);
+  audio_transport_ = transport;
+  return 0;
+}
+
+int32_t AudioDevice::Init() {
+  audio_queue_ =
+      std::make_unique<rtc::TaskQueue>(task_queue_factory_->CreateTaskQueue(
+          "AudioDevice", webrtc::TaskQueueFactory::Priority::NORMAL));
+
+  audio_task_ =
+      webrtc::RepeatingTaskHandle::Start(audio_queue_->Get(), [this]() {
+        webrtc::MutexLock lock(&mutex_);
+
+        if (playing_) {
+          int64_t elapsed_time_ms = -1;
+          int64_t ntp_time_ms = -1;
+          void* data = data_.data();
+
+          // Request the AudioData, otherwise WebRTC will ignore the packets.
+          // 10ms of audio data.
+          audio_transport_->PullRenderData(kBitsPerSample, kSampleRate,
+                                           kChannels, kSamplesPer10Ms, data,
+                                           &elapsed_time_ms, &ntp_time_ms);
+        }
+
+        return webrtc::TimeDelta::Millis(10);
+      });
+
+  initialized_ = true;
+  return 0;
+}
+
+int32_t AudioDevice::Terminate() {
+  if (!initialized_)
+    return 0;
+
+  initialized_ = false;
+
+  audio_queue_->PostTask([this] { audio_task_.Stop(); });
+
+  StopRecording();
+  StopPlayout();
+  return 0;
+}
+
+bool AudioDevice::Initialized() const {
+  return initialized_;
+}
+
+int16_t AudioDevice::PlayoutDevices() {
+  return 0;
+}
+
+int16_t AudioDevice::RecordingDevices() {
+  return 0;
+}
+
+int32_t AudioDevice::PlayoutDeviceName(uint16_t index,
+                                       char name[webrtc::kAdmMaxDeviceNameSize],
+                                       char guid[webrtc::kAdmMaxGuidSize]) {
+  return 0;
+}
+
+int32_t AudioDevice::RecordingDeviceName(
+    uint16_t index,
+    char name[webrtc::kAdmMaxDeviceNameSize],
+    char guid[webrtc::kAdmMaxGuidSize]) {
+  return 0;
+}
+
+int32_t AudioDevice::SetPlayoutDevice(uint16_t index) {
+  return 0;
+}
+
+int32_t AudioDevice::SetPlayoutDevice(WindowsDeviceType device) {
+  return 0;
+}
+
+int32_t AudioDevice::SetRecordingDevice(uint16_t index) {
+  return 0;
+}
+
+int32_t AudioDevice::SetRecordingDevice(WindowsDeviceType device) {
+  return 0;
+}
+
+int32_t AudioDevice::PlayoutIsAvailable(bool* available) {
+  return 0;
+}
+
+int32_t AudioDevice::InitPlayout() {
+  return 0;
+}
+
+bool AudioDevice::PlayoutIsInitialized() const {
+  return false;
+}
+
+int32_t AudioDevice::RecordingIsAvailable(bool* available) {
+  return 0;
+}
+
+int32_t AudioDevice::InitRecording() {
+  return 0;
+}
+
+bool AudioDevice::RecordingIsInitialized() const {
+  return false;
+}
+
+int32_t AudioDevice::StartPlayout() {
+  playing_ = true;
+  return 0;
+}
+
+int32_t AudioDevice::StopPlayout() {
+  playing_ = false;
+  return 0;
+}
+
+bool AudioDevice::Playing() const {
+  return false;
+}
+
+int32_t AudioDevice::StartRecording() {
+  return 0;
+}
+
+int32_t AudioDevice::StopRecording() {
+  return 0;
+}
+
+bool AudioDevice::Recording() const {
+  return false;
+}
+
+int32_t AudioDevice::InitSpeaker() {
+  return 0;
+}
+
+bool AudioDevice::SpeakerIsInitialized() const {
+  return false;
+}
+
+int32_t AudioDevice::InitMicrophone() {
+  return 0;
+}
+
+bool AudioDevice::MicrophoneIsInitialized() const {
+  return false;
+}
+
+int32_t AudioDevice::SpeakerVolumeIsAvailable(bool* available) {
+  return 0;
+}
+
+int32_t AudioDevice::SetSpeakerVolume(uint32_t volume) {
+  return 0;
+}
+
+int32_t AudioDevice::SpeakerVolume(uint32_t* volume) const {
+  return 0;
+}
+
+int32_t AudioDevice::MaxSpeakerVolume(uint32_t* maxVolume) const {
+  return 0;
+}
+
+int32_t AudioDevice::MinSpeakerVolume(uint32_t* minVolume) const {
+  return 0;
+}
+
+int32_t AudioDevice::MicrophoneVolumeIsAvailable(bool* available) {
+  return 0;
+}
+
+int32_t AudioDevice::SetMicrophoneVolume(uint32_t volume) {
+  return 0;
+}
+
+int32_t AudioDevice::MicrophoneVolume(uint32_t* volume) const {
+  return 0;
+}
+
+int32_t AudioDevice::MaxMicrophoneVolume(uint32_t* maxVolume) const {
+  return 0;
+}
+
+int32_t AudioDevice::MinMicrophoneVolume(uint32_t* minVolume) const {
+  return 0;
+}
+
+int32_t AudioDevice::SpeakerMuteIsAvailable(bool* available) {
+  return 0;
+}
+
+int32_t AudioDevice::SetSpeakerMute(bool enable) {
+  return 0;
+}
+
+int32_t AudioDevice::SpeakerMute(bool* enabled) const {
+  return 0;
+}
+
+int32_t AudioDevice::MicrophoneMuteIsAvailable(bool* available) {
+  return 0;
+}
+
+int32_t AudioDevice::SetMicrophoneMute(bool enable) {
+  return 0;
+}
+
+int32_t AudioDevice::MicrophoneMute(bool* enabled) const {
+  return 0;
+}
+
+int32_t AudioDevice::StereoPlayoutIsAvailable(bool* available) const {
+  return 0;
+}
+
+int32_t AudioDevice::SetStereoPlayout(bool enable) {
+  return 0;
+}
+
+int32_t AudioDevice::StereoPlayout(bool* enabled) const {
+  return 0;
+}
+
+int32_t AudioDevice::StereoRecordingIsAvailable(bool* available) const {
+  return 0;
+}
+
+int32_t AudioDevice::SetStereoRecording(bool enable) {
+  return 0;
+}
+
+int32_t AudioDevice::StereoRecording(bool* enabled) const {
+  return 0;
+}
+
+int32_t AudioDevice::PlayoutDelay(uint16_t* delayMS) const {
+  return 0;
+}
+
+bool AudioDevice::BuiltInAECIsAvailable() const {
+  return false;
+}
+
+bool AudioDevice::BuiltInAGCIsAvailable() const {
+  return false;
+}
+
+bool AudioDevice::BuiltInNSIsAvailable() const {
+  return false;
+}
+
+int32_t AudioDevice::EnableBuiltInAEC(bool enable) {
+  return 0;
+}
+
+int32_t AudioDevice::EnableBuiltInAGC(bool enable) {
+  return 0;
+}
+
+int32_t AudioDevice::EnableBuiltInNS(bool enable) {
+  return 0;
+}
+
+#if defined(WEBRTC_IOS)
+int AudioDevice::GetPlayoutAudioParameters(AudioParameters* params) const {
+  return 0;
+}
+
+int AudioDevice::GetRecordAudioParameters(AudioParameters* params) const {
+  return 0;
+}
+#endif  // WEBRTC_IOS
+
+int32_t AudioDevice::SetAudioDeviceSink(webrtc::AudioDeviceSink* sink) const {
+  return 0;
+}
+
+}  // namespace livekit
@@ -23,6 +23,8 @@
 #include "api/media_stream_interface.h"
 #include "api/video/video_frame.h"
 #include "api/video/video_rotation.h"
+#include "audio/remix_resample.h"
+#include "common_audio/include/audio_util.h"
 #include "rtc_base/logging.h"
 #include "rtc_base/ref_counted_object.h"
 #include "rtc_base/time_utils.h"
@@ -129,6 +131,93 @@ TrackState MediaStreamTrack::state() const {
 AudioTrack::AudioTrack(rtc::scoped_refptr<webrtc::AudioTrackInterface> track)
    : MediaStreamTrack(std::move(track)) {}

+void AudioTrack::add_sink(NativeAudioSink& sink) const {
+  track()->AddSink(&sink);
+}
+
+void AudioTrack::remove_sink(NativeAudioSink& sink) const {
+  track()->RemoveSink(&sink);
+}
+
+NativeAudioSink::NativeAudioSink(rust::Box<AudioSinkWrapper> observer)
+    : observer_(std::move(observer)) {}
+
+void NativeAudioSink::OnData(const void* audio_data,
+                             int bits_per_sample,
+                             int sample_rate,
+                             size_t number_of_channels,
+                             size_t number_of_frames) {
+  RTC_CHECK_EQ(16, bits_per_sample);
+
+  observer_->on_data(static_cast<const int16_t*>(audio_data), sample_rate,
+                     number_of_channels, number_of_frames);
+}
+
+std::unique_ptr<NativeAudioSink> new_native_audio_sink(
+    rust::Box<AudioSinkWrapper> observer) {
+  return std::make_unique<NativeAudioSink>(std::move(observer));
+}
+
+NativeAudioTrackSource::NativeAudioTrackSource() {
+  options_.echo_cancellation = false;
+  options_.auto_gain_control = false;
+  options_.noise_suppression = false;
+}
+
+webrtc::MediaSourceInterface::SourceState NativeAudioTrackSource::state()
+    const {
+  return webrtc::MediaSourceInterface::SourceState::kLive;
+}
+
+bool NativeAudioTrackSource::remote() const {
+  return false;
+}
+
+const cricket::AudioOptions NativeAudioTrackSource::options() const {
+  return options_;
+}
+
+void NativeAudioTrackSource::AddSink(webrtc::AudioTrackSinkInterface* sink) {
+  webrtc::MutexLock lock(&mutex_);
+  sinks_.push_back(sink);
+}
+
+void NativeAudioTrackSource::RemoveSink(webrtc::AudioTrackSinkInterface* sink) {
+  webrtc::MutexLock lock(&mutex_);
+  sinks_.erase(std::remove(sinks_.begin(), sinks_.end(), sink), sinks_.end());
+}
+
+void NativeAudioTrackSource::on_captured_frame(const int16_t* data,
+                                               int sample_rate,
+                                               size_t number_of_channels,
+                                               size_t number_of_frames) {
+  webrtc::MutexLock lock(&mutex_);
+  for (auto sink : sinks_) {
+    sink->OnData(data, 16, sample_rate, number_of_channels, number_of_frames);
+  }
+}
+
+AudioTrackSource::AudioTrackSource(
+    rtc::scoped_refptr<NativeAudioTrackSource> source)
+    : source_(std::move(source)) {}
+
+void AudioTrackSource::on_captured_frame(const int16_t* audio_data,
+                                         int sample_rate,
+                                         size_t number_of_channels,
+                                         size_t number_of_frames) const {
+  source_->on_captured_frame(audio_data, sample_rate, number_of_channels,
+                             number_of_frames);
+}
+
+rtc::scoped_refptr<NativeAudioTrackSource> AudioTrackSource::get() const {
+  return source_;
+}
+
+std::shared_ptr<AudioTrackSource> new_audio_track_source() {
+  return std::make_shared<AudioTrackSource>(
+      rtc::make_ref_counted<NativeAudioTrackSource>());
+}
+
 VideoTrack::VideoTrack(rtc::scoped_refptr<webrtc::VideoTrackInterface> track)
    : MediaStreamTrack(std::move(track)) {}

@@ -40,11 +40,13 @@ pub mod ffi {
    unsafe extern "C++" {
        include!("livekit/media_stream.h");

+        type NativeAudioSink;
        type NativeVideoFrameSink;
        type MediaStreamTrack;
        type MediaStream;
        type AudioTrack;
        type VideoTrack;
+        type AudioTrackSource;
        type AdaptedVideoTrackSource;

        fn id(self: &MediaStream) -> String;
@@ -61,6 +63,21 @@ pub mod ffi {
        fn set_enabled(self: &MediaStreamTrack, enable: bool) -> bool;
        fn state(self: &MediaStreamTrack) -> TrackState;

+        unsafe fn add_sink(self: &AudioTrack, sink: Pin<&mut NativeAudioSink>);
+        unsafe fn remove_sink(self: &AudioTrack, sink: Pin<&mut NativeAudioSink>);
+
+        fn new_native_audio_sink(observer: Box<AudioSinkWrapper>) -> UniquePtr<NativeAudioSink>;
+
+        unsafe fn on_captured_frame(
+            self: &AudioTrackSource,
+            data: *const i16,
+            sample_rate: i32,
+            nb_channels: usize,
+            nb_frames: usize,
+        );
+
+        fn new_audio_track_source() -> SharedPtr<AudioTrackSource>;
+
        unsafe fn add_sink(self: &VideoTrack, sink: Pin<&mut NativeVideoFrameSink>);
        unsafe fn remove_sink(self: &VideoTrack, sink: Pin<&mut NativeVideoFrameSink>);

@@ -90,8 +107,17 @@ pub mod ffi {
    }

    extern "Rust" {
+        type AudioSinkWrapper;
        type VideoFrameSinkWrapper;

+        unsafe fn on_data(
+            self: &AudioSinkWrapper,
+            data: *const i16,
+            sample_rate: i32,
+            nb_channels: usize,
+            nb_frames: usize,
+        );
+
        fn on_frame(self: &VideoFrameSinkWrapper, frame: UniquePtr<VideoFrame>);
        fn on_discarded_frame(self: &VideoFrameSinkWrapper);
        fn on_constraints_changed(
@@ -106,8 +132,33 @@ impl_thread_safety!(ffi::MediaStream, Send + Sync);
 impl_thread_safety!(ffi::AudioTrack, Send + Sync);
 impl_thread_safety!(ffi::VideoTrack, Send + Sync);
 impl_thread_safety!(ffi::NativeVideoFrameSink, Send + Sync);
+impl_thread_safety!(ffi::NativeAudioSink, Send + Sync);
+impl_thread_safety!(ffi::AudioTrackSource, Send + Sync);
 impl_thread_safety!(ffi::AdaptedVideoTrackSource, Send + Sync);

+pub trait AudioSink: Send {
+    fn on_data(&self, data: &[i16], sample_rate: i32, nb_channels: usize, nb_frames: usize);
+}
+
+pub struct AudioSinkWrapper {
+    observer: *mut dyn AudioSink,
+}
+
+impl AudioSinkWrapper {
+    /// # Safety
+    /// AudioSink must lives as long as AudioSinkWrapper does
+    pub unsafe fn new(observer: *mut dyn AudioSink) -> Self {
+        Self { observer }
+    }
+
+    fn on_data(&self, data: *const i16, sample_rate: i32, nb_channels: usize, nb_frames: usize) {
+        unsafe {
+            let data = std::slice::from_raw_parts(data, nb_channels * nb_frames);
+            (*self.observer).on_data(data, sample_rate, nb_channels, nb_frames);
+        }
+    }
+}
+
 pub trait VideoFrameSink: Send {
    fn on_frame(&self, frame: UniquePtr<VideoFrame>);
    fn on_discarded_frame(&self);
@@ -25,11 +25,14 @@
 #include "api/task_queue/default_task_queue_factory.h"
 #include "api/video_codecs/builtin_video_decoder_factory.h"
 #include "api/video_codecs/builtin_video_encoder_factory.h"
+#include "livekit/audio_device.h"
 #include "livekit/rtc_error.h"
 #include "livekit/rtp_parameters.h"
 #include "livekit/video_decoder_factory.h"
 #include "livekit/video_encoder_factory.h"
 #include "media/engine/webrtc_media_engine.h"
+#include "rtc_base/location.h"
+#include "rtc_base/thread.h"

 namespace livekit {

@@ -51,6 +54,14 @@ PeerConnectionFactory::PeerConnectionFactory(

  cricket::MediaEngineDependencies media_deps;
  media_deps.task_queue_factory = dependencies.task_queue_factory.get();
+
+  media_deps.adm = rtc_runtime_->worker_thread()
+                       ->Invoke<rtc::scoped_refptr<livekit::AudioDevice>>(
+                           RTC_FROM_HERE, [&] {
+                             return rtc::make_ref_counted<livekit::AudioDevice>(
+                                 media_deps.task_queue_factory);
+                           });
+
  media_deps.video_encoder_factory =
      std::move(std::make_unique<livekit::VideoEncoderFactory>());
  media_deps.video_decoder_factory =
@@ -96,6 +107,13 @@ std::shared_ptr<VideoTrack> PeerConnectionFactory::create_video_track(
      peer_factory_->CreateVideoTrack(label.c_str(), source->get().get()));
 }

+std::shared_ptr<AudioTrack> PeerConnectionFactory::create_audio_track(
+    rust::String label,
+    std::shared_ptr<AudioTrackSource> source) const {
+  return std::make_shared<AudioTrack>(
+      peer_factory_->CreateAudioTrack(label.c_str(), source->get().get()));
+}
+
 RtpCapabilities PeerConnectionFactory::get_rtp_sender_capabilities(
    MediaType type) const {
  return to_rust_rtp_capabilities(peer_factory_->GetRtpSenderCapabilities(
@@ -37,7 +37,9 @@ pub mod ffi {
        include!("livekit/webrtc.h");
        include!("livekit/rtp_parameters.h");

+        type AudioTrackSource = crate::media_stream::ffi::AudioTrackSource;
        type AdaptedVideoTrackSource = crate::media_stream::ffi::AdaptedVideoTrackSource;
+        type AudioTrack = crate::media_stream::ffi::AudioTrack;
        type VideoTrack = crate::media_stream::ffi::VideoTrack;
        type RtpCapabilities = crate::rtp_parameters::ffi::RtpCapabilities;
        type MediaType = crate::webrtc::ffi::MediaType;
@@ -72,6 +74,12 @@ pub mod ffi {
            source: SharedPtr<AdaptedVideoTrackSource>,
        ) -> SharedPtr<VideoTrack>;

+        fn create_audio_track(
+            self: &PeerConnectionFactory,
+            label: String,
+            source: SharedPtr<AudioTrackSource>,
+        ) -> SharedPtr<AudioTrack>;
+
        fn get_rtp_sender_capabilities(
            self: &PeerConnectionFactory,
            kind: MediaType,