feat: AudioResampler (#66)

examples/save_to_disk/src/main.rs

@@ -45,7 +45,7 @@ impl WavWriter {
             * self.header.bit_depth as u32
             * self.header.num_channels as u32);
 
-        let block_align = byte_rate / self.header.sample_rate as u32;
+        let block_align = byte_rate as u16 / self.header.sample_rate as u16;
 
         self.data.put_slice(b"RIFF");
         self.data.put_u32_le(0); // Placeholder for file size
@@ -56,7 +56,7 @@ impl WavWriter {
         self.data.put_u16_le(self.header.num_channels);
         self.data.put_u32_le(self.header.sample_rate);
         self.data.put_u32_le(byte_rate);
-        self.data.put_u16_le(32);
+        self.data.put_u16_le(block_align);
         self.data.put_u16_le(self.header.bit_depth);
         self.data.put_slice(b"data");
         self.data.put_u32_le(0); // Placeholder for data size

livekit-ffi/protocol/audio_frame.proto

@@ -36,6 +36,24 @@ message CaptureAudioFrameRequest {
 }
 message CaptureAudioFrameResponse {}
 
+// Create a new AudioResampler
+message NewAudioResamplerRequest {}
+message NewAudioResamplerResponse {
+  FFIHandleId handle = 1;
+}
+
+// Remix and resample an audio frame
+message RemixAndResampleRequest {
+  FFIHandleId resampler_handle = 1;
+  FFIHandleId buffer_handle = 2;
+  uint32 num_channels = 3;
+  uint32 sample_rate = 4;
+}
+
+message RemixAndResampleResponse {
+  FFIHandleId buffer_handle = 1;
+}
+
 ///
 /// AudioFrame buffer ///
 ///

livekit-ffi/protocol/ffi.proto

@@ -40,6 +40,8 @@ message FFIRequest {
     NewAudioStreamRequest new_audio_stream = 16;
     NewAudioSourceRequest new_audio_source = 17;
     CaptureAudioFrameRequest capture_audio_frame = 18;
+    NewAudioResamplerRequest new_audio_resampler = 19;
+    RemixAndResampleRequest remix_and_resample = 20;
   }
 }
 
@@ -72,6 +74,8 @@ message FFIResponse {
     NewAudioStreamResponse new_audio_stream = 16;
     NewAudioSourceResponse new_audio_source = 17;
     CaptureAudioFrameResponse capture_audio_frame = 18;
+    NewAudioResamplerResponse new_audio_resampler = 19;
+    RemixAndResampleResponse remix_and_resample = 20;
   }
 }
 

livekit-ffi/src/server/mod.rs

@@ -3,7 +3,7 @@ use crate::{FfiAsyncId, FfiError, FfiHandle, FfiHandleId, FfiResult};
 use dashmap::DashMap;
 use lazy_static::lazy_static;
 use livekit::prelude::*;
-use livekit::webrtc::native::yuv_helper;
+use livekit::webrtc::native::{audio_resampler, yuv_helper};
 use livekit::webrtc::prelude::*;
 use livekit::webrtc::video_frame::{native::I420BufferExt, BoxVideoFrameBuffer, I420Buffer};
 use parking_lot::Mutex;
@@ -11,6 +11,7 @@ use prost::Message;
 use std::collections::HashMap;
 use std::slice;
 use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::Arc;
 
 pub mod audio_frame;
 pub mod room;
@@ -608,6 +609,83 @@ impl FfiServer {
         Ok(proto::CaptureAudioFrameResponse::default())
     }
 
+    fn new_audio_resampler(
+        &'static self,
+        new_resampler: proto::NewAudioResamplerRequest,
+    ) -> FfiResult<proto::NewAudioResamplerResponse> {
+        let resampler = audio_resampler::AudioResampler::default();
+        let resampler = Arc::new(Mutex::new(resampler));
+
+        let handle_id = self.next_id() as FfiHandleId;
+        self.ffi_handles.insert(handle_id, Box::new(resampler));
+
+        Ok(proto::NewAudioResamplerResponse {
+            handle: Some(handle_id.into()),
+        })
+    }
+
+    fn remix_and_resample(
+        &'static self,
+        remix: proto::RemixAndResampleRequest,
+    ) -> FfiResult<proto::RemixAndResampleResponse> {
+        let resampler_id = remix
+            .resampler_handle
+            .as_ref()
+            .ok_or(FfiError::InvalidRequest("handle is empty"))?
+            .id as FfiHandleId;
+
+        let resampler = self
+            .ffi_handles
+            .get(&resampler_id)
+            .ok_or(FfiError::InvalidRequest("resampler not found"))?;
+
+        let resampler = resampler
+            .downcast_ref::<Arc<Mutex<audio_resampler::AudioResampler>>>()
+            .ok_or(FfiError::InvalidRequest("handle is not a resampler"))?;
+
+        let resampler = resampler.clone();
+
+        let buffer_id = remix
+            .buffer_handle
+            .as_ref()
+            .ok_or(FfiError::InvalidRequest("handle is empty"))?
+            .id as FfiHandleId;
+
+        let buffer = self
+            .ffi_handles
+            .get(&buffer_id)
+            .ok_or(FfiError::InvalidRequest("buffer not found"))?;
+
+        let buffer = buffer
+            .downcast_ref::<AudioFrame>()
+            .ok_or(FfiError::InvalidRequest("handle is not a buffer"))?;
+
+        let mut resampler = resampler.lock();
+        let data = resampler.remix_and_resample(
+            &buffer.data,
+            buffer.samples_per_channel,
+            buffer.num_channels,
+            buffer.sample_rate,
+            remix.num_channels,
+            remix.sample_rate,
+        );
+
+        let samples_per_channel = data.len() / remix.num_channels as usize;
+        let frame = AudioFrame {
+            data: data.to_owned(), // Copy?
+            num_channels: remix.num_channels,
+            samples_per_channel: samples_per_channel as u32,
+            sample_rate: remix.sample_rate,
+        };
+
+        let handle_id = self.next_id() as FfiHandleId;
+        self.ffi_handles.insert(handle_id, Box::new(frame));
+
+        Ok(proto::RemixAndResampleResponse {
+            buffer_handle: Some(handle_id.into()),
+        })
+    }
+
     pub fn handle_request(
         &'static self,
         request: proto::FfiRequest,
@@ -672,6 +750,12 @@ impl FfiServer {
             proto::ffi_request::Message::CaptureAudioFrame(push) => {
                 proto::ffi_response::Message::CaptureAudioFrame(self.on_capture_audio_frame(push)?)
             }
+            proto::ffi_request::Message::NewAudioResampler(new_res) => {
+                proto::ffi_response::Message::NewAudioResampler(self.new_audio_resampler(new_res)?)
+            }
+            proto::ffi_request::Message::RemixAndResample(remix) => {
+                proto::ffi_response::Message::RemixAndResample(self.remix_and_resample(remix)?)
+            }
         });
 
         Ok(res)

livekit-webrtc/src/lib.rs

@@ -46,6 +46,7 @@ pub mod video_stream;
 
 #[cfg(not(target_arch = "wasm32"))]
 pub mod native {
+    pub use crate::imp::audio_resampler;
     pub use crate::imp::yuv_helper;
     pub use webrtc_sys::webrtc::ffi::create_random_uuid;
 }

livekit-webrtc/src/native/audio_resampler.rs

@@ -0,0 +1,39 @@
+use cxx::UniquePtr;
+use webrtc_sys::audio_resampler as sys_ar;
+
+pub struct AudioResampler {
+    sys_handle: UniquePtr<sys_ar::ffi::AudioResampler>,
+}
+
+impl Default for AudioResampler {
+    fn default() -> Self {
+        Self {
+            sys_handle: sys_ar::ffi::create_audio_resampler(),
+        }
+    }
+}
+
+impl AudioResampler {
+    pub fn remix_and_resample(
+        &mut self,
+        src: &[i16],
+        samples_per_channel: u32,
+        num_channels: u32,
+        sample_rate: u32,
+        dst_num_channels: u32,
+        dst_sample_rate: u32,
+    ) -> &[i16] {
+        unsafe {
+            let len = self.sys_handle.pin_mut().remix_and_resample(
+                src.as_ptr(),
+                samples_per_channel as usize,
+                num_channels as usize,
+                sample_rate as i32,
+                dst_num_channels as usize,
+                dst_sample_rate as i32,
+            );
+
+            std::slice::from_raw_parts(self.sys_handle.data(), len)
+        }
+    }
+}

livekit-webrtc/src/native/mod.rs

@@ -1,3 +1,4 @@
+pub mod audio_resampler;
 pub mod audio_source;
 pub mod audio_stream;
 pub mod data_channel;

webrtc-sys/.gitignore

@@ -1 +0,0 @@
-/cmake-build-debug

webrtc-sys/build.rs

@@ -149,6 +149,7 @@ fn main() {
         "src/video_frame_buffer.rs",
         "src/yuv_helper.rs",
         "src/helper.rs",
+        "src/audio_resampler.rs",
     ]);
 
     builder.files(&[
@@ -169,6 +170,7 @@ fn main() {
         "src/video_encoder_factory.cpp",
         "src/video_decoder_factory.cpp",
         "src/audio_device.cpp",
+        "src/audio_resampler.cpp",
     ]);
 
     for include in includes {

webrtc-sys/include/livekit/audio_resampler.h

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2023 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the “License”);
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an “AS IS” BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <memory>
+
+#include "api/audio/audio_frame.h"
+#include "api/data_channel_interface.h"
+#include "common_audio/resampler/include/push_resampler.h"
+#include "livekit/webrtc.h"
+#include "rust/cxx.h"
+
+namespace livekit {
+
+class AudioResampler {
+ public:
+  size_t remix_and_resample(const int16_t* src,
+                            size_t samples_per_channel,
+                            size_t num_channels,
+                            int sample_rate_hz,
+                            size_t dest_num_channels,
+                            int dest_sample_rate_hz);
+
+  const int16_t* data() const;
+
+ private:
+  webrtc::AudioFrame frame_;
+  webrtc::PushResampler<int16_t> resampler_;
+};
+
+std::unique_ptr<AudioResampler> create_audio_resampler();
+
+}  // namespace livekit

webrtc-sys/src/audio_resampler.cpp

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2023 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the “License”);
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an “AS IS” BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "livekit/audio_resampler.h"
+
+#include <memory>
+
+#include "audio/remix_resample.h"
+
+namespace livekit {
+
+size_t AudioResampler::remix_and_resample(const int16_t* src,
+                                          size_t samples_per_channel,
+                                          size_t num_channels,
+                                          int sample_rate,
+                                          size_t dest_num_channels,
+                                          int dest_sample_rate) {
+  frame_.num_channels_ = dest_num_channels;
+  frame_.sample_rate_hz_ = dest_sample_rate;
+  webrtc::voe::RemixAndResample(src, samples_per_channel, num_channels,
+                                sample_rate, &resampler_, &frame_);
+
+  return frame_.num_channels() * frame_.samples_per_channel();
+}
+
+const int16_t* AudioResampler::data() const {
+  return frame_.data();
+}
+
+std::unique_ptr<AudioResampler> create_audio_resampler() {
+  return std::make_unique<AudioResampler>();
+}
+
+}  // namespace livekit

webrtc-sys/src/audio_resampler.rs

@@ -0,0 +1,26 @@
+use crate::impl_thread_safety;
+
+#[cxx::bridge(namespace = "livekit")]
+pub mod ffi {
+    unsafe extern "C++" {
+        include!("livekit/audio_resampler.h");
+
+        type AudioResampler;
+
+        unsafe fn remix_and_resample(
+            self: Pin<&mut AudioResampler>,
+            src: *const i16,
+            samples_per_channel: usize,
+            num_channels: usize,
+            sample_rate: i32,
+            dst_num_channels: usize,
+            dst_sample_rate: i32,
+        ) -> usize;
+
+        unsafe fn data(self: &AudioResampler) -> *const i16;
+
+        fn create_audio_resampler() -> UniquePtr<AudioResampler>;
+    }
+}
+
+impl_thread_safety!(ffi::AudioResampler, Send + Sync);

webrtc-sys/src/lib.rs

@@ -1,3 +1,4 @@
+pub mod audio_resampler;
 pub mod candidate;
 pub mod data_channel;
 pub mod helper;