diff --git a/examples/save_to_disk/src/main.rs b/examples/save_to_disk/src/main.rs index 61b36d3..9ef0dd3 100644 --- a/examples/save_to_disk/src/main.rs +++ b/examples/save_to_disk/src/main.rs @@ -45,7 +45,7 @@ impl WavWriter { * self.header.bit_depth as u32 * self.header.num_channels as u32); - let block_align = byte_rate / self.header.sample_rate as u32; + let block_align = byte_rate as u16 / self.header.sample_rate as u16; self.data.put_slice(b"RIFF"); self.data.put_u32_le(0); // Placeholder for file size @@ -56,7 +56,7 @@ impl WavWriter { self.data.put_u16_le(self.header.num_channels); self.data.put_u32_le(self.header.sample_rate); self.data.put_u32_le(byte_rate); - self.data.put_u16_le(32); + self.data.put_u16_le(block_align); self.data.put_u16_le(self.header.bit_depth); self.data.put_slice(b"data"); self.data.put_u32_le(0); // Placeholder for data size diff --git a/livekit-ffi/protocol/audio_frame.proto b/livekit-ffi/protocol/audio_frame.proto index ccedc2e..41d1197 100644 --- a/livekit-ffi/protocol/audio_frame.proto +++ b/livekit-ffi/protocol/audio_frame.proto @@ -36,6 +36,24 @@ message CaptureAudioFrameRequest { } message CaptureAudioFrameResponse {} +// Create a new AudioResampler +message NewAudioResamplerRequest {} +message NewAudioResamplerResponse { + FFIHandleId handle = 1; +} + +// Remix and resample an audio frame +message RemixAndResampleRequest { + FFIHandleId resampler_handle = 1; + FFIHandleId buffer_handle = 2; + uint32 num_channels = 3; + uint32 sample_rate = 4; +} + +message RemixAndResampleResponse { + FFIHandleId buffer_handle = 1; +} + /// /// AudioFrame buffer /// /// diff --git a/livekit-ffi/protocol/ffi.proto b/livekit-ffi/protocol/ffi.proto index ea75af7..df21e9f 100644 --- a/livekit-ffi/protocol/ffi.proto +++ b/livekit-ffi/protocol/ffi.proto @@ -40,6 +40,8 @@ message FFIRequest { NewAudioStreamRequest new_audio_stream = 16; NewAudioSourceRequest new_audio_source = 17; CaptureAudioFrameRequest capture_audio_frame = 18; + NewAudioResamplerRequest new_audio_resampler = 19; + RemixAndResampleRequest remix_and_resample = 20; } } @@ -72,6 +74,8 @@ message FFIResponse { NewAudioStreamResponse new_audio_stream = 16; NewAudioSourceResponse new_audio_source = 17; CaptureAudioFrameResponse capture_audio_frame = 18; + NewAudioResamplerResponse new_audio_resampler = 19; + RemixAndResampleResponse remix_and_resample = 20; } } diff --git a/livekit-ffi/src/server/mod.rs b/livekit-ffi/src/server/mod.rs index 7f0e2b6..d792fed 100644 --- a/livekit-ffi/src/server/mod.rs +++ b/livekit-ffi/src/server/mod.rs @@ -3,7 +3,7 @@ use crate::{FfiAsyncId, FfiError, FfiHandle, FfiHandleId, FfiResult}; use dashmap::DashMap; use lazy_static::lazy_static; use livekit::prelude::*; -use livekit::webrtc::native::yuv_helper; +use livekit::webrtc::native::{audio_resampler, yuv_helper}; use livekit::webrtc::prelude::*; use livekit::webrtc::video_frame::{native::I420BufferExt, BoxVideoFrameBuffer, I420Buffer}; use parking_lot::Mutex; @@ -11,6 +11,7 @@ use prost::Message; use std::collections::HashMap; use std::slice; use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; pub mod audio_frame; pub mod room; @@ -608,6 +609,83 @@ impl FfiServer { Ok(proto::CaptureAudioFrameResponse::default()) } + fn new_audio_resampler( + &'static self, + new_resampler: proto::NewAudioResamplerRequest, + ) -> FfiResult { + let resampler = audio_resampler::AudioResampler::default(); + let resampler = Arc::new(Mutex::new(resampler)); + + let handle_id = self.next_id() as FfiHandleId; + self.ffi_handles.insert(handle_id, Box::new(resampler)); + + Ok(proto::NewAudioResamplerResponse { + handle: Some(handle_id.into()), + }) + } + + fn remix_and_resample( + &'static self, + remix: proto::RemixAndResampleRequest, + ) -> FfiResult { + let resampler_id = remix + .resampler_handle + .as_ref() + .ok_or(FfiError::InvalidRequest("handle is empty"))? + .id as FfiHandleId; + + let resampler = self + .ffi_handles + .get(&resampler_id) + .ok_or(FfiError::InvalidRequest("resampler not found"))?; + + let resampler = resampler + .downcast_ref::>>() + .ok_or(FfiError::InvalidRequest("handle is not a resampler"))?; + + let resampler = resampler.clone(); + + let buffer_id = remix + .buffer_handle + .as_ref() + .ok_or(FfiError::InvalidRequest("handle is empty"))? + .id as FfiHandleId; + + let buffer = self + .ffi_handles + .get(&buffer_id) + .ok_or(FfiError::InvalidRequest("buffer not found"))?; + + let buffer = buffer + .downcast_ref::() + .ok_or(FfiError::InvalidRequest("handle is not a buffer"))?; + + let mut resampler = resampler.lock(); + let data = resampler.remix_and_resample( + &buffer.data, + buffer.samples_per_channel, + buffer.num_channels, + buffer.sample_rate, + remix.num_channels, + remix.sample_rate, + ); + + let samples_per_channel = data.len() / remix.num_channels as usize; + let frame = AudioFrame { + data: data.to_owned(), // Copy? + num_channels: remix.num_channels, + samples_per_channel: samples_per_channel as u32, + sample_rate: remix.sample_rate, + }; + + let handle_id = self.next_id() as FfiHandleId; + self.ffi_handles.insert(handle_id, Box::new(frame)); + + Ok(proto::RemixAndResampleResponse { + buffer_handle: Some(handle_id.into()), + }) + } + pub fn handle_request( &'static self, request: proto::FfiRequest, @@ -672,6 +750,12 @@ impl FfiServer { proto::ffi_request::Message::CaptureAudioFrame(push) => { proto::ffi_response::Message::CaptureAudioFrame(self.on_capture_audio_frame(push)?) } + proto::ffi_request::Message::NewAudioResampler(new_res) => { + proto::ffi_response::Message::NewAudioResampler(self.new_audio_resampler(new_res)?) + } + proto::ffi_request::Message::RemixAndResample(remix) => { + proto::ffi_response::Message::RemixAndResample(self.remix_and_resample(remix)?) + } }); Ok(res) diff --git a/livekit-webrtc/src/lib.rs b/livekit-webrtc/src/lib.rs index ee758f6..dac2efe 100644 --- a/livekit-webrtc/src/lib.rs +++ b/livekit-webrtc/src/lib.rs @@ -46,6 +46,7 @@ pub mod video_stream; #[cfg(not(target_arch = "wasm32"))] pub mod native { + pub use crate::imp::audio_resampler; pub use crate::imp::yuv_helper; pub use webrtc_sys::webrtc::ffi::create_random_uuid; } diff --git a/livekit-webrtc/src/native/audio_resampler.rs b/livekit-webrtc/src/native/audio_resampler.rs new file mode 100644 index 0000000..dc1eab8 --- /dev/null +++ b/livekit-webrtc/src/native/audio_resampler.rs @@ -0,0 +1,39 @@ +use cxx::UniquePtr; +use webrtc_sys::audio_resampler as sys_ar; + +pub struct AudioResampler { + sys_handle: UniquePtr, +} + +impl Default for AudioResampler { + fn default() -> Self { + Self { + sys_handle: sys_ar::ffi::create_audio_resampler(), + } + } +} + +impl AudioResampler { + pub fn remix_and_resample( + &mut self, + src: &[i16], + samples_per_channel: u32, + num_channels: u32, + sample_rate: u32, + dst_num_channels: u32, + dst_sample_rate: u32, + ) -> &[i16] { + unsafe { + let len = self.sys_handle.pin_mut().remix_and_resample( + src.as_ptr(), + samples_per_channel as usize, + num_channels as usize, + sample_rate as i32, + dst_num_channels as usize, + dst_sample_rate as i32, + ); + + std::slice::from_raw_parts(self.sys_handle.data(), len) + } + } +} diff --git a/livekit-webrtc/src/native/mod.rs b/livekit-webrtc/src/native/mod.rs index 6c5f63a..7db7bd0 100644 --- a/livekit-webrtc/src/native/mod.rs +++ b/livekit-webrtc/src/native/mod.rs @@ -1,3 +1,4 @@ +pub mod audio_resampler; pub mod audio_source; pub mod audio_stream; pub mod data_channel; diff --git a/webrtc-sys/.gitignore b/webrtc-sys/.gitignore deleted file mode 100644 index 3aa1f00..0000000 --- a/webrtc-sys/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/cmake-build-debug \ No newline at end of file diff --git a/webrtc-sys/build.rs b/webrtc-sys/build.rs index 36bfd36..75bcf7c 100644 --- a/webrtc-sys/build.rs +++ b/webrtc-sys/build.rs @@ -149,6 +149,7 @@ fn main() { "src/video_frame_buffer.rs", "src/yuv_helper.rs", "src/helper.rs", + "src/audio_resampler.rs", ]); builder.files(&[ @@ -169,6 +170,7 @@ fn main() { "src/video_encoder_factory.cpp", "src/video_decoder_factory.cpp", "src/audio_device.cpp", + "src/audio_resampler.cpp", ]); for include in includes { diff --git a/webrtc-sys/include/livekit/audio_resampler.h b/webrtc-sys/include/livekit/audio_resampler.h new file mode 100644 index 0000000..e088044 --- /dev/null +++ b/webrtc-sys/include/livekit/audio_resampler.h @@ -0,0 +1,47 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the “License”); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an “AS IS” BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "api/audio/audio_frame.h" +#include "api/data_channel_interface.h" +#include "common_audio/resampler/include/push_resampler.h" +#include "livekit/webrtc.h" +#include "rust/cxx.h" + +namespace livekit { + +class AudioResampler { + public: + size_t remix_and_resample(const int16_t* src, + size_t samples_per_channel, + size_t num_channels, + int sample_rate_hz, + size_t dest_num_channels, + int dest_sample_rate_hz); + + const int16_t* data() const; + + private: + webrtc::AudioFrame frame_; + webrtc::PushResampler resampler_; +}; + +std::unique_ptr create_audio_resampler(); + +} // namespace livekit diff --git a/webrtc-sys/src/audio_resampler.cpp b/webrtc-sys/src/audio_resampler.cpp new file mode 100644 index 0000000..6b0e0ad --- /dev/null +++ b/webrtc-sys/src/audio_resampler.cpp @@ -0,0 +1,47 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the “License”); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an “AS IS” BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "livekit/audio_resampler.h" + +#include + +#include "audio/remix_resample.h" + +namespace livekit { + +size_t AudioResampler::remix_and_resample(const int16_t* src, + size_t samples_per_channel, + size_t num_channels, + int sample_rate, + size_t dest_num_channels, + int dest_sample_rate) { + frame_.num_channels_ = dest_num_channels; + frame_.sample_rate_hz_ = dest_sample_rate; + webrtc::voe::RemixAndResample(src, samples_per_channel, num_channels, + sample_rate, &resampler_, &frame_); + + return frame_.num_channels() * frame_.samples_per_channel(); +} + +const int16_t* AudioResampler::data() const { + return frame_.data(); +} + +std::unique_ptr create_audio_resampler() { + return std::make_unique(); +} + +} // namespace livekit \ No newline at end of file diff --git a/webrtc-sys/src/audio_resampler.rs b/webrtc-sys/src/audio_resampler.rs new file mode 100644 index 0000000..42ecf79 --- /dev/null +++ b/webrtc-sys/src/audio_resampler.rs @@ -0,0 +1,26 @@ +use crate::impl_thread_safety; + +#[cxx::bridge(namespace = "livekit")] +pub mod ffi { + unsafe extern "C++" { + include!("livekit/audio_resampler.h"); + + type AudioResampler; + + unsafe fn remix_and_resample( + self: Pin<&mut AudioResampler>, + src: *const i16, + samples_per_channel: usize, + num_channels: usize, + sample_rate: i32, + dst_num_channels: usize, + dst_sample_rate: i32, + ) -> usize; + + unsafe fn data(self: &AudioResampler) -> *const i16; + + fn create_audio_resampler() -> UniquePtr; + } +} + +impl_thread_safety!(ffi::AudioResampler, Send + Sync); diff --git a/webrtc-sys/src/lib.rs b/webrtc-sys/src/lib.rs index 19350bd..0efc7eb 100644 --- a/webrtc-sys/src/lib.rs +++ b/webrtc-sys/src/lib.rs @@ -1,3 +1,4 @@ +pub mod audio_resampler; pub mod candidate; pub mod data_channel; pub mod helper;