feat: audio support (#45)

- Receive/Send audio frames
- LocalAudioTrack

livekit-webrtc/src/audio_frame.rs

@@ -0,0 +1,7 @@
+#[derive(Debug, Clone)]
+pub struct AudioFrame {
+    pub data: Vec<i16>,
+    pub sample_rate_hz: u32,
+    pub num_channels: usize,
+    pub samples_per_channel: usize,
+}

livekit-webrtc/src/audio_source.rs

@@ -0,0 +1,25 @@
+use crate::imp::audio_source as imp_as;
+
+#[cfg(not(target_arch = "wasm32"))]
+pub mod native {
+    use super::imp_as;
+    use crate::audio_frame::AudioFrame;
+    use std::fmt::{Debug, Formatter};
+
+    #[derive(Default, Clone)]
+    pub struct NativeAudioSource {
+        pub(crate) handle: imp_as::NativeAudioSource,
+    }
+
+    impl Debug for NativeAudioSource {
+        fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
+            f.debug_struct("NativeAudioSource").finish()
+        }
+    }
+
+    impl NativeAudioSource {
+        pub fn capture_frame(&self, frame: AudioFrame) {
+            self.handle.capture_frame(frame)
+        }
+    }
+}

livekit-webrtc/src/audio_stream.rs

@@ -0,0 +1,48 @@
+use crate::imp::audio_stream as stream_imp;
+
+#[cfg(not(target_arch = "wasm32"))]
+pub mod native {
+    use super::stream_imp;
+    use crate::audio_frame::AudioFrame;
+    use crate::media_stream::RtcAudioTrack;
+    use futures::stream::Stream;
+    use std::fmt::{Debug, Formatter};
+    use std::pin::Pin;
+    use std::task::{Context, Poll};
+
+    pub struct NativeAudioStream {
+        pub(crate) handle: stream_imp::NativeAudioStream,
+    }
+
+    impl Debug for NativeAudioStream {
+        fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
+            f.debug_struct("NativeAudioStream")
+                .field("track", &self.track())
+                .finish()
+        }
+    }
+
+    impl NativeAudioStream {
+        pub fn new(audio_track: RtcAudioTrack) -> Self {
+            Self {
+                handle: stream_imp::NativeAudioStream::new(audio_track),
+            }
+        }
+
+        pub fn track(&self) -> RtcAudioTrack {
+            self.handle.track()
+        }
+
+        pub fn close(&mut self) {
+            self.handle.close()
+        }
+    }
+
+    impl Stream for NativeAudioStream {
+        type Item = AudioFrame;
+
+        fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
+            Pin::new(&mut self.get_mut().handle).poll_next(cx)
+        }
+    }
+}

livekit-webrtc/src/lib.rs

@@ -26,6 +26,9 @@ pub struct RtcError {
     pub message: String,
 }
 
+pub mod audio_frame;
+pub mod audio_source;
+pub mod audio_stream;
 pub mod data_channel;
 pub mod ice_candidate;
 pub mod media_stream;

livekit-webrtc/src/native/audio_source.rs

@@ -0,0 +1,34 @@
+use crate::audio_frame::AudioFrame;
+use cxx::SharedPtr;
+use webrtc_sys::media_stream as sys_ms;
+
+#[derive(Clone)]
+pub struct NativeAudioSource {
+    sys_handle: SharedPtr<sys_ms::ffi::AudioTrackSource>,
+}
+
+impl Default for NativeAudioSource {
+    fn default() -> Self {
+        Self {
+            sys_handle: sys_ms::ffi::new_audio_track_source(),
+        }
+    }
+}
+
+impl NativeAudioSource {
+    pub fn sys_handle(&self) -> SharedPtr<sys_ms::ffi::AudioTrackSource> {
+        self.sys_handle.clone()
+    }
+
+    pub fn capture_frame(&self, frame: AudioFrame) {
+        // TODO(theomonnom): Should we check for 10ms worth of data here?
+        unsafe {
+            self.sys_handle.on_captured_frame(
+                frame.data.as_ptr(),
+                frame.sample_rate_hz as i32,
+                frame.num_channels,
+                frame.samples_per_channel,
+            )
+        }
+    }
+}

livekit-webrtc/src/native/audio_stream.rs

@@ -0,0 +1,80 @@
+use crate::{audio_frame::AudioFrame, media_stream::RtcAudioTrack};
+use cxx::UniquePtr;
+use futures::stream::Stream;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+use tokio::sync::mpsc;
+use webrtc_sys::media_stream as sys_ms;
+
+pub struct NativeAudioStream {
+    native_observer: UniquePtr<sys_ms::ffi::NativeAudioSink>,
+    _observer: Box<AudioTrackObserver>,
+    audio_track: RtcAudioTrack,
+    frame_rx: mpsc::UnboundedReceiver<AudioFrame>,
+}
+
+impl NativeAudioStream {
+    pub fn new(audio_track: RtcAudioTrack) -> Self {
+        let (frame_tx, frame_rx) = mpsc::unbounded_channel();
+        let mut observer = Box::new(AudioTrackObserver { frame_tx });
+        let mut native_observer = unsafe {
+            sys_ms::ffi::new_native_audio_sink(Box::new(sys_ms::AudioSinkWrapper::new(
+                &mut *observer,
+            )))
+        };
+
+        unsafe {
+            sys_ms::ffi::media_to_audio(audio_track.sys_handle())
+                .add_sink(native_observer.pin_mut());
+        }
+
+        Self {
+            native_observer,
+            _observer: observer,
+            audio_track,
+            frame_rx,
+        }
+    }
+
+    pub fn track(&self) -> RtcAudioTrack {
+        self.audio_track.clone()
+    }
+
+    pub fn close(&mut self) {
+        self.frame_rx.close();
+        unsafe {
+            sys_ms::ffi::media_to_audio(self.audio_track.sys_handle())
+                .remove_sink(self.native_observer.pin_mut());
+        }
+    }
+}
+
+impl Drop for NativeAudioStream {
+    fn drop(&mut self) {
+        self.close();
+    }
+}
+
+impl Stream for NativeAudioStream {
+    type Item = AudioFrame;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
+        self.frame_rx.poll_recv(cx)
+    }
+}
+
+pub struct AudioTrackObserver {
+    frame_tx: mpsc::UnboundedSender<AudioFrame>,
+}
+
+impl sys_ms::AudioSink for AudioTrackObserver {
+    fn on_data(&self, data: &[i16], sample_rate: i32, nb_channels: usize, nb_frames: usize) {
+        // TODO(theomonnom): Should we avoid copy here?
+        let _ = self.frame_tx.send(AudioFrame {
+            data: data.to_owned(),
+            sample_rate_hz: sample_rate as u32,
+            num_channels: nb_channels,
+            samples_per_channel: nb_frames,
+        });
+    }
+}

livekit-webrtc/src/native/mod.rs

@@ -1,3 +1,5 @@
+pub mod audio_source;
+pub mod audio_stream;
 pub mod data_channel;
 pub mod ice_candidate;
 pub mod media_stream;

livekit-webrtc/src/native/peer_connection_factory.rs

@@ -1,6 +1,7 @@
+use crate::audio_source::native::NativeAudioSource;
 use crate::imp::media_stream as imp_ms;
 use crate::imp::peer_connection as imp_pc;
-use crate::media_stream::RtcVideoTrack;
+use crate::media_stream::{RtcAudioTrack, RtcVideoTrack};
 use crate::peer_connection::PeerConnection;
 use crate::peer_connection_factory::{
     ContinualGatheringPolicy, IceServer, IceTransportsType, RtcConfiguration,
@@ -132,6 +133,16 @@ impl PeerConnectionFactory {
         }
     }
 
+    pub fn create_audio_track(&self, label: &str, source: NativeAudioSource) -> RtcAudioTrack {
+        RtcAudioTrack {
+            handle: imp_ms::RtcAudioTrack {
+                sys_handle: self
+                    .sys_handle
+                    .create_audio_track(label.to_string(), source.handle.sys_handle()),
+            },
+        }
+    }
+
     pub fn get_rtp_sender_capabilities(&self, media_type: MediaType) -> RtpCapabilities {
         self.sys_handle
             .get_rtp_sender_capabilities(media_type.into())

livekit-webrtc/src/native/rtp_transceiver.rs

@@ -19,6 +19,7 @@ impl From<sys_webrtc::ffi::RtpTransceiverDirection> for RtpTransceiverDirection
             sys_webrtc::ffi::RtpTransceiverDirection::SendOnly => Self::SendOnly,
             sys_webrtc::ffi::RtpTransceiverDirection::RecvOnly => Self::RecvOnly,
             sys_webrtc::ffi::RtpTransceiverDirection::Inactive => Self::Inactive,
+            sys_webrtc::ffi::RtpTransceiverDirection::Stopped => Self::Stopped,
             _ => panic!("unknown RtpTransceiverDirection"),
         }
     }
@@ -31,6 +32,7 @@ impl From<RtpTransceiverDirection> for sys_webrtc::ffi::RtpTransceiverDirection
             RtpTransceiverDirection::SendOnly => Self::SendOnly,
             RtpTransceiverDirection::RecvOnly => Self::RecvOnly,
             RtpTransceiverDirection::Inactive => Self::Inactive,
+            RtpTransceiverDirection::Stopped => Self::Stopped,
             _ => panic!("unknown RtpTransceiverDirection"),
         }
     }

livekit-webrtc/src/peer_connection_factory.rs

@@ -63,16 +63,22 @@ impl PeerConnectionFactory {
 
 pub mod native {
     use super::PeerConnectionFactory;
-    use crate::media_stream::RtcVideoTrack;
+    use crate::audio_source::native::NativeAudioSource;
+    use crate::media_stream::{RtcAudioTrack, RtcVideoTrack};
     use crate::video_source::native::NativeVideoSource;
 
     pub trait PeerConnectionFactoryExt {
         fn create_video_track(&self, label: &str, source: NativeVideoSource) -> RtcVideoTrack;
+        fn create_audio_track(&self, label: &str, source: NativeAudioSource) -> RtcAudioTrack;
     }
 
     impl PeerConnectionFactoryExt for PeerConnectionFactory {
         fn create_video_track(&self, label: &str, source: NativeVideoSource) -> RtcVideoTrack {
             self.handle.create_video_track(label, source)
         }
+
+        fn create_audio_track(&self, label: &str, source: NativeAudioSource) -> RtcAudioTrack {
+            self.handle.create_audio_track(label, source)
+        }
     }
 }

livekit-webrtc/src/prelude.rs

@@ -1,3 +1,4 @@
+pub use crate::audio_frame::AudioFrame;
 pub use crate::data_channel::{
     DataBuffer, DataChannel, DataChannelError, DataChannelInit, DataState,
 };