feat: audio support (#45)

- Receive/Send audio frames
- LocalAudioTrack

livekit-webrtc/src/native/audio_source.rs

@@ -0,0 +1,34 @@
+use crate::audio_frame::AudioFrame;
+use cxx::SharedPtr;
+use webrtc_sys::media_stream as sys_ms;
+
+#[derive(Clone)]
+pub struct NativeAudioSource {
+    sys_handle: SharedPtr<sys_ms::ffi::AudioTrackSource>,
+}
+
+impl Default for NativeAudioSource {
+    fn default() -> Self {
+        Self {
+            sys_handle: sys_ms::ffi::new_audio_track_source(),
+        }
+    }
+}
+
+impl NativeAudioSource {
+    pub fn sys_handle(&self) -> SharedPtr<sys_ms::ffi::AudioTrackSource> {
+        self.sys_handle.clone()
+    }
+
+    pub fn capture_frame(&self, frame: AudioFrame) {
+        // TODO(theomonnom): Should we check for 10ms worth of data here?
+        unsafe {
+            self.sys_handle.on_captured_frame(
+                frame.data.as_ptr(),
+                frame.sample_rate_hz as i32,
+                frame.num_channels,
+                frame.samples_per_channel,
+            )
+        }
+    }
+}

livekit-webrtc/src/native/audio_stream.rs

@@ -0,0 +1,80 @@
+use crate::{audio_frame::AudioFrame, media_stream::RtcAudioTrack};
+use cxx::UniquePtr;
+use futures::stream::Stream;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+use tokio::sync::mpsc;
+use webrtc_sys::media_stream as sys_ms;
+
+pub struct NativeAudioStream {
+    native_observer: UniquePtr<sys_ms::ffi::NativeAudioSink>,
+    _observer: Box<AudioTrackObserver>,
+    audio_track: RtcAudioTrack,
+    frame_rx: mpsc::UnboundedReceiver<AudioFrame>,
+}
+
+impl NativeAudioStream {
+    pub fn new(audio_track: RtcAudioTrack) -> Self {
+        let (frame_tx, frame_rx) = mpsc::unbounded_channel();
+        let mut observer = Box::new(AudioTrackObserver { frame_tx });
+        let mut native_observer = unsafe {
+            sys_ms::ffi::new_native_audio_sink(Box::new(sys_ms::AudioSinkWrapper::new(
+                &mut *observer,
+            )))
+        };
+
+        unsafe {
+            sys_ms::ffi::media_to_audio(audio_track.sys_handle())
+                .add_sink(native_observer.pin_mut());
+        }
+
+        Self {
+            native_observer,
+            _observer: observer,
+            audio_track,
+            frame_rx,
+        }
+    }
+
+    pub fn track(&self) -> RtcAudioTrack {
+        self.audio_track.clone()
+    }
+
+    pub fn close(&mut self) {
+        self.frame_rx.close();
+        unsafe {
+            sys_ms::ffi::media_to_audio(self.audio_track.sys_handle())
+                .remove_sink(self.native_observer.pin_mut());
+        }
+    }
+}
+
+impl Drop for NativeAudioStream {
+    fn drop(&mut self) {
+        self.close();
+    }
+}
+
+impl Stream for NativeAudioStream {
+    type Item = AudioFrame;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
+        self.frame_rx.poll_recv(cx)
+    }
+}
+
+pub struct AudioTrackObserver {
+    frame_tx: mpsc::UnboundedSender<AudioFrame>,
+}
+
+impl sys_ms::AudioSink for AudioTrackObserver {
+    fn on_data(&self, data: &[i16], sample_rate: i32, nb_channels: usize, nb_frames: usize) {
+        // TODO(theomonnom): Should we avoid copy here?
+        let _ = self.frame_tx.send(AudioFrame {
+            data: data.to_owned(),
+            sample_rate_hz: sample_rate as u32,
+            num_channels: nb_channels,
+            samples_per_channel: nb_frames,
+        });
+    }
+}

livekit-webrtc/src/native/mod.rs

@@ -1,3 +1,5 @@
+pub mod audio_source;
+pub mod audio_stream;
 pub mod data_channel;
 pub mod ice_candidate;
 pub mod media_stream;

livekit-webrtc/src/native/peer_connection_factory.rs

@@ -1,6 +1,7 @@
+use crate::audio_source::native::NativeAudioSource;
 use crate::imp::media_stream as imp_ms;
 use crate::imp::peer_connection as imp_pc;
-use crate::media_stream::RtcVideoTrack;
+use crate::media_stream::{RtcAudioTrack, RtcVideoTrack};
 use crate::peer_connection::PeerConnection;
 use crate::peer_connection_factory::{
     ContinualGatheringPolicy, IceServer, IceTransportsType, RtcConfiguration,
@@ -132,6 +133,16 @@ impl PeerConnectionFactory {
         }
     }
 
+    pub fn create_audio_track(&self, label: &str, source: NativeAudioSource) -> RtcAudioTrack {
+        RtcAudioTrack {
+            handle: imp_ms::RtcAudioTrack {
+                sys_handle: self
+                    .sys_handle
+                    .create_audio_track(label.to_string(), source.handle.sys_handle()),
+            },
+        }
+    }
+
     pub fn get_rtp_sender_capabilities(&self, media_type: MediaType) -> RtpCapabilities {
         self.sys_handle
             .get_rtp_sender_capabilities(media_type.into())

livekit-webrtc/src/native/rtp_transceiver.rs

@@ -19,6 +19,7 @@ impl From<sys_webrtc::ffi::RtpTransceiverDirection> for RtpTransceiverDirection
             sys_webrtc::ffi::RtpTransceiverDirection::SendOnly => Self::SendOnly,
             sys_webrtc::ffi::RtpTransceiverDirection::RecvOnly => Self::RecvOnly,
             sys_webrtc::ffi::RtpTransceiverDirection::Inactive => Self::Inactive,
+            sys_webrtc::ffi::RtpTransceiverDirection::Stopped => Self::Stopped,
             _ => panic!("unknown RtpTransceiverDirection"),
         }
     }
@@ -31,6 +32,7 @@ impl From<RtpTransceiverDirection> for sys_webrtc::ffi::RtpTransceiverDirection
             RtpTransceiverDirection::SendOnly => Self::SendOnly,
             RtpTransceiverDirection::RecvOnly => Self::RecvOnly,
             RtpTransceiverDirection::Inactive => Self::Inactive,
+            RtpTransceiverDirection::Stopped => Self::Stopped,
             _ => panic!("unknown RtpTransceiverDirection"),
         }
     }