feat: audio support (#45)

- Receive/Send audio frames
- LocalAudioTrack

livekit/src/room/options.rs

@@ -40,38 +40,63 @@ pub struct VideoPreset {
     pub height: u32,
 }
 
+#[derive(Debug, Clone)]
+pub struct AudioEncoding {
+    pub max_bitrate: u64,
+}
+
 #[derive(Debug, Clone)]
 pub struct AudioPreset {
-    pub max_bitrate: u32,
+    pub encoding: AudioEncoding,
 }
 
 impl AudioPreset {
-    pub const fn new(max_bitrate: u32) -> Self {
-        Self { max_bitrate }
+    pub const fn new(max_bitrate: u64) -> Self {
+        Self {
+            encoding: AudioEncoding { max_bitrate },
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct AudioCaptureOptions {
+    pub echo_cancellation: bool,
+    pub noise_suppression: bool,
+    pub auto_gain_control: bool,
+}
+
+impl Default for AudioCaptureOptions {
+    fn default() -> Self {
+        Self {
+            echo_cancellation: true,
+            noise_suppression: true,
+            auto_gain_control: true,
+        }
     }
 }
 
 #[derive(Clone, Debug)]
 pub struct VideoCaptureOptions {
-    pub preset: VideoPreset,
+    pub resolution: VideoResolution,
 }
 
 impl Default for VideoCaptureOptions {
     fn default() -> Self {
         Self {
-            preset: video::H720,
+            resolution: video::H720.resolution(),
         }
     }
 }
 
 #[derive(Clone, Debug)]
 pub struct TrackPublishOptions {
-    pub dynacast: bool,
+    // If the encodings aren't set, LiveKit will compute the most appropriate ones
+    pub video_encoding: Option<VideoEncoding>,
+    pub audio_encoding: Option<AudioEncoding>,
     pub video_codec: VideoCodec,
     pub dtx: bool,
     pub red: bool,
     pub simulcast: bool,
-    pub screenshare: bool,
     pub name: String,
     pub source: TrackSource,
 }
@@ -79,12 +104,12 @@ pub struct TrackPublishOptions {
 impl Default for TrackPublishOptions {
     fn default() -> Self {
         Self {
-            dynacast: false,
+            video_encoding: None,
+            audio_encoding: None,
             video_codec: VideoCodec::VP8,
             dtx: true,
             red: true,
             simulcast: true,
-            screenshare: false,
             name: "unnamed track".to_owned(),
             source: TrackSource::Unknown,
         }
@@ -120,7 +145,8 @@ pub fn compute_video_encodings(
     height: u32,
     options: &TrackPublishOptions,
 ) -> Vec<RtpEncodingParameters> {
-    let encoding = compute_appropriate_encoding(options.screenshare, width, height);
+    let screenshare = options.source == TrackSource::Screenshare;
+    let encoding = compute_appropriate_encoding(screenshare, width, height);
 
     let initial_preset = VideoPreset {
         width,
@@ -135,8 +161,7 @@ pub fn compute_video_encodings(
         return into_rtp_encodings(width, height, &[initial_preset]);
     }
 
-    let mut simulcast_presets =
-        compute_default_simulcast_presets(options.screenshare, &initial_preset);
+    let mut simulcast_presets = compute_default_simulcast_presets(screenshare, &initial_preset);
 
     let mid_preset = simulcast_presets.pop();
     let low_preset = simulcast_presets.pop();

livekit/src/room/participant/local_participant.rs

@@ -1,10 +1,12 @@
 use super::{ConnectionQuality, ParticipantInner};
+use crate::options;
 use crate::options::compute_video_encodings;
 use crate::options::video_layers_from_encodings;
 use crate::options::TrackPublishOptions;
 use crate::prelude::*;
 use crate::proto;
 use crate::rtc_engine::RtcEngine;
+use livekit_webrtc::rtp_parameters::RtpEncodingParameters;
 use parking_lot::RwLockReadGuard;
 use std::collections::HashMap;
 use std::sync::Arc;
@@ -53,13 +55,24 @@ impl LocalParticipant {
                 // Get the video dimension
                 // TODO(theomonnom): Use MediaStreamTrack::getSettings() on web
                 let capture_options = video_track.capture_options();
-                req.width = capture_options.preset.width;
-                req.height = capture_options.preset.height;
+                req.width = capture_options.resolution.width;
+                req.height = capture_options.resolution.height;
 
                 encodings = compute_video_encodings(req.width, req.height, &options);
                 req.layers = video_layers_from_encodings(req.width, req.height, &encodings);
             }
-            LocalTrack::Audio(_audio_track) => {}
+            LocalTrack::Audio(_audio_track) => {
+                // Setup audio encoding
+                let audio_encoding = options
+                    .audio_encoding
+                    .as_ref()
+                    .unwrap_or(&options::audio::SPEECH.encoding);
+
+                encodings.push(RtpEncodingParameters {
+                    max_bitrate: Some(audio_encoding.max_bitrate),
+                    ..Default::default()
+                });
+            }
         }
 
         let track_info = self.rtc_engine.add_track(req).await?;

livekit/src/room/track/local_audio_track.rs

@@ -1,79 +1,100 @@
 use super::TrackInner;
+use crate::options::AudioCaptureOptions;
 use crate::prelude::*;
 use crate::proto;
+use crate::rtc_engine::lk_runtime::LkRuntime;
+use crate::webrtc::peer_connection_factory::native::PeerConnectionFactoryExt;
 use livekit_webrtc as rtc;
+use parking_lot::Mutex;
+use rtc::audio_source::native::NativeAudioSource;
 use std::sync::Arc;
 use tokio::sync::mpsc;
 
+#[derive(Debug)]
+pub struct LocalAudioTrackInner {
+    track_inner: TrackInner,
+    capture_options: Mutex<AudioCaptureOptions>,
+}
+
 #[derive(Clone, Debug)]
 pub struct LocalAudioTrack {
-    pub(crate) inner: Arc<TrackInner>,
+    inner: Arc<LocalAudioTrackInner>,
 }
 
 impl LocalAudioTrack {
     pub(crate) fn new(
-        sid: TrackSid,
         name: String,
         rtc_track: rtc::media_stream::RtcAudioTrack,
+        capture_options: AudioCaptureOptions,
     ) -> Self {
         Self {
-            inner: Arc::new(TrackInner::new(
-                sid,
-                name,
-                TrackKind::Audio,
-                rtc::media_stream::MediaStreamTrack::Audio(rtc_track),
-            )),
+            inner: Arc::new(LocalAudioTrackInner {
+                track_inner: TrackInner::new(
+                    "unknown".to_string().into(), // sid
+                    name,
+                    TrackKind::Audio,
+                    rtc::media_stream::MediaStreamTrack::Audio(rtc_track),
+                ),
+                capture_options: Mutex::new(capture_options),
+            }),
         }
     }
 
+    #[inline]
+    pub fn capture_options(&self) -> AudioCaptureOptions {
+        self.inner.capture_options.lock().clone()
+    }
+
     #[inline]
     pub fn sid(&self) -> TrackSid {
-        self.inner.sid()
+        self.inner.track_inner.sid()
     }
 
     #[inline]
     pub fn name(&self) -> String {
-        self.inner.name()
+        self.inner.track_inner.name()
     }
 
     #[inline]
     pub fn kind(&self) -> TrackKind {
-        self.inner.kind()
+        self.inner.track_inner.kind()
     }
 
     #[inline]
     pub fn source(&self) -> TrackSource {
-        self.inner.source()
+        self.inner.track_inner.source()
     }
 
     #[inline]
     pub fn stream_state(&self) -> StreamState {
-        self.inner.stream_state()
+        self.inner.track_inner.stream_state()
     }
 
     #[inline]
     pub fn start(&self) {
-        self.inner.start()
+        self.inner.track_inner.start()
     }
 
     #[inline]
     pub fn stop(&self) {
-        self.inner.stop()
+        self.inner.track_inner.stop()
     }
 
     #[inline]
     pub fn muted(&self) -> bool {
-        self.inner.muted()
+        self.inner.track_inner.muted()
     }
 
     #[inline]
     pub fn set_muted(&self, muted: bool) {
-        self.inner.set_muted(muted)
+        self.inner.track_inner.set_muted(muted)
     }
 
     #[inline]
     pub fn rtc_track(&self) -> rtc::media_stream::RtcAudioTrack {
-        if let rtc::media_stream::MediaStreamTrack::Audio(audio) = self.inner.rtc_track() {
+        if let rtc::media_stream::MediaStreamTrack::Audio(audio) =
+            self.inner.track_inner.rtc_track()
+        {
             audio
         } else {
             unreachable!()
@@ -82,12 +103,12 @@ impl LocalAudioTrack {
 
     #[inline]
     pub fn register_observer(&self) -> mpsc::UnboundedReceiver<TrackEvent> {
-        self.inner.register_observer()
+        self.inner.track_inner.register_observer()
     }
 
     #[inline]
     pub(crate) fn transceiver(&self) -> Option<rtc::rtp_transceiver::RtpTransceiver> {
-        self.inner.transceiver()
+        self.inner.track_inner.transceiver()
     }
 
     #[inline]
@@ -95,11 +116,25 @@ impl LocalAudioTrack {
         &self,
         transceiver: Option<rtc::rtp_transceiver::RtpTransceiver>,
     ) {
-        self.inner.update_transceiver(transceiver)
+        self.inner.track_inner.update_transceiver(transceiver)
     }
 
     #[inline]
     pub(crate) fn update_info(&self, info: proto::TrackInfo) {
-        self.inner.update_info(info)
+        self.inner.track_inner.update_info(info)
+    }
+}
+
+impl LocalAudioTrack {
+    pub fn create_audio_track(
+        name: &str,
+        options: AudioCaptureOptions,
+        source: NativeAudioSource,
+    ) -> LocalAudioTrack {
+        let rtc_track = LkRuntime::instance()
+            .pc_factory
+            .create_audio_track(&rtc::native::create_random_uuid(), source);
+
+        Self::new(name.to_string(), rtc_track, options)
     }
 }

livekit/src/room/track/local_video_track.rs

@@ -39,6 +39,7 @@ impl LocalVideoTrack {
         }
     }
 
+    #[inline]
     pub fn capture_options(&self) -> VideoCaptureOptions {
         self.inner.capture_options.lock().clone()
     }

livekit/src/rtc_engine/rtc_session.rs

@@ -1,5 +1,6 @@
 use super::{rtc_events, EngineError, EngineResult, SimulateScenario};
 use crate::options::TrackPublishOptions;
+use crate::prelude::TrackKind;
 use crate::rtc_engine::lk_runtime::LkRuntime;
 use crate::rtc_engine::peer_transport::PeerTransport;
 use crate::rtc_engine::rtc_events::{RtcEvent, RtcEvents};
@@ -652,38 +653,38 @@ impl SessionInner {
             .peer_connection()
             .add_transceiver(track.rtc_track(), init)?;
 
-        let capabilities = LkRuntime::instance()
-            .pc_factory
-            .get_rtp_sender_capabilities(track.kind().into());
+        if track.kind() == TrackKind::Video {
+            let capabilities = LkRuntime::instance()
+                .pc_factory
+                .get_rtp_sender_capabilities(track.kind().into());
 
-        let mut matched = Vec::new();
-        let mut partial_matched = Vec::new();
-        let mut unmatched = Vec::new();
+            let mut matched = Vec::new();
+            let mut partial_matched = Vec::new();
+            let mut unmatched = Vec::new();
 
-        for codec in capabilities.codecs {
-            let mime_type = codec.mime_type.to_lowercase();
-            if mime_type == "audio/opus" {
-                matched.push(codec);
-            } else if mime_type == format!("video/{}", options.video_codec.as_str()) {
-                if let Some(sdp_fmtp_line) = codec.sdp_fmtp_line.as_ref() {
-                    // for h264 codecs that have sdpFmtpLine available, use only if the
-                    // profile-level-id is 42e01f for cross-browser compatibility
-                    if sdp_fmtp_line.contains("profile-level-id=42e01f") {
-                        matched.push(codec);
-                        continue;
+            for codec in capabilities.codecs {
+                let mime_type = codec.mime_type.to_lowercase();
+                if mime_type == format!("video/{}", options.video_codec.as_str()) {
+                    if let Some(sdp_fmtp_line) = codec.sdp_fmtp_line.as_ref() {
+                        // for h264 codecs that have sdpFmtpLine available, use only if the
+                        // profile-level-id is 42e01f for cross-browser compatibility
+                        if sdp_fmtp_line.contains("profile-level-id=42e01f") {
+                            matched.push(codec);
+                            continue;
+                        }
                     }
+                    partial_matched.push(codec);
+                } else {
+                    unmatched.push(codec);
                 }
-                partial_matched.push(codec);
-            } else {
-                unmatched.push(codec);
             }
+
+            matched.append(&mut partial_matched);
+            matched.append(&mut unmatched);
+
+            transceiver.set_codec_preferences(matched)?;
         }
 
-        matched.append(&mut partial_matched);
-        matched.append(&mut unmatched);
-
-        transceiver.set_codec_preferences(matched)?;
-
         Ok(transceiver)
     }