feat: audio support (#45)

- Receive/Send audio frames
- LocalAudioTrack

examples/simple_room/src/app.rs

@@ -1,12 +1,15 @@
 use crate::events::UiCmd;
 use crate::logo_track::LogoTrack;
+use crate::sine_track::SineTrack;
 use crate::video_renderer::VideoRenderer;
 use crate::{events::AsyncCmd, video_grid::VideoGrid};
 use egui::{Rounding, Stroke};
 use egui_wgpu::WgpuConfiguration;
+use futures::StreamExt;
 use image::ImageFormat;
 use livekit::options::{TrackPublishOptions, VideoCaptureOptions};
 use livekit::prelude::*;
+use livekit::webrtc::audio_stream::native::NativeAudioStream;
 use livekit::webrtc::native::yuv_helper;
 use livekit::webrtc::video_frame::native::I420BufferExt;
 use livekit::webrtc::video_frame::{I420Buffer, VideoFrame, VideoRotation};
@@ -36,6 +39,7 @@ use winit::{
 struct Session {
     room: Room,
     logo_track: LogoTrack,
+    sine_track: SineTrack,
     close_tx: oneshot::Sender<()>,
     handle: tokio::task::JoinHandle<()>,
 }
@@ -113,6 +117,7 @@ pub fn run(rt: tokio::runtime::Runtime) {
                         if let Ok((room, room_events)) = res {
                             let (close_tx, close_rx) = oneshot::channel();
                             let logo_track = LogoTrack::new(room.session());
+                            let sine_track = SineTrack::new(room.session());
                             let handle = tokio::spawn(room_task(
                                 state.clone(),
                                 room_events,
@@ -123,6 +128,7 @@ pub fn run(rt: tokio::runtime::Runtime) {
                             *state.session.lock() = Some(Session {
                                 room,
                                 logo_track,
+                                sine_track,
                                 close_tx,
                                 handle,
                             });
@@ -156,6 +162,12 @@ pub fn run(rt: tokio::runtime::Runtime) {
                             }
                         }
                     }
+                    AsyncCmd::ToggleSine => {
+                        if let Some(session) = state.session.lock().as_mut() {
+                            let sine_track = &mut session.sine_track;
+                            sine_track.publish().await.unwrap();
+                        }
+                    }
                 }
             }
         });
@@ -213,8 +225,15 @@ impl App {
                                     self.video_renderers
                                         .insert((participant.sid(), track.sid()), video_renderer);
                                 }
-                                RemoteTrack::Audio(_) => {
-                                    // The demo doesn't support Audio rendering at the moment.
+                                RemoteTrack::Audio(audio_track) => {
+                                    tokio::spawn(async move {
+                                        let mut stream =
+                                            NativeAudioStream::new(audio_track.rtc_track());
+
+                                        while let Some(_frame) = stream.next().await {
+                                            // Received audio frames
+                                        }
+                                    });
                                 }
                             };
                         }
@@ -323,9 +342,12 @@ impl App {
                 });
 
                 ui.menu_button("Publish", |ui| {
-                    if ui.button("CustomTrack - LK Logo").clicked() {
+                    if ui.button("Logo").clicked() {
                         let _ = self.cmd_tx.send(AsyncCmd::ToggleLogo);
                     }
+                    if ui.button("SineWave").clicked() {
+                        let _ = self.cmd_tx.send(AsyncCmd::ToggleSine);
+                    }
                 });
             });
         });

examples/simple_room/src/events.rs

@@ -7,6 +7,7 @@ pub enum AsyncCmd {
     RoomDisconnect,
     SimulateScenario { scenario: SimulateScenario },
     ToggleLogo, // Unpublish/Publish a logo track
+    ToggleSine,
 }
 
 #[derive(Debug)]

examples/simple_room/src/logo_track.rs

@@ -56,7 +56,7 @@ impl LogoTrack {
     }
 
     pub async fn publish(&mut self) -> Result<(), RoomError> {
-        self.unpublish().await;
+        self.unpublish().await?;
 
         let (close_tx, close_rx) = oneshot::channel();
         let track = LocalVideoTrack::create_video_track(

examples/simple_room/src/main.rs

@@ -1,6 +1,7 @@
 mod app;
 mod events;
 mod logo_track;
+mod sine_track;
 mod video_grid;
 mod video_renderer;
 

examples/simple_room/src/sine_track.rs

@@ -0,0 +1,127 @@
+use livekit::options::{AudioCaptureOptions, TrackPublishOptions};
+use livekit::webrtc::audio_frame::AudioFrame;
+use livekit::{prelude::*, webrtc::audio_source::native::NativeAudioSource};
+use parking_lot::Mutex;
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::oneshot;
+use tokio::task::JoinHandle;
+
+#[derive(Clone)]
+struct FrameData {
+    pub sample_rate: u32,
+    pub freq: f64,
+    pub amplitude: f64,
+}
+
+impl Default for FrameData {
+    fn default() -> Self {
+        Self {
+            sample_rate: 48000,
+            freq: 440.0,
+            amplitude: 1.0,
+        }
+    }
+}
+
+struct TrackHandle {
+    frame_data: Arc<Mutex<FrameData>>,
+    close_tx: oneshot::Sender<()>,
+    track: LocalAudioTrack,
+    task: JoinHandle<()>,
+}
+
+pub struct SineTrack {
+    rtc_source: NativeAudioSource,
+    session: RoomSession,
+    handle: Option<TrackHandle>,
+}
+
+impl SineTrack {
+    pub fn new(session: RoomSession) -> Self {
+        Self {
+            rtc_source: NativeAudioSource::default(),
+            session,
+            handle: None,
+        }
+    }
+
+    pub async fn publish(&mut self) -> Result<(), RoomError> {
+        let (close_tx, close_rx) = oneshot::channel();
+        let track = LocalAudioTrack::create_audio_track(
+            "sine_wave",
+            AudioCaptureOptions {
+                auto_gain_control: false,
+                echo_cancellation: false,
+                noise_suppression: false,
+            },
+            self.rtc_source.clone(),
+        );
+
+        let data = Arc::new(Mutex::new(FrameData::default()));
+        let task = tokio::spawn(Self::track_task(
+            close_rx,
+            self.rtc_source.clone(),
+            data.clone(),
+        ));
+
+        self.session
+            .local_participant()
+            .publish_track(
+                LocalTrack::Audio(track.clone()),
+                TrackPublishOptions {
+                    source: TrackSource::Microphone,
+                    ..Default::default()
+                },
+            )
+            .await?;
+
+        let handle = TrackHandle {
+            frame_data: data,
+            close_tx,
+            track,
+            task,
+        };
+
+        self.handle = Some(handle);
+        Ok(())
+    }
+
+    async fn track_task(
+        mut close_rx: oneshot::Receiver<()>,
+        rtc_source: NativeAudioSource,
+        frame_options: Arc<Mutex<FrameData>>,
+    ) {
+        let mut interval = tokio::time::interval(Duration::from_millis(10));
+        let mut samples_10ms = Vec::<i16>::new();
+
+        loop {
+            interval.tick().await;
+
+            let data = frame_options.lock();
+            let samples_count_10ms = (data.sample_rate / 100) as usize;
+
+            if samples_10ms.capacity() != samples_count_10ms {
+                samples_10ms.resize(samples_count_10ms, 0i16);
+            }
+
+            for i in 0..samples_count_10ms {
+                let val = data.amplitude
+                    * f64::sin(
+                        std::f64::consts::PI * 2.0 * data.freq * i as f64
+                            / samples_count_10ms as f64,
+                    );
+
+                // WebRTC uses 16-bit signed PCM
+                samples_10ms[i] = (val * 32768.0) as i16;
+            }
+
+            rtc_source.capture_frame(AudioFrame {
+                data: samples_10ms.clone(),
+                sample_rate_hz: data.sample_rate,
+                num_channels: 1,
+                samples_per_channel: samples_count_10ms,
+            });
+        }
+    }
+}