diff --git a/examples/Cargo.lock b/examples/Cargo.lock index 35e7fa4..ce8a9ce 100644 --- a/examples/Cargo.lock +++ b/examples/Cargo.lock @@ -2038,6 +2038,7 @@ dependencies = [ "egui_demo_lib", "futures", "livekit", + "parking_lot", "tokio", "tracing", "tracing-subscriber", diff --git a/examples/simple_room/Cargo.toml b/examples/simple_room/Cargo.toml index d9e266a..a5438f2 100644 --- a/examples/simple_room/Cargo.toml +++ b/examples/simple_room/Cargo.toml @@ -11,6 +11,7 @@ livekit = { path = "../.." } futures = "0.3" wgpu = "0.14.0" winit = "0.27.5" +parking_lot = "0.12.1" egui = { git = "https://github.com/emilk/egui" } egui-wgpu = { git = "https://github.com/emilk/egui", features = ["winit"] } diff --git a/examples/simple_room/src/app.rs b/examples/simple_room/src/app.rs new file mode 100644 index 0000000..5ba5395 --- /dev/null +++ b/examples/simple_room/src/app.rs @@ -0,0 +1,243 @@ +use crate::events::DemoEvent; +use crate::video_grid::VideoGrid; +use crate::video_renderer::VideoRenderer; +use egui_wgpu::WgpuConfiguration; +use parking_lot::Mutex; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; +use tokio::sync::mpsc; + +use livekit::room::Room; + +const URL: &str = "ws://localhost:7880"; +const TOKEN : &str = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjIzODQ4MDY0NzMsImlzcyI6IkFQSXpLYkFTaUNWYWtnSiIsIm5hbWUiOiJuYXRpdmUiLCJuYmYiOjE2NjQ4MDY0NzMsInN1YiI6Im5hdGl2ZSIsInZpZGVvIjp7InJvb21DcmVhdGUiOnRydWUsInJvb21Kb2luIjp0cnVlfX0.BgVdBnq3XFD3_BQHoe1azqjifYysubgFl6Qlzu9IQGI"; + +// eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjIzODQ4MDY3MzAsImlzcyI6IkFQSXpLYkFTaUNWYWtnSiIsIm5hbWUiOiJ3ZWIiLCJuYmYiOjE2NjQ4MDY3MzAsInN1YiI6IndlYiIsInZpZGVvIjp7InJvb21DcmVhdGUiOnRydWUsInJvb21Kb2luIjp0cnVlfX0.VbDoULjX1CVGZu2sPy3SvWYlVZUBXxQVPmdB9BnmlN4 + +use winit::{ + event::*, + event_loop::{ControlFlow, EventLoop}, + window::{WindowBuilder, WindowId}, +}; + +struct AppState { + room: Mutex, + connecting: AtomicBool, +} + +struct App { + state: Arc, + + renderers: Vec, + egui_context: egui::Context, + egui_state: egui_winit::State, + egui_painter: egui_wgpu::winit::Painter, + window: winit::window::Window, + event_tx: mpsc::UnboundedSender, + + // UI State + lk_url: String, + lk_token: String, +} + +pub fn run(rt: tokio::runtime::Runtime) { + rt.block_on(async { + let event_loop = EventLoop::new(); + let window = WindowBuilder::new() + .with_title("LiveKit - NativeSDK") + .build(&event_loop) + .unwrap(); + + let egui_context = egui::Context::default(); + let egui_state = egui_winit::State::new(&event_loop); + let mut egui_painter = egui_wgpu::winit::Painter::new(WgpuConfiguration::default(), 1, 32); + + unsafe { + egui_painter.set_window(Some(&window)); + } + + let (event_tx, mut event_rx) = mpsc::unbounded_channel::(); + let (event_tx, mut event_rx) = mpsc::unbounded_channel::(); + + let state = Arc::new(AppState { + room: Mutex::new(Room::new()), + connecting: AtomicBool::new(false), + }); + + let mut app = App { + state: state.clone(), + renderers: Vec::default(), + egui_context, + egui_state, + egui_painter, + window, + event_tx, + lk_url: "ws://localhost:8080/".to_owned(), + lk_token: "your token".to_owned(), + }; + + // Async event loop + tokio::spawn(async move { + while let Some(event) = event_rx.recv().await { + match event { + DemoEvent::RoomConnect { url, token } => { + state.connecting.store(true, Ordering::SeqCst); + + let mut room = state.room.lock(); + room.connect(&url, &token).await.unwrap(); + + state.connecting.store(false, Ordering::SeqCst); + } + } + } + }); + + tokio::task::block_in_place(move || loop { + // UI/Main Thread + event_loop.run(move |event, _, control_flow| { + app.update(event, control_flow); + }); + }); + }); +} + +impl App { + fn update(&mut self, event: Event<'_, T>, control_flow: &mut ControlFlow) { + match event { + Event::WindowEvent { window_id, event } => { + if let Some(flow) = self.on_window_event(window_id, event) { + *control_flow = flow; + } + } + Event::RedrawRequested(window_id) if window_id == self.window.id() => { + self.render(); + } + Event::RedrawEventsCleared => { + self.window.request_redraw(); + } + _ => {} + }; + } + + fn on_window_event( + &mut self, + _window_id: WindowId, + event: WindowEvent<'_>, + ) -> Option { + if self + .egui_state + .on_event(&self.egui_context, &event) + .consumed + { + return None; + } + + match event { + WindowEvent::CloseRequested => Some(ControlFlow::Exit), + WindowEvent::Resized(inner_size) => { + self.egui_painter + .on_window_resized(inner_size.width, inner_size.height); + None + } + WindowEvent::ScaleFactorChanged { new_inner_size, .. } => { + self.egui_painter + .on_window_resized(new_inner_size.width, new_inner_size.height); + None + } + _ => None, + } + } + + fn ui(&mut self, ui: &mut egui::Ui) { + egui::TopBottomPanel::top("top_panel").show(ui.ctx(), |ui| { + egui::menu::bar(ui, |ui| { + ui.menu_button("Tools", |ui| { + if ui.button("Logs").clicked() {} + if ui.button("Profiler").clicked() {} + if ui.button("WebRTC Stats").clicked() {} + }); + ui.menu_button("Simulate", |ui| {}); + }); + }); + + egui::SidePanel::right("room_panel") + .default_width(128.0) + .show(ui.ctx(), |ui| { + ui.heading("Livekit - Connect to a room"); + + ui.separator(); + + ui.horizontal(|ui| { + ui.label("URL: "); + ui.text_edit_singleline(&mut self.lk_url); + }); + + ui.horizontal(|ui| { + ui.label("Token: "); + ui.text_edit_singleline(&mut self.lk_token); + }); + + ui.horizontal(|ui| { + let connecting = self.state.connecting.load(Ordering::SeqCst); + ui.set_enabled(!connecting); + if ui.button("Connect").clicked() { + self.event_tx + .send(DemoEvent::RoomConnect { + url: self.lk_url.clone(), + token: self.lk_token.clone(), + }) + .unwrap(); + } + + if connecting { + ui.spinner(); + } + }); + + ui.allocate_space(ui.available_size()); + }); + + egui::CentralPanel::default().show(ui.ctx(), |ui| { + egui::ScrollArea::vertical().show(ui, |ui| { + VideoGrid::new("default_grid") + .max_columns(6) + .show(ui, |ui| { + for _ in 0..20 { + ui.video_frame(|ui| { + egui::Frame::none() + .fill(egui::Color32::DARK_GRAY) + .show(ui, |ui| { + ui.allocate_space(ui.available_size()); + }); + }); + } + }); + }); + }); + } + + fn render(&mut self) { + let raw_inputs = self.egui_state.take_egui_input(&self.window); + let full_output = self.egui_context.clone().run(raw_inputs, |ctx| { + egui::CentralPanel::default().show(ctx, |ui| { + self.ui(ui); + }); + }); + let clipped_primitives = self.egui_context.tessellate(full_output.shapes); + + self.egui_painter.paint_and_update_textures( + egui_winit::native_pixels_per_point(&self.window), + egui::Rgba::BLACK, + &clipped_primitives, + &full_output.textures_delta, + ); + + self.egui_state.handle_platform_output( + &self.window, + &self.egui_context, + full_output.platform_output, + ); + } +} diff --git a/examples/simple_room/src/events.rs b/examples/simple_room/src/events.rs new file mode 100644 index 0000000..29d785e --- /dev/null +++ b/examples/simple_room/src/events.rs @@ -0,0 +1,9 @@ +#[derive(Debug)] +pub enum AsyncCmd { + RoomConnect { url: String, token: String }, +} + +#[derive(Debug)] +pub enum UiCmd { + ConnectResult, +} diff --git a/examples/simple_room/src/main.rs b/examples/simple_room/src/main.rs index a35bbbf..cebde08 100644 --- a/examples/simple_room/src/main.rs +++ b/examples/simple_room/src/main.rs @@ -1,185 +1,7 @@ -use std::convert::TryInto; -use std::ops::DerefMut; -use std::{num::NonZeroU32, time::Duration}; - -use egui_wgpu::WgpuConfiguration; -use livekit::webrtc::media_stream::VideoTrack; -use livekit::webrtc::video_frame_buffer::{ - PlanarYuv8Buffer, PlanarYuvBuffer, VideoFrameBufferTrait, -}; -use livekit::webrtc::yuv_helper; -use std::sync::{Arc, Mutex}; -use video_renderer::VideoRenderer; -use wgpu::{Device, Queue}; - -use tokio::time::sleep; - -use livekit::room::track::remote_track::RemoteTrackHandle; -use livekit::room::{Room, RoomError}; - -const URL: &str = "ws://localhost:7880"; -const TOKEN : &str = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjIzODQ4MDY0NzMsImlzcyI6IkFQSXpLYkFTaUNWYWtnSiIsIm5hbWUiOiJuYXRpdmUiLCJuYmYiOjE2NjQ4MDY0NzMsInN1YiI6Im5hdGl2ZSIsInZpZGVvIjp7InJvb21DcmVhdGUiOnRydWUsInJvb21Kb2luIjp0cnVlfX0.BgVdBnq3XFD3_BQHoe1azqjifYysubgFl6Qlzu9IQGI"; - -// eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjIzODQ4MDY3MzAsImlzcyI6IkFQSXpLYkFTaUNWYWtnSiIsIm5hbWUiOiJ3ZWIiLCJuYmYiOjE2NjQ4MDY3MzAsInN1YiI6IndlYiIsInZpZGVvIjp7InJvb21DcmVhdGUiOnRydWUsInJvb21Kb2luIjp0cnVlfX0.VbDoULjX1CVGZu2sPy3SvWYlVZUBXxQVPmdB9BnmlN4 - +mod events; +mod video_grid; mod video_renderer; - -use winit::{ - event::*, - event_loop::{ControlFlow, EventLoop}, - window::{Window, WindowBuilder, WindowId}, -}; - -struct AppState { - room: Room, - demo: egui_demo_lib::DemoWindows, - egui_context: egui::Context, - egui_state: egui_winit::State, - egui_painter: egui_wgpu::winit::Painter, - window: winit::window::Window, -} - -impl AppState { - fn on_event(&mut self, event: Event<'_, T>, control_flow: &mut ControlFlow) { - match event { - Event::WindowEvent { window_id, event } => { - if let Some(flow) = self.on_window_event(window_id, event) { - *control_flow = flow; - } - } - Event::RedrawRequested(window_id) if window_id == self.window.id() => { - self.render(); - } - Event::RedrawEventsCleared => { - self.window.request_redraw(); - } - _ => {} - }; - } - - fn on_window_event( - &mut self, - _window_id: WindowId, - event: WindowEvent<'_>, - ) -> Option { - if self - .egui_state - .on_event(&self.egui_context, &event) - .consumed - { - return None; - } - - match event { - WindowEvent::CloseRequested => Some(ControlFlow::Exit), - WindowEvent::Resized(inner_size) => { - self.egui_painter - .on_window_resized(inner_size.width, inner_size.height); - None - } - WindowEvent::ScaleFactorChanged { new_inner_size, .. } => { - self.egui_painter - .on_window_resized(new_inner_size.width, new_inner_size.height); - None - } - _ => None, - } - } - - fn render(&mut self) { - let raw_inputs = self.egui_state.take_egui_input(&self.window); - let full_output = self.egui_context.run(raw_inputs, |ctx| { - //self.ui(ctx); - }); - let clipped_primitives = self.egui_context.tessellate(full_output.shapes); - - self.egui_painter.paint_and_update_textures( - egui_winit::native_pixels_per_point(&self.window), - egui::Rgba::BLACK, - &clipped_primitives, - &full_output.textures_delta, - ); - - self.egui_state.handle_platform_output( - &self.window, - &self.egui_context, - full_output.platform_output, - ); - } -} - -struct App { - rt: tokio::runtime::Runtime, -} - -impl App { - pub fn new(rt: tokio::runtime::Runtime) -> Self { - Self { rt } - } - - pub fn run(&mut self) { - self.rt.block_on(async { - let event_loop = EventLoop::new(); - let window = WindowBuilder::new().build(&event_loop).unwrap(); - - let egui_context = egui::Context::default(); - let egui_state = egui_winit::State::new(&event_loop); - let mut egui_painter = - egui_wgpu::winit::Painter::new(WgpuConfiguration::default(), 1, 32); - unsafe { - egui_painter.set_window(Some(&window)); - } - - let mut inner = AppState { - room: Room::new(), - demo: egui_demo_lib::DemoWindows::default(), - egui_context, - egui_state, - egui_painter, - window, - }; - - inner - .room - .events() - .on_participant_connected(|_event| async move {}); - - inner.room.events().on_track_subscribed({ - let test = Arc::new(Mutex::new(None)); - - let egui_render = inner.egui_painter.render_state().clone().unwrap(); - - move |event| { - let test = test.clone(); - let egui_render = egui_render.clone(); - - async move { - let track = event.publication.track().unwrap(); - if let RemoteTrackHandle::Video(video_track) = track { - *test.lock().unwrap() = - Some(VideoRenderer::new(egui_render, video_track.rtc_track())) - } - } - } - }); - - inner.room.connect(URL, TOKEN).await.unwrap(); - - tokio::spawn(async { - loop { - println!("Test"); - tokio::time::sleep(Duration::from_secs(5)).await; - } - }); - - tokio::task::block_in_place(move || loop { - event_loop.run(move |event, _, control_flow| { - inner.on_event(event, control_flow); - }); - }); - }); - } -} +mod app; fn main() { let rt = tokio::runtime::Builder::new_multi_thread() @@ -187,6 +9,5 @@ fn main() { .build() .unwrap(); - let mut app = App::new(rt); - app.run(); + app::run(rt); } diff --git a/examples/simple_room/src/video_grid.rs b/examples/simple_room/src/video_grid.rs new file mode 100644 index 0000000..bc14d75 --- /dev/null +++ b/examples/simple_room/src/video_grid.rs @@ -0,0 +1,175 @@ +use std::cmp; + +#[derive(Debug, Clone, Default, PartialEq)] +struct State { + num_videos: u32, +} + +impl State { + pub fn load(ctx: &egui::Context, id: egui::Id) -> Option { + ctx.data().get_temp(id) + } + + pub fn store(self, ctx: &egui::Context, id: egui::Id) { + ctx.data().insert_temp(id, self); + } +} + +pub const DEFAULT_VIDEO_SIZE: egui::Vec2 = egui::vec2(320.0, 180.0); +pub const DEFAULT_MAX_COLUMNS: u32 = 4; +pub const DEFAULT_SPACING: f32 = 16.0; + +pub struct VideoGrid { + id: egui::Id, + + // Current frame + available_rect: egui::Rect, + prev_state: State, + curr_state: State, + video_index: u32, // Kinda "cursor" + + // Options + min_video_size: egui::Vec2, + max_columns: u32, + spacing: f32, +} + +impl VideoGrid { + pub fn new(id_source: impl std::hash::Hash) -> Self { + Self { + id: egui::Id::new(id_source), + available_rect: egui::Rect::NAN, + prev_state: State::default(), + curr_state: State::default(), + video_index: 0, + min_video_size: DEFAULT_VIDEO_SIZE, + max_columns: DEFAULT_MAX_COLUMNS, + spacing: DEFAULT_SPACING, + } + } + + pub fn show( + mut self, + ui: &mut egui::Ui, + grid: impl FnOnce(&mut VideoGridContext) -> R, + ) -> egui::InnerResponse { + // TODO(theomonnom): Should I care about the current egui layout? + + let prev_state = State::load(ui.ctx(), self.id); + let is_first_frame = prev_state.is_none(); + + self.prev_state = prev_state.unwrap_or_default(); + self.available_rect = ui.available_rect_before_wrap(); + + ui.ctx() + .check_for_id_clash(self.id, self.available_rect, "VideoGrid"); + + ui.allocate_ui_at_rect(self.available_rect, |ui| { + ui.set_visible(!is_first_frame); + + let mut ctx = VideoGridContext { + layout: &mut self, + ui, + }; + let res = grid(&mut ctx); + + // Save the new state + if self.curr_state != self.prev_state { + self.curr_state.clone().store(ui.ctx(), self.id); + ui.ctx().request_repaint(); + } + + res + }) + } + + fn next_frame_rect(&mut self) -> egui::Rect { + assert!(self.available_rect.is_finite()); + assert!(self.spacing <= self.min_video_size.x); + + // increment the amount of videos for the next frame + self.curr_state.num_videos += 1; + + let num_videos = self.prev_state.num_videos; + if num_videos == 0 { + return egui::Rect::NOTHING; + } + + let max_columns = self.max_columns; + let minimum_size = self.min_video_size; + let available_size = self.available_rect.size(); + + let calc_min_width = + |columns: u32| columns as f32 * minimum_size.x + (columns - 1) as f32 * self.spacing; + + let total_columns = { + let mut est = (available_size.x / minimum_size.x) as u32 + 1; + if available_size.x < calc_min_width(est) { + est -= 1; + } + cmp::max(1, cmp::min(est, max_columns)) + }; + + let aspect_ratio = minimum_size.x / minimum_size.y; + let remaining_width = available_size.x - calc_min_width(total_columns); + let w = minimum_size.x + remaining_width / total_columns as f32; + let h = w / aspect_ratio; + + let x_index = self.video_index % total_columns; + let y_index = self.video_index / total_columns; + + let x = { + let mut x = x_index as f32 * (w + self.spacing); + + // vertically center the last row + let total_rows = num_videos / total_columns + 1; + if (y_index + 1) == total_rows { + let nb_items = num_videos - (total_rows - 1) * total_columns; // nb. of items on the last row + x += (total_columns - nb_items) as f32 * (w + self.spacing) / 2.0; + } + + x + }; + let y = y_index as f32 * (h + self.spacing); + + let min = egui::pos2(x, y) + self.available_rect.left_top().to_vec2(); + let max = egui::pos2(w, h) + min.to_vec2(); + + self.video_index += 1; + + egui::Rect { min, max } + } +} + +impl VideoGrid { + pub fn min_video_size(mut self, min_video_size: egui::Vec2) -> Self { + self.min_video_size = min_video_size; + self + } + + pub fn max_columns(mut self, max_columns: u32) -> Self { + self.max_columns = max_columns; + self + } + + pub fn spacing(mut self, spacing: f32) -> Self { + self.spacing = spacing; + self + } +} + +pub struct VideoGridContext<'a> { + layout: &'a mut VideoGrid, + ui: &'a mut egui::Ui, +} + +impl<'a> VideoGridContext<'a> { + pub fn video_frame(&mut self, add_contents: impl FnOnce(&mut egui::Ui)) -> egui::Response { + let frame_rect = self.layout.next_frame_rect(); + + let mut child_ui = self.ui.child_ui(frame_rect, egui::Layout::default()); + add_contents(&mut child_ui); + + self.ui.allocate_rect(frame_rect, egui::Sense::hover()) + } +} diff --git a/examples/simple_room/src/video_renderer.rs b/examples/simple_room/src/video_renderer.rs index 124322b..6ae4371 100644 --- a/examples/simple_room/src/video_renderer.rs +++ b/examples/simple_room/src/video_renderer.rs @@ -149,8 +149,6 @@ impl VideoRenderer { copy_layout, copy_size, ); - - println!("wrote"); }) });