livekit-utils crate + VideoRenderer proto

examples/Cargo.toml

@@ -1,3 +1,4 @@
 [workspace]
 members = ["*"]
-exclude = ["target"]
+exclude = ["target"]
+resolver = "2"

examples/simple_room/Cargo.toml

@@ -9,3 +9,10 @@ tracing = "0.1"
 tracing-subscriber = "0.3"
 livekit = { path = "../.." }
 futures = "0.3"
+wgpu = "0.14.0"
+winit = "0.27.5"
+
+egui = { git = "https://github.com/emilk/egui" }
+egui-wgpu = { git = "https://github.com/emilk/egui", features = ["winit"] }
+egui-winit = { git = "https://github.com/emilk/egui" }
+egui_demo_lib = { git = "https://github.com/emilk/egui" }

examples/simple_room/src/main.rs

@@ -1,36 +1,192 @@
-use std::time::Duration;
+use std::convert::TryInto;
+use std::ops::DerefMut;
+use std::{num::NonZeroU32, time::Duration};
+
+use egui_wgpu::WgpuConfiguration;
+use livekit::webrtc::media_stream::VideoTrack;
+use livekit::webrtc::video_frame_buffer::{
+    PlanarYuv8Buffer, PlanarYuvBuffer, VideoFrameBufferTrait,
+};
+use livekit::webrtc::yuv_helper;
+use std::sync::{Arc, Mutex};
+use video_renderer::VideoRenderer;
+use wgpu::{Device, Queue};
 
-use livekit::room::RoomError;
-use livekit::room::{track::remote_track::RemoteTrackHandle, Room};
 use tokio::time::sleep;
 
+use livekit::room::track::remote_track::RemoteTrackHandle;
+use livekit::room::{Room, RoomError};
+
 const URL: &str = "ws://localhost:7880";
 const TOKEN : &str = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjIzODQ4MDY0NzMsImlzcyI6IkFQSXpLYkFTaUNWYWtnSiIsIm5hbWUiOiJuYXRpdmUiLCJuYmYiOjE2NjQ4MDY0NzMsInN1YiI6Im5hdGl2ZSIsInZpZGVvIjp7InJvb21DcmVhdGUiOnRydWUsInJvb21Kb2luIjp0cnVlfX0.BgVdBnq3XFD3_BQHoe1azqjifYysubgFl6Qlzu9IQGI";
 
 // eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjIzODQ4MDY3MzAsImlzcyI6IkFQSXpLYkFTaUNWYWtnSiIsIm5hbWUiOiJ3ZWIiLCJuYmYiOjE2NjQ4MDY3MzAsInN1YiI6IndlYiIsInZpZGVvIjp7InJvb21DcmVhdGUiOnRydWUsInJvb21Kb2luIjp0cnVlfX0.VbDoULjX1CVGZu2sPy3SvWYlVZUBXxQVPmdB9BnmlN4
 
-#[tokio::main]
-async fn main() -> Result<(), RoomError> {
-    tracing_subscriber::fmt::init();
+mod video_renderer;
 
-    let mut room = Room::new();
-    room.events()
-        .on_participant_connected(|_event| async move {});
+use winit::{
+    event::*,
+    event_loop::{ControlFlow, EventLoop},
+    window::{Window, WindowBuilder, WindowId},
+};
 
-    room.events().on_track_subscribed(|event| async move {
-        let track = event.publication.track().unwrap();
-        if let RemoteTrackHandle::Video(video_track) = track {
-            let rtc_track = video_track.rtc_track();
-            rtc_track.set_should_receive(true);
-            rtc_track.on_frame(Box::new(|_frame, _buffer| {
-                // called on libwebrtc worker_thread
-                println!("Received frame");
-            }));
-        }
-    });
-
-    room.connect(URL, TOKEN).await?;
-
-    sleep(Duration::from_secs(200)).await;
-    Ok(())
+struct AppState {
+    room: Room,
+    demo: egui_demo_lib::DemoWindows,
+    egui_context: egui::Context,
+    egui_state: egui_winit::State,
+    egui_painter: egui_wgpu::winit::Painter,
+    window: winit::window::Window,
+}
+
+impl AppState {
+    fn on_event<T>(&mut self, event: Event<'_, T>, control_flow: &mut ControlFlow) {
+        match event {
+            Event::WindowEvent { window_id, event } => {
+                if let Some(flow) = self.on_window_event(window_id, event) {
+                    *control_flow = flow;
+                }
+            }
+            Event::RedrawRequested(window_id) if window_id == self.window.id() => {
+                self.render();
+            }
+            Event::RedrawEventsCleared => {
+                self.window.request_redraw();
+            }
+            _ => {}
+        };
+    }
+
+    fn on_window_event(
+        &mut self,
+        _window_id: WindowId,
+        event: WindowEvent<'_>,
+    ) -> Option<ControlFlow> {
+        if self
+            .egui_state
+            .on_event(&self.egui_context, &event)
+            .consumed
+        {
+            return None;
+        }
+
+        match event {
+            WindowEvent::CloseRequested => Some(ControlFlow::Exit),
+            WindowEvent::Resized(inner_size) => {
+                self.egui_painter
+                    .on_window_resized(inner_size.width, inner_size.height);
+                None
+            }
+            WindowEvent::ScaleFactorChanged { new_inner_size, .. } => {
+                self.egui_painter
+                    .on_window_resized(new_inner_size.width, new_inner_size.height);
+                None
+            }
+            _ => None,
+        }
+    }
+
+    fn render(&mut self) {
+        let raw_inputs = self.egui_state.take_egui_input(&self.window);
+        let full_output = self.egui_context.run(raw_inputs, |ctx| {
+            //self.ui(ctx);
+        });
+        let clipped_primitives = self.egui_context.tessellate(full_output.shapes);
+
+        self.egui_painter.paint_and_update_textures(
+            egui_winit::native_pixels_per_point(&self.window),
+            egui::Rgba::BLACK,
+            &clipped_primitives,
+            &full_output.textures_delta,
+        );
+
+        self.egui_state.handle_platform_output(
+            &self.window,
+            &self.egui_context,
+            full_output.platform_output,
+        );
+    }
+}
+
+struct App {
+    rt: tokio::runtime::Runtime,
+}
+
+impl App {
+    pub fn new(rt: tokio::runtime::Runtime) -> Self {
+        Self { rt }
+    }
+
+    pub fn run(&mut self) {
+        self.rt.block_on(async {
+            let event_loop = EventLoop::new();
+            let window = WindowBuilder::new().build(&event_loop).unwrap();
+
+            let egui_context = egui::Context::default();
+            let egui_state = egui_winit::State::new(&event_loop);
+            let mut egui_painter =
+                egui_wgpu::winit::Painter::new(WgpuConfiguration::default(), 1, 32);
+            unsafe {
+                egui_painter.set_window(Some(&window));
+            }
+
+            let mut inner = AppState {
+                room: Room::new(),
+                demo: egui_demo_lib::DemoWindows::default(),
+                egui_context,
+                egui_state,
+                egui_painter,
+                window,
+            };
+
+            inner
+                .room
+                .events()
+                .on_participant_connected(|_event| async move {});
+
+            inner.room.events().on_track_subscribed({
+                let test = Arc::new(Mutex::new(None));
+
+                let egui_render = inner.egui_painter.render_state().clone().unwrap();
+
+                move |event| {
+                    let test = test.clone();
+                    let egui_render = egui_render.clone();
+
+                    async move {
+                        let track = event.publication.track().unwrap();
+                        if let RemoteTrackHandle::Video(video_track) = track {
+                            *test.lock().unwrap() =
+                                Some(VideoRenderer::new(egui_render, video_track.rtc_track()))
+                        }
+                    }
+                }
+            });
+
+            inner.room.connect(URL, TOKEN).await.unwrap();
+
+            tokio::spawn(async {
+                loop {
+                    println!("Test");
+                    tokio::time::sleep(Duration::from_secs(5)).await;
+                }
+            });
+
+            tokio::task::block_in_place(move || loop {
+                event_loop.run(move |event, _, control_flow| {
+                    inner.on_event(event, control_flow);
+                });
+            });
+        });
+    }
+}
+
+fn main() {
+    let rt = tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()
+        .unwrap();
+
+    let mut app = App::new(rt);
+    app.run();
 }

examples/simple_room/src/video_renderer.rs

@@ -0,0 +1,172 @@
+use livekit::webrtc::media_stream::VideoTrack;
+use livekit::webrtc::video_frame_buffer::PlanarYuv8Buffer;
+use livekit::webrtc::video_frame_buffer::PlanarYuvBuffer;
+use livekit::webrtc::video_frame_buffer::VideoFrameBufferTrait;
+use livekit::webrtc::yuv_helper;
+use std::convert::TryInto;
+use std::num::NonZeroU32;
+use std::{
+    ops::DerefMut,
+    sync::{Arc, Mutex},
+};
+
+pub struct VideoRenderer {
+    internal: Arc<Mutex<RendererInternal>>,
+    rtc_track: Arc<VideoTrack>,
+}
+
+struct RendererInternal {
+    render_state: egui_wgpu::RenderState,
+    width: u32,
+    height: u32,
+    rgba_data: Vec<u8>,
+    texture: Option<wgpu::Texture>,
+    texture_view: Option<wgpu::TextureView>,
+    egui_texture: Option<egui::TextureId>,
+}
+
+impl RendererInternal {
+    fn ensure_texture_size(&mut self, width: u32, height: u32) {
+        if self.width == width && self.height == height {
+            return;
+        }
+
+        self.width = width;
+        self.height = height;
+        self.rgba_data.resize((width * height * 4) as usize, 0);
+
+        self.texture = Some(
+            self.render_state
+                .device
+                .create_texture(&wgpu::TextureDescriptor {
+                    label: Some("lk-videotexture"),
+                    usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
+                    dimension: wgpu::TextureDimension::D2,
+                    size: wgpu::Extent3d {
+                        width,
+                        height,
+                        ..Default::default()
+                    },
+                    sample_count: 1,
+                    mip_level_count: 1,
+                    format: wgpu::TextureFormat::Rgba8UnormSrgb,
+                }),
+        );
+
+        self.texture_view = Some(self.texture.as_mut().unwrap().create_view(
+            &wgpu::TextureViewDescriptor {
+                label: Some("lk-videotexture-view"),
+                format: Some(wgpu::TextureFormat::Rgba8UnormSrgb),
+                dimension: Some(wgpu::TextureViewDimension::D2),
+                mip_level_count: NonZeroU32::new(1),
+                array_layer_count: NonZeroU32::new(1),
+                ..Default::default()
+            },
+        ));
+
+        if let Some(texture_id) = self.egui_texture {
+            // Update the existing texture
+            self.render_state
+                .renderer
+                .write()
+                .update_egui_texture_from_wgpu_texture(
+                    &*self.render_state.device,
+                    self.texture_view.as_ref().unwrap(),
+                    wgpu::FilterMode::Linear,
+                    texture_id,
+                );
+        } else {
+            self.egui_texture = Some(self.render_state.renderer.write().register_native_texture(
+                &*self.render_state.device,
+                self.texture_view.as_ref().unwrap(),
+                wgpu::FilterMode::Linear,
+            ));
+        }
+    }
+}
+
+impl VideoRenderer {
+    pub fn new(render_state: egui_wgpu::RenderState, rtc_track: Arc<VideoTrack>) -> Self {
+        let internal = Arc::new(Mutex::new(RendererInternal {
+            render_state,
+            width: 0,
+            height: 0,
+            rgba_data: Vec::default(),
+            texture: None,
+            texture_view: None,
+            egui_texture: None,
+        }));
+
+        rtc_track.on_frame({
+            let internal = internal.clone();
+
+            Box::new(move |_frame, buffer| {
+                let mut internal = internal.lock().unwrap();
+                let buffer = buffer.to_i420();
+
+                let width: u32 = buffer.width().try_into().unwrap();
+                let height: u32 = buffer.height().try_into().unwrap();
+
+                internal.ensure_texture_size(width, height);
+
+                let rgba_ptr = internal.rgba_data.deref_mut();
+                let rgba_stride = buffer.width() * 4;
+
+                yuv_helper::i420_to_abgr(
+                    buffer.data_y(),
+                    buffer.stride_y(),
+                    buffer.data_u(),
+                    buffer.stride_u(),
+                    buffer.data_v(),
+                    buffer.stride_v(),
+                    rgba_ptr,
+                    rgba_stride,
+                    buffer.width(),
+                    buffer.height(),
+                );
+
+                let copy_desc = wgpu::ImageCopyTexture {
+                    texture: internal.texture.as_ref().unwrap(),
+                    mip_level: 0,
+                    origin: wgpu::Origin3d::default(),
+                    aspect: wgpu::TextureAspect::default(),
+                };
+
+                let copy_layout = wgpu::ImageDataLayout {
+                    bytes_per_row: Some(NonZeroU32::new(width * 4).unwrap()),
+                    ..Default::default()
+                };
+
+                let copy_size = wgpu::Extent3d {
+                    width,
+                    height,
+                    ..Default::default()
+                };
+
+                internal.render_state.queue.write_texture(
+                    copy_desc,
+                    &internal.rgba_data,
+                    copy_layout,
+                    copy_size,
+                );
+
+                println!("wrote");
+            })
+        });
+
+        Self {
+            rtc_track,
+            internal,
+        }
+    }
+
+    pub fn texture_id(&self) -> Option<egui::TextureId> {
+        self.internal.lock().unwrap().egui_texture.clone()
+    }
+}
+
+impl Drop for VideoRenderer {
+    fn drop(&mut self) {
+        self.rtc_track.on_frame(Box::new(|_, _| {}));
+    }
+}