talksik/client-sdk-rust
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

livekit-utils crate + VideoRenderer proto

Browse Source
This commit is contained in:
Théo Monnom
2022-12-02 16:39:42 +01:00
parent 59e7ccbffe
commit 686f5db969
30 changed files with 2211 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+7
View File
@@ -484,6 +484,7 @@ name = "livekit"
version = "0.1.0"
dependencies = [
"livekit-core",
"livekit-webrtc",
]
[[package]]
@@ -493,6 +494,7 @@ dependencies = [
"futures",
"futures-util",
"lazy_static",
"livekit-utils",
"livekit-webrtc",
"parking_lot",
"prost",
@@ -507,6 +509,10 @@ dependencies = [
"url",
]
[[package]]
name = "livekit-utils"
version = "0.1.0"
[[package]]
name = "livekit-webrtc"
version = "0.1.0"
@@ -514,6 +520,7 @@ dependencies = [
"cxx",
"env_logger",
"libwebrtc-sys",
"livekit-utils",
"log",
"thiserror",
"tokio",
Cargo.toml
+2
View File
@@ -9,8 +9,10 @@ exclude = ["libwebrtc"]
members = [
"crates/livekit-core",
"crates/livekit-webrtc",
"crates/livekit-utils",
"crates/livekit-webrtc/libwebrtc-sys"
]
[dependencies]
livekit-core = { path = "crates/livekit-core" }
livekit-webrtc = { path = "crates/livekit-webrtc" }
crates/livekit-core/Cargo.toml
+2 -1
View File
@@ -5,6 +5,8 @@ edition = "2021"
license = "Apache-2.0"
[dependencies]
livekit-webrtc = { path = "../livekit-webrtc" }
livekit-utils = { path = "../livekit-utils" }
serde = { version = "1", features = ["derive"] }
serde_json = "1.0"
tokio-tungstenite = { version = "0.17.2", features = ["native-tls"] }
@@ -16,7 +18,6 @@ futures-util = "0.3.23"
thiserror = "1.0"
prost = "0.11.0"
prost-types = "0.11.1"
livekit-webrtc = { path = "../livekit-webrtc" }
lazy_static = "1.4.0"
tracing = "0.1"
crates/livekit-core/src/lib.rs
-1
View File
@@ -7,6 +7,5 @@ pub mod proto {
mod events;
mod rtc_engine;
mod signal_client;
mod utils;
pub mod room;
crates/livekit-core/src/room/participant/mod.rs
+10 -10
View File
@@ -4,8 +4,8 @@ use crate::room::id::{ParticipantIdentity, ParticipantSid, TrackSid};
use crate::room::participant::local_participant::LocalParticipant;
use crate::room::participant::remote_participant::RemoteParticipant;
use crate::room::publication::{TrackPublication, TrackPublicationTrait};
use crate::utils::wrap_variants;
use futures_util::future::BoxFuture;
use livekit_utils::enum_dispatch;
use parking_lot::{Mutex, RwLock};
use std::collections::HashMap;
use std::sync::Arc;
@@ -74,7 +74,7 @@ pub enum ParticipantHandle {
}
impl ParticipantHandle {
// TODO(theomonnom): Add async support to wrap_variants ...
// TODO(theomonnom): Add async support to wrap_variants ...
pub(crate) async fn update_info(&self, info: ParticipantInfo) {
match self {
Self::Local(inner) => inner.clone().update_info(info).await,
@@ -84,20 +84,20 @@ impl ParticipantHandle {
}
impl ParticipantInternalTrait for ParticipantHandle {
wrap_variants!(
enum_dispatch!(
[Local, Remote]
fnc!(internal_events, Arc<ParticipantEvents>, []);
fnc!(internal_events, &Self, [], Arc<ParticipantEvents>);
);
}
impl ParticipantTrait for ParticipantHandle {
wrap_variants!(
enum_dispatch!(
[Local, Remote]
fnc!(events, Arc<ParticipantEvents>, []);
fnc!(sid, ParticipantSid, []);
fnc!(identity, ParticipantIdentity, []);
fnc!(name, String, []);
fnc!(metadata, String, []);
fnc!(events, &Self, [], Arc<ParticipantEvents>);
fnc!(sid, &Self, [], ParticipantSid);
fnc!(identity, &Self, [], ParticipantIdentity);
fnc!(name, &Self, [], String);
fnc!(metadata, &Self, [], String);
);
}
crates/livekit-core/src/room/publication/mod.rs
+15 -11
View File
@@ -5,7 +5,7 @@ use crate::room::id::TrackSid;
use crate::room::track::local_track::LocalTrackHandle;
use crate::room::track::remote_track::RemoteTrackHandle;
use crate::room::track::{TrackHandle, TrackKind, TrackSource};
use crate::utils::wrap_variants;
use livekit_utils::enum_dispatch;
use parking_lot::Mutex;
use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
use std::sync::Arc;
@@ -38,7 +38,11 @@ pub(super) struct TrackPublicationShared {
}
impl TrackPublicationShared {
pub fn new(info: TrackInfo, participant: ParticipantSid, track: Option<TrackHandle>) -> Arc<Self> {
pub fn new(
info: TrackInfo,
participant: ParticipantSid,
track: Option<TrackHandle>,
) -> Arc<Self> {
Arc::new(Self {
track: Mutex::new(track),
name: Mutex::new(info.name),
@@ -88,21 +92,21 @@ impl TrackPublication {
}
impl TrackPublicationInternalTrait for TrackPublication {
wrap_variants!(
enum_dispatch!(
[Local, Remote]
fnc!(update_track, (), [track: Option<TrackHandle>]);
fnc!(update_info, (), [info: TrackInfo]);
fnc!(update_track, &Self, [track: Option<TrackHandle>], ());
fnc!(update_info, &Self, [info: TrackInfo], ());
);
}
impl TrackPublicationTrait for TrackPublication {
wrap_variants!(
enum_dispatch!(
[Local, Remote]
fnc!(sid, TrackSid, []);
fnc!(name, String, []);
fnc!(kind, TrackKind, []);
fnc!(source, TrackSource, []);
fnc!(simulcasted, bool, []);
fnc!(sid, &Self, [], TrackSid);
fnc!(name, &Self, [], String);
fnc!(kind, &Self, [], TrackKind);
fnc!(source, &Self, [], TrackSource);
fnc!(simulcasted, &Self, [], bool);
);
}
crates/livekit-core/src/room/track/mod.rs
+8 -8
View File
@@ -4,7 +4,7 @@ use crate::room::track::local_audio_track::LocalAudioTrack;
use crate::room::track::local_video_track::LocalVideoTrack;
use crate::room::track::remote_audio_track::RemoteAudioTrack;
use crate::room::track::remote_video_track::RemoteVideoTrack;
use crate::utils::wrap_variants;
use livekit_utils::enum_dispatch;
use livekit_webrtc::media_stream::{MediaStreamTrackHandle, MediaStreamTrackTrait};
use parking_lot::Mutex;
use std::sync::atomic::AtomicU8;
@@ -150,14 +150,14 @@ pub enum TrackHandle {
}
impl TrackTrait for TrackHandle {
wrap_variants!(
enum_dispatch!(
[LocalVideo, LocalAudio, RemoteVideo, RemoteAudio]
fnc!(sid, TrackSid, []);
fnc!(name, String, []);
fnc!(kind, TrackKind, []);
fnc!(stream_state, StreamState, []);
fnc!(start, (), []);
fnc!(stop, (), []);
fnc!(sid, &Self, [], TrackSid);
fnc!(name, &Self, [], String);
fnc!(kind, &Self, [], TrackKind);
fnc!(stream_state, &Self, [], StreamState);
fnc!(start, &Self, [], ());
fnc!(stop, &Self, [], ());
);
}
crates/livekit-core/src/room/track/remote_track.rs
+8 -8
View File
@@ -5,7 +5,7 @@ use crate::room::id::TrackSid;
use crate::room::track::remote_audio_track::RemoteAudioTrack;
use crate::room::track::remote_video_track::RemoteVideoTrack;
use crate::room::track::TrackHandle;
use crate::utils::wrap_variants;
use livekit_utils::enum_dispatch;
use super::TrackTrait;
@@ -16,14 +16,14 @@ pub enum RemoteTrackHandle {
}
impl TrackTrait for RemoteTrackHandle {
wrap_variants!(
enum_dispatch!(
[Audio, Video]
fnc!(sid, TrackSid, []);
fnc!(name, String, []);
fnc!(kind, TrackKind, []);
fnc!(stream_state, StreamState, []);
fnc!(start, (), []);
fnc!(stop, (), []);
fnc!(sid, &Self, [], TrackSid);
fnc!(name, &Self, [], String);
fnc!(kind, &Self, [], TrackKind);
fnc!(stream_state, &Self, [], StreamState);
fnc!(start, &Self, [], ());
fnc!(stop, &Self, [], ());
);
}
crates/livekit-utils/Cargo.toml
+8
View File
@@ -0,0 +1,8 @@
[package]
name = "livekit-utils"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
crates/livekit-core/src/utils.rs → crates/livekit-utils/src/enum_dispatch.rs
+7 -8
View File
@@ -1,4 +1,5 @@
macro_rules! wrap_variants {
#[macro_export]
macro_rules! enum_dispatch {
// This arm is used to avoid nested loops with the arguments
// The arguments are transformed to $combined_args TokenTree
(@match $self:ident $fnc:ident $combined_args:tt [$($variant:ident),+]) => {
@@ -9,18 +10,16 @@ macro_rules! wrap_variants {
}
};
($fnc:ident, $ret:ty, [$($arg:ident: $t:ty),*], [$($variant:ident),+]) => {
fn $fnc(&self, $($arg: $t),*) -> $ret {
wrap_variants!(@match self $fnc ($($arg,)*) [$($variant),+])
($fnc:ident, $self:ty, [$($arg:ident: $t:ty),*], $ret:ty, [$($variant:ident),+]) => {
fn $fnc(self: $self, $($arg: $t),*) -> $ret {
enum_dispatch!(@match self $fnc ($($arg,)*) [$($variant),+])
}
};
($variants:tt $(fnc!($fnc:ident, $ret:ty, $args:tt);)+) => {
($variants:tt $(fnc!($fnc:ident, $self:ty, $args:tt, $ret:ty);)+) => {
$(
wrap_variants!($fnc, $ret, $args, $variants);
enum_dispatch!($fnc, $self, $args, $ret, $variants);
)*
};
}
pub(crate) use wrap_variants;
crates/livekit-utils/src/lib.rs
+1
View File
@@ -0,0 +1 @@
pub mod enum_dispatch;
crates/livekit-webrtc/Cargo.toml
+2 -1
View File
@@ -6,10 +6,11 @@ homepage = "https://livekit.io"
[dependencies]
libwebrtc-sys = { path = "./libwebrtc-sys" }
livekit-utils = { path = "../livekit-utils" }
tokio = { version = "1", features = ["full"] }
cxx = "1.0"
log = "0.4"
thiserror = "1.0"
[dev-dependencies]
env_logger = "0.9"
env_logger = "0.9"
crates/livekit-webrtc/libwebrtc-sys/.gitignore
-1
View File
@@ -1,2 +1 @@
/libwebrtc
/cmake-build-debug
crates/livekit-webrtc/libwebrtc-sys/build.rs
+14 -7
View File
@@ -55,17 +55,17 @@ fn main() {
let target_os = "windows";
//let target_arch = "arm64";
let libwebrtc_dir = path::PathBuf::from("libwebrtc");
let libwebrtc_dir = path::PathBuf::from("libwebrtc/src");
// Just required for the bridge build to succeed.
let includes = &[
path::PathBuf::from("./include"),
libwebrtc_dir.join("include/"),
libwebrtc_dir.join("include/third_party/abseil-cpp/"),
libwebrtc_dir.join("include/third_party/libc++/"),
libwebrtc_dir.clone(),
libwebrtc_dir.join("third_party/abseil-cpp/"),
libwebrtc_dir.join("third_party/libc++/"),
// For mac & ios
libwebrtc_dir.join("include/sdk/objc"),
libwebrtc_dir.join("include/sdk/objc/base"),
libwebrtc_dir.join("sdk/objc"),
libwebrtc_dir.join("sdk/objc/base"),
];
let mut builder = cxx_build::bridges(&[
@@ -81,6 +81,7 @@ fn main() {
"src/webrtc.rs",
"src/video_frame.rs",
"src/video_frame_buffer.rs",
"src/yuv_helper.rs",
]);
builder.file("src/peer_connection.cpp");
@@ -100,7 +101,12 @@ fn main() {
println!(
"cargo:rustc-link-search=native={}",
libwebrtc_dir.canonicalize().unwrap().to_str().unwrap()
libwebrtc_dir
.join("out/Default/obj")
.canonicalize()
.unwrap()
.to_str()
.unwrap()
);
match target_os {
@@ -124,6 +130,7 @@ fn main() {
.flag("/std:c++17")
.flag("/EHsc")
.define("WEBRTC_WIN", None)
//.define("WEBRTC_ENABLE_SYMBOL_EXPORT", None) Not necessary when using WebRTC as a static library
.define("NOMINMAX", None);
}
"macos" => {
crates/livekit-webrtc/libwebrtc-sys/compile_flags.txt
+3 -3
View File
@@ -1,8 +1,8 @@
-xc++
-std=c++17
-Iinclude
-Ilibwebrtc/include
-Ilibwebrtc/include/third_party/abseil-cpp
-Ilibwebrtc/include/third_party/libc++
-Ilibwebrtc/src
-Ilibwebrtc/src/third_party/abseil-cpp
-Ilibwebrtc/src/third_party/libc++
-I../../../target/cxxbridge
-DWEBRTC_WIN
crates/livekit-webrtc/libwebrtc-sys/include/livekit/yuv_helper.h
+30
View File
@@ -0,0 +1,30 @@
//
// Created by Théo Monnom on 01/12/2022.
//
#ifndef CLIENT_SDK_NATIVE_YUV_HELPER_H
#define CLIENT_SDK_NATIVE_YUV_HELPER_H
#include <memory>
#include "api/video/yuv_helper.h"
namespace livekit {
static void i420_to_abgr(const uint8_t* src_y,
int src_stride_y,
const uint8_t* src_u,
int src_stride_u,
const uint8_t* src_v,
int src_stride_v,
uint8_t* dst_rgba,
int dst_stride_abgr,
int width,
int height) {
webrtc::I420ToABGR(src_y, src_stride_y, src_u, src_stride_u, src_v,
src_stride_v, dst_rgba, dst_stride_abgr, width, height);
}
} // namespace livekit
#endif // CLIENT_SDK_NATIVE_YUV_HELPER_H
crates/livekit-webrtc/libwebrtc-sys/libwebrtc/.gclient
+11
View File
@@ -0,0 +1,11 @@
solutions = [
{
"name" : 'src',
"url" : 'https://github.com/webrtc-sdk/webrtc.git',
"deps_file" : 'DEPS',
"managed" : False,
"custom_deps" : {
},
"custom_vars": {},
},
]
crates/livekit-webrtc/libwebrtc-sys/libwebrtc/.gitignore
+3
View File
@@ -0,0 +1,3 @@
.cipd
src
.gclient_*
crates/livekit-webrtc/libwebrtc-sys/libwebrtc/compile.py
+19
View File
@@ -0,0 +1,19 @@
import subprocess
GN_ARGS = [
"is_debug=false",
"treat_warnings_as_errors=false",
'target_os="win"',
'target_cpu="x64"',
"rtc_include_tests=false",
"rtc_use_h264=false",
"is_component_build=false",
"rtc_build_examples=false",
"use_rtti=true",
"rtc_build_tools=false",
"use_custom_libcxx=false",
"strip_debug_info=true",
"symbol_level=0"
]
subprocess.call(["gn", "gen", "out/Default", "--args=" + ' '.join(GN_ARGS)], shell=True)
crates/livekit-webrtc/libwebrtc-sys/src/lib.rs
+1
View File
@@ -10,6 +10,7 @@ pub mod rtp_transceiver;
pub mod video_frame;
pub mod video_frame_buffer;
pub mod webrtc;
pub mod yuv_helper;
pub const MEDIA_TYPE_VIDEO: &str = "video";
pub const MEDIA_TYPE_AUDIO: &str = "audio";
crates/livekit-webrtc/libwebrtc-sys/src/yuv_helper.rs
+19
View File
@@ -0,0 +1,19 @@
#[cxx::bridge(namespace = "livekit")]
pub mod ffi {
unsafe extern "C++" {
include!("livekit/yuv_helper.h");
unsafe fn i420_to_abgr(
src_y: *const u8,
src_stride_y: i32,
src_u: *const u8,
src_stride_u: i32,
src_v: *const u8,
src_stride_v: i32,
dst_abgr: *mut u8,
dst_stride_abgr: i32,
width: i32,
height: i32,
);
}
}
crates/livekit-webrtc/src/lib.rs
+1
View File
@@ -9,3 +9,4 @@ pub mod rtp_transceiver;
pub mod video_frame;
pub mod video_frame_buffer;
pub mod webrtc;
pub mod yuv_helper;
crates/livekit-webrtc/src/video_frame_buffer.rs
+15 -3
View File
@@ -1,5 +1,6 @@
use cxx::UniquePtr;
use libwebrtc_sys::video_frame_buffer as vfb_sys;
use livekit_utils::enum_dispatch;
use std::pin::Pin;
use std::slice;
use vfb_sys::ffi::VideoFrameBufferType;
@@ -53,6 +54,15 @@ impl VideoFrameBuffer {
}
}
impl VideoFrameBufferTrait for VideoFrameBuffer {
enum_dispatch!(
[Native, I420, I420A, I422, I444, I010, NV12]
fnc!(width, &Self, [], i32);
fnc!(height, &Self, [], i32);
fnc!(to_i420, Self, [], I420Buffer);
);
}
macro_rules! recursive_cast {
($ptr:expr $(, $fnc:ident)*) => {
{
@@ -146,21 +156,23 @@ macro_rules! impl_yuv8_buffer {
fn data_y(&self) -> &[u8] {
let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
unsafe {
slice::from_raw_parts((*ptr).data_y(), self.stride_y().try_into().unwrap())
slice::from_raw_parts((*ptr).data_y(), (self.width() * self.height()) as usize)
}
}
fn data_u(&self) -> &[u8] {
let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
unsafe {
slice::from_raw_parts((*ptr).data_u(), self.stride_u().try_into().unwrap())
let chroma_height = (self.height() + 1) / 2;
slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * chroma_height) as usize)
}
}
fn data_v(&self) -> &[u8] {
let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*);
unsafe {
slice::from_raw_parts((*ptr).data_v(), self.stride_v().try_into().unwrap())
let chroma_height = (self.height() + 1) / 2;
slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * chroma_height) as usize)
}
}
}
crates/livekit-webrtc/src/yuv_helper.rs
+43
View File
@@ -0,0 +1,43 @@
use std::convert::TryInto;
use libwebrtc_sys::yuv_helper as yuv_sys;
pub fn i420_to_abgr(
src_y: &[u8],
src_stride_y: i32,
src_u: &[u8],
src_stride_u: i32,
src_v: &[u8],
src_stride_v: i32,
dst_abgr: &mut [u8],
dst_stride_abgr: i32,
width: i32,
height: i32,
) {
// Assert minimum capacity for safety
let chroma_height = (height + 1) / 2; // the buffer should be padded?
let min_y: usize = (src_stride_y * height).try_into().unwrap();
let min_u: usize = (src_stride_u * chroma_height).try_into().unwrap();
let min_v: usize = (src_stride_v * chroma_height).try_into().unwrap();
let min_abgr: usize = (dst_stride_abgr * height).try_into().unwrap();
assert!(src_y.len() >= min_y);
assert!(src_u.len() >= min_u);
assert!(src_v.len() >= min_v);
assert!(dst_abgr.len() >= min_abgr);
unsafe {
yuv_sys::ffi::i420_to_abgr(
src_y.as_ptr(),
src_stride_y,
src_u.as_ptr(),
src_stride_u,
src_v.as_ptr(),
src_stride_v,
dst_abgr.as_mut_ptr(),
dst_stride_abgr,
width,
height,
);
}
}
examples/Cargo.lock
Generated
+1616 -2
View File
File diff suppressed because it is too large Load Diff
examples/Cargo.toml
+2 -1
View File
@@ -1,3 +1,4 @@
[workspace]
members = ["*"]
exclude = ["target"]
exclude = ["target"]
resolver = "2"
examples/simple_room/Cargo.toml
+7
View File
@@ -9,3 +9,10 @@ tracing = "0.1"
tracing-subscriber = "0.3"
livekit = { path = "../.." }
futures = "0.3"
wgpu = "0.14.0"
winit = "0.27.5"
egui = { git = "https://github.com/emilk/egui" }
egui-wgpu = { git = "https://github.com/emilk/egui", features = ["winit"] }
egui-winit = { git = "https://github.com/emilk/egui" }
egui_demo_lib = { git = "https://github.com/emilk/egui" }
examples/simple_room/src/main.rs
+181 -25
View File
@@ -1,36 +1,192 @@
use std::time::Duration;
use std::convert::TryInto;
use std::ops::DerefMut;
use std::{num::NonZeroU32, time::Duration};
use egui_wgpu::WgpuConfiguration;
use livekit::webrtc::media_stream::VideoTrack;
use livekit::webrtc::video_frame_buffer::{
PlanarYuv8Buffer, PlanarYuvBuffer, VideoFrameBufferTrait,
};
use livekit::webrtc::yuv_helper;
use std::sync::{Arc, Mutex};
use video_renderer::VideoRenderer;
use wgpu::{Device, Queue};
use livekit::room::RoomError;
use livekit::room::{track::remote_track::RemoteTrackHandle, Room};
use tokio::time::sleep;
use livekit::room::track::remote_track::RemoteTrackHandle;
use livekit::room::{Room, RoomError};
const URL: &str = "ws://localhost:7880";
const TOKEN : &str = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjIzODQ4MDY0NzMsImlzcyI6IkFQSXpLYkFTaUNWYWtnSiIsIm5hbWUiOiJuYXRpdmUiLCJuYmYiOjE2NjQ4MDY0NzMsInN1YiI6Im5hdGl2ZSIsInZpZGVvIjp7InJvb21DcmVhdGUiOnRydWUsInJvb21Kb2luIjp0cnVlfX0.BgVdBnq3XFD3_BQHoe1azqjifYysubgFl6Qlzu9IQGI";
// eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjIzODQ4MDY3MzAsImlzcyI6IkFQSXpLYkFTaUNWYWtnSiIsIm5hbWUiOiJ3ZWIiLCJuYmYiOjE2NjQ4MDY3MzAsInN1YiI6IndlYiIsInZpZGVvIjp7InJvb21DcmVhdGUiOnRydWUsInJvb21Kb2luIjp0cnVlfX0.VbDoULjX1CVGZu2sPy3SvWYlVZUBXxQVPmdB9BnmlN4
#[tokio::main]
async fn main() -> Result<(), RoomError> {
tracing_subscriber::fmt::init();
mod video_renderer;
let mut room = Room::new();
room.events()
.on_participant_connected(|_event| async move {});
use winit::{
event::*,
event_loop::{ControlFlow, EventLoop},
window::{Window, WindowBuilder, WindowId},
};
room.events().on_track_subscribed(|event| async move {
let track = event.publication.track().unwrap();
if let RemoteTrackHandle::Video(video_track) = track {
let rtc_track = video_track.rtc_track();
rtc_track.set_should_receive(true);
rtc_track.on_frame(Box::new(|_frame, _buffer| {
// called on libwebrtc worker_thread
println!("Received frame");
}));
}
});
room.connect(URL, TOKEN).await?;
sleep(Duration::from_secs(200)).await;
Ok(())
struct AppState {
room: Room,
demo: egui_demo_lib::DemoWindows,
egui_context: egui::Context,
egui_state: egui_winit::State,
egui_painter: egui_wgpu::winit::Painter,
window: winit::window::Window,
}
impl AppState {
fn on_event<T>(&mut self, event: Event<'_, T>, control_flow: &mut ControlFlow) {
match event {
Event::WindowEvent { window_id, event } => {
if let Some(flow) = self.on_window_event(window_id, event) {
*control_flow = flow;
}
}
Event::RedrawRequested(window_id) if window_id == self.window.id() => {
self.render();
}
Event::RedrawEventsCleared => {
self.window.request_redraw();
}
_ => {}
};
}
fn on_window_event(
&mut self,
_window_id: WindowId,
event: WindowEvent<'_>,
) -> Option<ControlFlow> {
if self
.egui_state
.on_event(&self.egui_context, &event)
.consumed
{
return None;
}
match event {
WindowEvent::CloseRequested => Some(ControlFlow::Exit),
WindowEvent::Resized(inner_size) => {
self.egui_painter
.on_window_resized(inner_size.width, inner_size.height);
None
}
WindowEvent::ScaleFactorChanged { new_inner_size, .. } => {
self.egui_painter
.on_window_resized(new_inner_size.width, new_inner_size.height);
None
}
_ => None,
}
}
fn render(&mut self) {
let raw_inputs = self.egui_state.take_egui_input(&self.window);
let full_output = self.egui_context.run(raw_inputs, |ctx| {
//self.ui(ctx);
});
let clipped_primitives = self.egui_context.tessellate(full_output.shapes);
self.egui_painter.paint_and_update_textures(
egui_winit::native_pixels_per_point(&self.window),
egui::Rgba::BLACK,
&clipped_primitives,
&full_output.textures_delta,
);
self.egui_state.handle_platform_output(
&self.window,
&self.egui_context,
full_output.platform_output,
);
}
}
struct App {
rt: tokio::runtime::Runtime,
}
impl App {
pub fn new(rt: tokio::runtime::Runtime) -> Self {
Self { rt }
}
pub fn run(&mut self) {
self.rt.block_on(async {
let event_loop = EventLoop::new();
let window = WindowBuilder::new().build(&event_loop).unwrap();
let egui_context = egui::Context::default();
let egui_state = egui_winit::State::new(&event_loop);
let mut egui_painter =
egui_wgpu::winit::Painter::new(WgpuConfiguration::default(), 1, 32);
unsafe {
egui_painter.set_window(Some(&window));
}
let mut inner = AppState {
room: Room::new(),
demo: egui_demo_lib::DemoWindows::default(),
egui_context,
egui_state,
egui_painter,
window,
};
inner
.room
.events()
.on_participant_connected(|_event| async move {});
inner.room.events().on_track_subscribed({
let test = Arc::new(Mutex::new(None));
let egui_render = inner.egui_painter.render_state().clone().unwrap();
move |event| {
let test = test.clone();
let egui_render = egui_render.clone();
async move {
let track = event.publication.track().unwrap();
if let RemoteTrackHandle::Video(video_track) = track {
*test.lock().unwrap() =
Some(VideoRenderer::new(egui_render, video_track.rtc_track()))
}
}
}
});
inner.room.connect(URL, TOKEN).await.unwrap();
tokio::spawn(async {
loop {
println!("Test");
tokio::time::sleep(Duration::from_secs(5)).await;
}
});
tokio::task::block_in_place(move || loop {
event_loop.run(move |event, _, control_flow| {
inner.on_event(event, control_flow);
});
});
});
}
}
fn main() {
let rt = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.unwrap();
let mut app = App::new(rt);
app.run();
}
examples/simple_room/src/video_renderer.rs
+172
View File
@@ -0,0 +1,172 @@
use livekit::webrtc::media_stream::VideoTrack;
use livekit::webrtc::video_frame_buffer::PlanarYuv8Buffer;
use livekit::webrtc::video_frame_buffer::PlanarYuvBuffer;
use livekit::webrtc::video_frame_buffer::VideoFrameBufferTrait;
use livekit::webrtc::yuv_helper;
use std::convert::TryInto;
use std::num::NonZeroU32;
use std::{
ops::DerefMut,
sync::{Arc, Mutex},
};
pub struct VideoRenderer {
internal: Arc<Mutex<RendererInternal>>,
rtc_track: Arc<VideoTrack>,
}
struct RendererInternal {
render_state: egui_wgpu::RenderState,
width: u32,
height: u32,
rgba_data: Vec<u8>,
texture: Option<wgpu::Texture>,
texture_view: Option<wgpu::TextureView>,
egui_texture: Option<egui::TextureId>,
}
impl RendererInternal {
fn ensure_texture_size(&mut self, width: u32, height: u32) {
if self.width == width && self.height == height {
return;
}
self.width = width;
self.height = height;
self.rgba_data.resize((width * height * 4) as usize, 0);
self.texture = Some(
self.render_state
.device
.create_texture(&wgpu::TextureDescriptor {
label: Some("lk-videotexture"),
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
dimension: wgpu::TextureDimension::D2,
size: wgpu::Extent3d {
width,
height,
..Default::default()
},
sample_count: 1,
mip_level_count: 1,
format: wgpu::TextureFormat::Rgba8UnormSrgb,
}),
);
self.texture_view = Some(self.texture.as_mut().unwrap().create_view(
&wgpu::TextureViewDescriptor {
label: Some("lk-videotexture-view"),
format: Some(wgpu::TextureFormat::Rgba8UnormSrgb),
dimension: Some(wgpu::TextureViewDimension::D2),
mip_level_count: NonZeroU32::new(1),
array_layer_count: NonZeroU32::new(1),
..Default::default()
},
));
if let Some(texture_id) = self.egui_texture {
// Update the existing texture
self.render_state
.renderer
.write()
.update_egui_texture_from_wgpu_texture(
&*self.render_state.device,
self.texture_view.as_ref().unwrap(),
wgpu::FilterMode::Linear,
texture_id,
);
} else {
self.egui_texture = Some(self.render_state.renderer.write().register_native_texture(
&*self.render_state.device,
self.texture_view.as_ref().unwrap(),
wgpu::FilterMode::Linear,
));
}
}
}
impl VideoRenderer {
pub fn new(render_state: egui_wgpu::RenderState, rtc_track: Arc<VideoTrack>) -> Self {
let internal = Arc::new(Mutex::new(RendererInternal {
render_state,
width: 0,
height: 0,
rgba_data: Vec::default(),
texture: None,
texture_view: None,
egui_texture: None,
}));
rtc_track.on_frame({
let internal = internal.clone();
Box::new(move |_frame, buffer| {
let mut internal = internal.lock().unwrap();
let buffer = buffer.to_i420();
let width: u32 = buffer.width().try_into().unwrap();
let height: u32 = buffer.height().try_into().unwrap();
internal.ensure_texture_size(width, height);
let rgba_ptr = internal.rgba_data.deref_mut();
let rgba_stride = buffer.width() * 4;
yuv_helper::i420_to_abgr(
buffer.data_y(),
buffer.stride_y(),
buffer.data_u(),
buffer.stride_u(),
buffer.data_v(),
buffer.stride_v(),
rgba_ptr,
rgba_stride,
buffer.width(),
buffer.height(),
);
let copy_desc = wgpu::ImageCopyTexture {
texture: internal.texture.as_ref().unwrap(),
mip_level: 0,
origin: wgpu::Origin3d::default(),
aspect: wgpu::TextureAspect::default(),
};
let copy_layout = wgpu::ImageDataLayout {
bytes_per_row: Some(NonZeroU32::new(width * 4).unwrap()),
..Default::default()
};
let copy_size = wgpu::Extent3d {
width,
height,
..Default::default()
};
internal.render_state.queue.write_texture(
copy_desc,
&internal.rgba_data,
copy_layout,
copy_size,
);
println!("wrote");
})
});
Self {
rtc_track,
internal,
}
}
pub fn texture_id(&self) -> Option<egui::TextureId> {
self.internal.lock().unwrap().egui_texture.clone()
}
}
impl Drop for VideoRenderer {
fn drop(&mut self) {
self.rtc_track.on_frame(Box::new(|_, _| {}));
}
}
src/lib.rs
+4
View File
@@ -1,2 +1,6 @@
// export everything inside livekit-core
pub use livekit_core::*;
pub mod webrtc {
pub use livekit_webrtc::*;
}