diff --git a/Cargo.lock b/Cargo.lock index 5d887b7..60bf3fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -418,9 +418,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" +checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5" dependencies = [ "futures-core", "futures-sink", @@ -428,9 +428,9 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" +checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608" [[package]] name = "futures-executor" @@ -462,9 +462,9 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" +checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364" [[package]] name = "futures-task" @@ -779,7 +779,6 @@ checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" name = "livekit" version = "0.1.1" dependencies = [ - "futures", "futures-util", "lazy_static", "livekit-utils", @@ -792,6 +791,7 @@ dependencies = [ "serde_json", "thiserror", "tokio", + "tokio-stream", "tokio-tungstenite", "tracing", "url", @@ -801,6 +801,7 @@ dependencies = [ name = "livekit-ffi" version = "0.1.1" dependencies = [ + "futures-util", "lazy_static", "livekit", "parking_lot", @@ -815,6 +816,7 @@ dependencies = [ name = "livekit-utils" version = "0.1.1" dependencies = [ + "futures-util", "parking_lot", "tokio", ] @@ -825,10 +827,15 @@ version = "0.1.1" dependencies = [ "cxx", "env_logger 0.9.3", + "futures", + "js-sys", "livekit-utils", "log", "thiserror", "tokio", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", "webrtc-sys", ] @@ -1354,17 +1361,6 @@ dependencies = [ "serde", ] -[[package]] -name = "sha-1" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - [[package]] name = "sha1" version = "0.10.5" @@ -1536,9 +1532,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.23.0" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eab6d665857cc6ca78d6e80303a02cea7a7851e85dfbd77cbdc09bd129f1ef46" +checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" dependencies = [ "autocfg", "bytes", @@ -1551,7 +1547,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] @@ -1576,10 +1572,21 @@ dependencies = [ ] [[package]] -name = "tokio-tungstenite" -version = "0.17.2" +name = "tokio-stream" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f714dd15bead90401d77e04243611caec13726c2408afd5b31901dfcdcb3b181" +checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54319c93411147bced34cb5609a80e0a8e44c5999c93903a81cd866630ec0bfd" dependencies = [ "futures-util", "log", @@ -1649,9 +1656,9 @@ checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "tungstenite" -version = "0.17.3" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e27992fd6a8c29ee7eef28fc78349aa244134e10ad447ce3b9f0ac0ed0fa4ce0" +checksum = "30ee6ab729cd4cf0fd55218530c4522ed30b7b6081752839b68fcec8d0960788" dependencies = [ "base64 0.13.1", "byteorder", @@ -1661,7 +1668,7 @@ dependencies = [ "log", "native-tls", "rand", - "sha-1", + "sha1", "thiserror", "url", "utf-8", @@ -1899,12 +1906,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ "windows_aarch64_gnullvm", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", + "windows_aarch64_msvc 0.42.1", + "windows_i686_gnu 0.42.1", + "windows_i686_msvc 0.42.1", + "windows_x86_64_gnu 0.42.1", "windows_x86_64_gnullvm", - "windows_x86_64_msvc 0.42.2", + "windows_x86_64_msvc 0.42.1", ] [[package]] @@ -1918,24 +1925,24 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.42.2" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" dependencies = [ "windows_aarch64_gnullvm", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", + "windows_aarch64_msvc 0.42.1", + "windows_i686_gnu 0.42.1", + "windows_i686_msvc 0.42.1", + "windows_x86_64_gnu 0.42.1", "windows_x86_64_gnullvm", - "windows_x86_64_msvc 0.42.2", + "windows_x86_64_msvc 0.42.1", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.2" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" [[package]] name = "windows_aarch64_msvc" @@ -1945,9 +1952,9 @@ checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" [[package]] name = "windows_aarch64_msvc" -version = "0.42.2" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" [[package]] name = "windows_i686_gnu" @@ -1957,9 +1964,9 @@ checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" [[package]] name = "windows_i686_gnu" -version = "0.42.2" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" [[package]] name = "windows_i686_msvc" @@ -1969,9 +1976,9 @@ checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" [[package]] name = "windows_i686_msvc" -version = "0.42.2" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" [[package]] name = "windows_x86_64_gnu" @@ -1981,15 +1988,15 @@ checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" [[package]] name = "windows_x86_64_gnu" -version = "0.42.2" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.2" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" [[package]] name = "windows_x86_64_msvc" @@ -1999,9 +2006,9 @@ checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" [[package]] name = "windows_x86_64_msvc" -version = "0.42.2" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" [[package]] name = "winreg" diff --git a/examples/Cargo.lock b/examples/Cargo.lock index a59cab1..d2e6cd0 100644 --- a/examples/Cargo.lock +++ b/examples/Cargo.lock @@ -120,9 +120,9 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" [[package]] name = "ash" -version = "0.37.1+1.3.235" +version = "0.37.2+1.3.238" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "911015c962d56e2e4052f40182ca5462ba60a3d2ff04e827c365a0ab3d65726d" +checksum = "28bf19c1f0a470be5fbf7522a308a05df06610252c5bcf5143e1b23f629a9a03" dependencies = [ "libloading", ] @@ -145,6 +145,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" + [[package]] name = "base64ct" version = "1.5.3" @@ -166,6 +172,12 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" +[[package]] +name = "bit_field" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc827186963e592360843fb5ba4b973e145841266c1357f7180c43526f2e5b61" + [[package]] name = "bitflags" version = "1.3.2" @@ -356,6 +368,12 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "combine" version = "4.6.6" @@ -443,6 +461,40 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset 0.8.0", + "scopeguard", +] + [[package]] name = "crossbeam-utils" version = "0.8.14" @@ -475,6 +527,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + [[package]] name = "crypto-common" version = "0.1.6" @@ -491,36 +549,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35" -[[package]] -name = "curl" -version = "0.4.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "509bd11746c7ac09ebd19f0b17782eae80aadee26237658a6b4808afb5c11a22" -dependencies = [ - "curl-sys", - "libc", - "openssl-probe", - "openssl-sys", - "schannel", - "socket2", - "winapi", -] - -[[package]] -name = "curl-sys" -version = "0.4.59+curl-7.86.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cfce34829f448b08f55b7db6d0009e23e2e86a34e8c2b366269bf5799b4a407" -dependencies = [ - "cc", - "libc", - "libz-sys", - "openssl-sys", - "pkg-config", - "vcpkg", - "winapi", -] - [[package]] name = "cxx" version = "1.0.85" @@ -743,6 +771,15 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "encoding_rs" +version = "0.8.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" +dependencies = [ + "cfg-if", +] + [[package]] name = "epaint" version = "0.20.0" @@ -779,6 +816,22 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "exr" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8af5ef47e2ed89d23d0ecbc1b681b30390069de70260937877514377fc24feb" +dependencies = [ + "bit_field", + "flume", + "half", + "lebe", + "miniz_oxide", + "smallvec", + "threadpool", + "zune-inflate", +] + [[package]] name = "fastrand" version = "1.8.0" @@ -816,6 +869,19 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "flume" +version = "0.10.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1657b4441c3403d9f7b3409e47575237dac27b1b5726df654a6ecbf92f0f7577" +dependencies = [ + "futures-core", + "futures-sink", + "nanorand", + "pin-project", + "spin", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1020,8 +1086,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", +] + +[[package]] +name = "gif" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3edd93c6756b4dfaf2709eafcc345ba2636565295c198a9cfbf75fa5e3e00b06" +dependencies = [ + "color_quant", + "weezl", ] [[package]] @@ -1081,6 +1159,34 @@ dependencies = [ "bitflags", ] +[[package]] +name = "h2" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b4af3693f1b705df946e9fe5631932443781d0aabb423b62fcd4d73f6d2fd0" +dependencies = [ + "crunchy", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -1131,12 +1237,66 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + [[package]] name = "httparse" version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + +[[package]] +name = "hyper" +version = "0.14.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -1153,6 +1313,25 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "image" +version = "0.24.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b7ea949b537b0fd0af141fff8c77690f2ce96f4f41f042ccb6c69c6c965945" +dependencies = [ + "bytemuck", + "byteorder", + "color_quant", + "exr", + "gif", + "jpeg-decoder", + "num-rational", + "num-traits", + "png", + "scoped_threadpool", + "tiff", +] + [[package]] name = "indexmap" version = "1.9.2" @@ -1175,6 +1354,12 @@ dependencies = [ "web-sys", ] +[[package]] +name = "ipnet" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" + [[package]] name = "itertools" version = "0.10.5" @@ -1220,10 +1405,19 @@ dependencies = [ ] [[package]] -name = "js-sys" -version = "0.3.60" +name = "jpeg-decoder" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +checksum = "bc0000e42512c92e31c2252315bda326620a4e034105e900c98ec492fa077b3e" +dependencies = [ + "rayon", +] + +[[package]] +name = "js-sys" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" dependencies = [ "wasm-bindgen", ] @@ -1245,6 +1439,12 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +[[package]] +name = "lebe" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" + [[package]] name = "libc" version = "0.2.139" @@ -1261,18 +1461,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "libz-sys" -version = "1.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "link-cplusplus" version = "1.0.8" @@ -1286,7 +1474,6 @@ dependencies = [ name = "livekit" version = "0.1.1" dependencies = [ - "futures", "futures-util", "lazy_static", "livekit-utils", @@ -1299,6 +1486,7 @@ dependencies = [ "serde_json", "thiserror", "tokio", + "tokio-stream", "tokio-tungstenite", "tracing", "url", @@ -1308,6 +1496,7 @@ dependencies = [ name = "livekit-utils" version = "0.1.1" dependencies = [ + "futures-util", "parking_lot", "tokio", ] @@ -1317,10 +1506,15 @@ name = "livekit-webrtc" version = "0.1.1" dependencies = [ "cxx", + "futures", + "js-sys", "livekit-utils", "log", "thiserror", "tokio", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", "webrtc-sys", ] @@ -1376,6 +1570,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "memoffset" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" +dependencies = [ + "autocfg", +] + [[package]] name = "metal" version = "0.24.0" @@ -1390,6 +1593,12 @@ dependencies = [ "objc", ] +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1443,6 +1652,15 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "nanorand" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" +dependencies = [ + "getrandom", +] + [[package]] name = "native-tls" version = "0.2.11" @@ -1528,7 +1746,7 @@ dependencies = [ "bitflags", "cfg-if", "libc", - "memoffset", + "memoffset 0.6.5", ] [[package]] @@ -1541,7 +1759,7 @@ dependencies = [ "bitflags", "cfg-if", "libc", - "memoffset", + "memoffset 0.6.5", ] [[package]] @@ -1570,6 +1788,27 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.15" @@ -1783,6 +2022,26 @@ dependencies = [ "indexmap", ] +[[package]] +name = "pin-project" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.9" @@ -1979,6 +2238,28 @@ dependencies = [ "cty", ] +[[package]] +name = "rayon" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -2031,6 +2312,43 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1382d1f0a252c4bf97dc20d979a2fdd05b024acd7c2ed0f7595d7817666a157" +[[package]] +name = "reqwest" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9" +dependencies = [ + "base64 0.21.0", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "rustc-hash" version = "1.1.0" @@ -2077,6 +2395,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" +[[package]] +name = "scoped_threadpool" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8" + [[package]] name = "scopeguard" version = "1.1.0" @@ -2155,6 +2479,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "servo-fontconfig" version = "0.5.1" @@ -2176,17 +2512,6 @@ dependencies = [ "pkg-config", ] -[[package]] -name = "sha-1" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - [[package]] name = "sha1" version = "0.10.5" @@ -2227,6 +2552,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14a5df39617d7c8558154693a1bb8157a4aab8179209540cc0b10e5dc24e0b18" + [[package]] name = "simple_room" version = "0.1.1" @@ -2235,6 +2566,7 @@ dependencies = [ "egui-wgpu", "egui-winit", "futures", + "image", "livekit", "parking_lot", "tokio", @@ -2307,6 +2639,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "spin" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dccf47db1b41fa1573ed27ccf5e08e3ca771cb994f776668c5ebda893b248fc" +dependencies = [ + "lock_api", +] + [[package]] name = "spirv" version = "0.2.0+1.5.4" @@ -2415,6 +2756,26 @@ dependencies = [ "once_cell", ] +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "tiff" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7449334f9ff2baf290d55d73983a7d6fa15e01198faef72af07e2a8db851e471" +dependencies = [ + "flate2", + "jpeg-decoder", + "weezl", +] + [[package]] name = "time" version = "0.3.17" @@ -2484,9 +2845,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.24.1" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae" +checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" dependencies = [ "autocfg", "bytes", @@ -2499,7 +2860,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] @@ -2524,10 +2885,21 @@ dependencies = [ ] [[package]] -name = "tokio-tungstenite" -version = "0.17.2" +name = "tokio-stream" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f714dd15bead90401d77e04243611caec13726c2408afd5b31901dfcdcb3b181" +checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54319c93411147bced34cb5609a80e0a8e44c5999c93903a81cd866630ec0bfd" dependencies = [ "futures-util", "log", @@ -2537,6 +2909,20 @@ dependencies = [ "tungstenite", ] +[[package]] +name = "tokio-util" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + [[package]] name = "toml" version = "0.5.10" @@ -2546,6 +2932,12 @@ dependencies = [ "serde", ] +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + [[package]] name = "tracing" version = "0.1.37" @@ -2604,6 +2996,12 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + [[package]] name = "ttf-parser" version = "0.18.0" @@ -2612,11 +3010,11 @@ checksum = "dbf554b6e535f9a160b2ed4ea83f99000f21cbc0a693df26e258eaf2c226a151" [[package]] name = "tungstenite" -version = "0.17.3" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e27992fd6a8c29ee7eef28fc78349aa244134e10ad447ce3b9f0ac0ed0fa4ce0" +checksum = "30ee6ab729cd4cf0fd55218530c4522ed30b7b6081752839b68fcec8d0960788" dependencies = [ - "base64", + "base64 0.13.1", "byteorder", "bytes", "http", @@ -2624,7 +3022,7 @@ dependencies = [ "log", "native-tls", "rand", - "sha-1", + "sha1", "thiserror", "url", "utf-8", @@ -2730,6 +3128,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -2738,9 +3146,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -2748,9 +3156,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" dependencies = [ "bumpalo", "log", @@ -2763,9 +3171,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" +checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" dependencies = [ "cfg-if", "js-sys", @@ -2775,9 +3183,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2785,9 +3193,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2", "quote", @@ -2798,9 +3206,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" [[package]] name = "wayland-client" @@ -2877,9 +3285,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.60" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" dependencies = [ "js-sys", "wasm-bindgen", @@ -2907,17 +3315,23 @@ dependencies = [ name = "webrtc-sys" version = "0.1.1" dependencies = [ - "curl", "cxx", "cxx-build", "flate2", "glob", "log", "regex", + "reqwest", "tar", "zip", ] +[[package]] +name = "weezl" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9193164d4de03a926d909d3bc7c30543cecb35400c02114792c2cae20d5e2dbb" + [[package]] name = "wgpu" version = "0.14.2" @@ -3070,12 +3484,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04662ed0e3e5630dfa9b26e4cb823b817f1a9addda855d973a9458c236556244" dependencies = [ "windows_aarch64_gnullvm", - "windows_aarch64_msvc 0.42.0", - "windows_i686_gnu 0.42.0", - "windows_i686_msvc 0.42.0", - "windows_x86_64_gnu 0.42.0", + "windows_aarch64_msvc 0.42.1", + "windows_i686_gnu 0.42.1", + "windows_i686_msvc 0.42.1", + "windows_x86_64_gnu 0.42.1", "windows_x86_64_gnullvm", - "windows_x86_64_msvc 0.42.0", + "windows_x86_64_msvc 0.42.1", ] [[package]] @@ -3098,19 +3512,43 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ "windows_aarch64_gnullvm", - "windows_aarch64_msvc 0.42.0", - "windows_i686_gnu 0.42.0", - "windows_i686_msvc 0.42.0", - "windows_x86_64_gnu 0.42.0", + "windows_aarch64_msvc 0.42.1", + "windows_i686_gnu 0.42.1", + "windows_i686_msvc 0.42.1", + "windows_x86_64_gnu 0.42.1", "windows_x86_64_gnullvm", - "windows_x86_64_msvc 0.42.0", + "windows_x86_64_msvc 0.42.1", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc 0.42.1", + "windows_i686_gnu 0.42.1", + "windows_i686_msvc 0.42.1", + "windows_x86_64_gnu 0.42.1", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc 0.42.1", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" +checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" [[package]] name = "windows_aarch64_msvc" @@ -3120,9 +3558,9 @@ checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" [[package]] name = "windows_aarch64_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" +checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" [[package]] name = "windows_i686_gnu" @@ -3132,9 +3570,9 @@ checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" [[package]] name = "windows_i686_gnu" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" +checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" [[package]] name = "windows_i686_msvc" @@ -3144,9 +3582,9 @@ checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" [[package]] name = "windows_i686_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" +checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" [[package]] name = "windows_x86_64_gnu" @@ -3156,15 +3594,15 @@ checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" [[package]] name = "windows_x86_64_gnu" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" +checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" +checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" [[package]] name = "windows_x86_64_msvc" @@ -3174,9 +3612,9 @@ checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" [[package]] name = "windows_x86_64_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" +checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" [[package]] name = "winit" @@ -3211,6 +3649,15 @@ dependencies = [ "x11-dl", ] +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] + [[package]] name = "wio" version = "0.2.2" @@ -3325,3 +3772,12 @@ dependencies = [ "cc", "libc", ] + +[[package]] +name = "zune-inflate" +version = "0.2.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a01728b79fb9b7e28a8c11f715e1cd8dc2cda7416a007d66cac55cebb3a8ac6b" +dependencies = [ + "simd-adler32", +] diff --git a/examples/simple_room/Cargo.toml b/examples/simple_room/Cargo.toml index 50bc7eb..1c9ccdd 100644 --- a/examples/simple_room/Cargo.toml +++ b/examples/simple_room/Cargo.toml @@ -16,3 +16,5 @@ parking_lot = "0.12.1" egui = "0.20.1" egui-wgpu = { version = "0.20.0", features = ["winit"] } egui-winit = "0.20.1" + +image = "0.24.5" diff --git a/examples/simple_room/src/app.rs b/examples/simple_room/src/app.rs index 0c62e73..ab80c0f 100644 --- a/examples/simple_room/src/app.rs +++ b/examples/simple_room/src/app.rs @@ -1,9 +1,16 @@ use crate::events::UiCmd; +use crate::logo_track::LogoTrack; use crate::video_renderer::VideoRenderer; use crate::{events::AsyncCmd, video_grid::VideoGrid}; use egui::{Rounding, Stroke}; use egui_wgpu::WgpuConfiguration; +use image::ImageFormat; +use livekit::options::{TrackPublishOptions, VideoCaptureOptions}; use livekit::prelude::*; +use livekit::webrtc::native::yuv_helper; +use livekit::webrtc::video_frame::native::I420BufferExt; +use livekit::webrtc::video_frame::{I420Buffer, VideoFrame, VideoRotation}; +use livekit::webrtc::video_source::native::NativeVideoSource; use livekit::SimulateScenario; use parking_lot::Mutex; use std::collections::HashMap; @@ -11,6 +18,7 @@ use std::sync::{ atomic::{AtomicBool, Ordering}, Arc, }; +use std::time::Duration; use tokio::sync::{mpsc, oneshot}; // Useful default constants for developing @@ -25,16 +33,22 @@ use winit::{ window::{WindowBuilder, WindowId}, }; +struct Session { + room: Room, + logo_track: LogoTrack, + close_tx: oneshot::Sender<()>, + handle: tokio::task::JoinHandle<()>, +} + struct AppState { - room: Mutex>, - close_tx: Mutex>>, + session: Mutex>, connecting: AtomicBool, } struct App { state: Arc, - video_renderers: HashMap<(ParticipantSid, TrackSid), VideoRenderer>, + egui_context: egui::Context, egui_state: egui_winit::State, egui_painter: egui_wgpu::winit::Painter, @@ -69,8 +83,7 @@ pub fn run(rt: tokio::runtime::Runtime) { let (ui_cmd_tx, ui_cmd_rx) = mpsc::unbounded_channel::(); let state = Arc::new(AppState { - room: Default::default(), - close_tx: Default::default(), + session: Default::default(), connecting: AtomicBool::new(false), }); @@ -94,45 +107,53 @@ pub fn run(rt: tokio::runtime::Runtime) { while let Some(event) = async_cmd_rx.recv().await { match event { AsyncCmd::RoomConnect { url, token } => { - if let Some(close_tx) = state.close_tx.lock().take() { - let _ = state.room.lock().take().unwrap().close().await; - let _ = close_tx.send(()); - } - state.connecting.store(true, Ordering::SeqCst); let res = Room::connect(&url, &token).await; - match res { - Ok((room, room_events)) => { - let (close_tx, close_rx) = oneshot::channel(); - state.room.lock().replace(room); - state.close_tx.lock().replace(close_tx); + if let Ok((room, room_events)) = res { + let (close_tx, close_rx) = oneshot::channel(); + let logo_track = LogoTrack::new(room.session()); + let handle = tokio::spawn(room_task( + state.clone(), + room_events, + close_rx, + ui_cmd_tx.clone(), + )); - tokio::spawn(room_task( - state.clone(), - room_events, - close_rx, - ui_cmd_tx.clone(), - )); + *state.session.lock() = Some(Session { + room, + logo_track, + close_tx, + handle, + }); - let _ = ui_cmd_tx.send(UiCmd::ConnectResult { result: Ok(()) }); - } - Err(err) => { - let _ = ui_cmd_tx.send(UiCmd::ConnectResult { result: Err(err) }); - } + let _ = ui_cmd_tx.send(UiCmd::ConnectResult { result: Ok(()) }); + } else if let Err(err) = res { + let _ = ui_cmd_tx.send(UiCmd::ConnectResult { result: Err(err) }); } state.connecting.store(false, Ordering::SeqCst); } AsyncCmd::RoomDisconnect => { - if let Some(close_tx) = state.close_tx.lock().take() { - let _ = state.room.lock().take().unwrap().close().await; - let _ = close_tx.send(()); + if let Some(session) = state.session.lock().take() { + let _ = session.room.close().await; + let _ = session.close_tx.send(()); + let _ = session.handle.await; } } AsyncCmd::SimulateScenario { scenario } => { - if let Some(room) = state.room.lock().as_ref() { - let _ = room.session().simulate_scenario(scenario).await; + if let Some(session) = state.session.lock().as_ref() { + let _ = session.room.session().simulate_scenario(scenario).await; + } + } + AsyncCmd::ToggleLogo => { + if let Some(session) = state.session.lock().as_mut() { + let logo_track = &mut session.logo_track; + if !logo_track.is_published() { + logo_track.publish().await.unwrap(); + } else { + logo_track.unpublish().await.unwrap(); + } } } } @@ -140,7 +161,7 @@ pub fn run(rt: tokio::runtime::Runtime) { }); tokio::task::block_in_place(move || loop { - // UI/Main Thread + // ui/main thread event_loop.run(move |event, _, control_flow| { app.update(event, control_flow); }); @@ -183,7 +204,7 @@ impl App { track, participant, .. } => { match track.clone() { - RemoteTrackHandle::Video(video_track) => { + RemoteTrack::Video(video_track) => { // Create a new VideoRenderer let video_renderer = VideoRenderer::new( self.egui_painter.render_state().clone().unwrap(), @@ -192,7 +213,7 @@ impl App { self.video_renderers .insert((participant.sid(), track.sid()), video_renderer); } - RemoteTrackHandle::Audio(_) => { + RemoteTrack::Audio(_) => { // The demo doesn't support Audio rendering at the moment. } }; @@ -300,6 +321,12 @@ impl App { }); } }); + + ui.menu_button("Publish", |ui| { + if ui.button("CustomTrack - LK Logo").clicked() { + let _ = self.cmd_tx.send(AsyncCmd::ToggleLogo); + } + }); }); }); @@ -321,9 +348,9 @@ impl App { ui.horizontal(|ui| { let connecting = self.state.connecting.load(Ordering::SeqCst); + let session = self.state.session.lock(); - let room = self.state.room.lock(); - ui.add_enabled_ui(!connecting && room.is_none(), |ui| { + ui.add_enabled_ui(!connecting && session.is_none(), |ui| { if ui.button("Connect").clicked() { self.connection_failure = None; let _ = self.cmd_tx.send(AsyncCmd::RoomConnect { @@ -337,7 +364,7 @@ impl App { ui.spinner(); } - if room.is_some() { + if session.is_some() { if ui.button("Disconnect").clicked() { let _ = self.cmd_tx.send(AsyncCmd::RoomDisconnect); } @@ -352,17 +379,16 @@ impl App { { // Room Info - let room = self.state.room.lock(); - if let Some(room) = room.as_ref() { - ui.label(format!("Name: {}", room.session().name())); - ui.label(format!("SID: {}", room.session().sid())); + if let Some(session) = self.state.session.lock().as_ref() { + ui.label(format!("Name: {}", session.room.session().name())); + ui.label(format!("SID: {}", session.room.session().sid())); ui.label(format!( "ConnectionState: {:?}", - room.session().connection_state() + session.room.session().connection_state() )); ui.label(format!( "ParticipantCount: {:?}", - room.session().participants().len() + 1 + session.room.session().participants().len() + 1 )); } } @@ -408,12 +434,15 @@ impl App { ); } - let name = self.state.room.lock().as_ref().and_then(|room| { - room.session() - .participants() - .get(participant_sid) - .map(|p| p.name()) - }); + let name = + self.state.session.lock().as_ref().and_then(|session| { + session + .room + .session() + .participants() + .get(participant_sid) + .map(|p| p.name()) + }); if let Some(name) = name { ui.painter().text( diff --git a/examples/simple_room/src/events.rs b/examples/simple_room/src/events.rs index 64239fc..46bd655 100644 --- a/examples/simple_room/src/events.rs +++ b/examples/simple_room/src/events.rs @@ -6,6 +6,7 @@ pub enum AsyncCmd { RoomConnect { url: String, token: String }, RoomDisconnect, SimulateScenario { scenario: SimulateScenario }, + ToggleLogo, // Unpublish/Publish a logo track } #[derive(Debug)] diff --git a/examples/simple_room/src/logo_track.rs b/examples/simple_room/src/logo_track.rs new file mode 100644 index 0000000..6dd802b --- /dev/null +++ b/examples/simple_room/src/logo_track.rs @@ -0,0 +1,206 @@ +use image::ImageFormat; +use image::RgbaImage; +use livekit::options::{TrackPublishOptions, VideoCaptureOptions}; +use livekit::prelude::*; +use livekit::webrtc::{ + native::yuv_helper, + video_frame::native::I420BufferExt, + video_frame::{I420Buffer, VideoFrame, VideoRotation}, + video_source::native::NativeVideoSource, +}; +use parking_lot::Mutex; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::oneshot; +use tokio::task::JoinHandle; + +// The logo must not be bigger than the framebuffer +const PIXEL_SIZE: usize = 4; +const FRAME_RATE: u64 = 30; +const MOVE_SPEED: i32 = 16; +const FB_WIDTH: usize = 1280; +const FB_HEIGHT: usize = 720; + +#[derive(Clone)] +struct FrameData { + image: Arc, + framebuffer: Arc>>, + video_frame: Arc>>, + pos: (u32, u32), + direction: (i32, i32), +} + +struct TrackHandle { + close_tx: oneshot::Sender<()>, + track: LocalVideoTrack, + task: JoinHandle<()>, +} + +pub struct LogoTrack { + rtc_source: NativeVideoSource, + session: RoomSession, + handle: Option, +} + +impl LogoTrack { + pub fn new(session: RoomSession) -> Self { + Self { + rtc_source: NativeVideoSource::default(), + session, + handle: None, + } + } + + pub fn is_published(&self) -> bool { + self.handle.is_some() + } + + pub async fn publish(&mut self) -> Result<(), RoomError> { + self.unpublish().await; + + let (close_tx, close_rx) = oneshot::channel(); + let track = LocalVideoTrack::create_video_track( + "livekit_logo", + VideoCaptureOptions::default(), + self.rtc_source.clone(), + ); + + let task = tokio::spawn(Self::track_task(close_rx, self.rtc_source.clone())); + + self.session + .local_participant() + .publish_track( + LocalTrack::Video(track.clone()), + TrackPublishOptions { + source: TrackSource::Camera, + ..Default::default() + }, + ) + .await?; + + let handle = TrackHandle { + close_tx, + task, + track, + }; + + self.handle = Some(handle); + Ok(()) + } + + pub async fn unpublish(&mut self) -> Result<(), RoomError> { + if let Some(handle) = self.handle.take() { + let _ = handle.close_tx.send(()); + let _ = handle.task.await; + + self.session + .local_participant() + .unpublish_track(handle.track.sid(), true) + .await?; + } + Ok(()) + } + + async fn track_task(mut close_rx: oneshot::Receiver<()>, rtc_source: NativeVideoSource) { + let mut interval = tokio::time::interval(Duration::from_millis(1000 / FRAME_RATE)); + + let image = tokio::task::spawn_blocking(|| { + image::load_from_memory_with_format(include_bytes!("moving-logo.png"), ImageFormat::Png) + .unwrap() + .to_rgba8() + }) + .await + .unwrap(); + + let mut data = FrameData { + image: Arc::new(image), + framebuffer: Arc::new(Mutex::new(vec![0u8; (FB_WIDTH * FB_HEIGHT * 4) as usize])), + video_frame: Arc::new(Mutex::new(VideoFrame { + rotation: VideoRotation::VideoRotation0, + buffer: I420Buffer::new(FB_WIDTH as u32, FB_HEIGHT as u32), + timestamp: 0, + })), + pos: (0, 0), + direction: (1, 1), + }; + + loop { + tokio::select! { + _ = &mut close_rx => { + break; + } + _ = interval.tick() => {} + } + + data.pos.0 = (data.pos.0 as i32 + data.direction.0 * MOVE_SPEED) as u32; + data.pos.1 = (data.pos.1 as i32 + data.direction.1 * MOVE_SPEED) as u32; + + if data.pos.0 >= (FB_WIDTH - data.image.width() as usize) as u32 { + data.direction.0 = -1; + } else if data.pos.0 <= 0 { + data.direction.0 = 1; + } + + if data.pos.1 >= (FB_HEIGHT - data.image.height() as usize) as u32 { + data.direction.1 = -1; + } else if data.pos.1 <= 0 { + data.direction.1 = 1; + } + + tokio::task::spawn_blocking({ + let data = data.clone(); + let source = rtc_source.clone(); + move || { + let image = data.image.as_raw(); + let mut framebuffer = data.framebuffer.lock(); + let mut video_frame = data.video_frame.lock(); + let i420_buffer = &mut video_frame.buffer; + + let stride_y = i420_buffer.stride_y(); + let stride_u = i420_buffer.stride_u(); + let stride_v = i420_buffer.stride_v(); + let (data_y, data_u, data_v) = i420_buffer.data_mut(); + + framebuffer.fill(0); + for i in 0..data.image.height() as usize { + let x = data.pos.0 as usize; + let y = data.pos.1 as usize; + let frame_width = data.image.width() as usize; + let logo_stride = frame_width * PIXEL_SIZE; + let row_start = (x + ((i + y) * FB_WIDTH)) * PIXEL_SIZE; + let row_end = row_start + logo_stride; + + framebuffer[row_start..row_end].copy_from_slice( + &image[i * logo_stride..i * logo_stride + logo_stride], + ); + } + + yuv_helper::abgr_to_i420( + &framebuffer, + (FB_WIDTH * PIXEL_SIZE) as i32, + data_y, + stride_y, + data_u, + stride_u, + data_v, + stride_v, + FB_WIDTH as i32, + FB_HEIGHT as i32, + ) + .unwrap(); + + source.capture_frame(&*video_frame); + } + }) + .await; + } + } +} + +impl Drop for LogoTrack { + fn drop(&mut self) { + if let Some(handle) = self.handle.take() { + let _ = handle.close_tx.send(()); + } + } +} diff --git a/examples/simple_room/src/main.rs b/examples/simple_room/src/main.rs index 351ba58..1e87b11 100644 --- a/examples/simple_room/src/main.rs +++ b/examples/simple_room/src/main.rs @@ -1,5 +1,6 @@ mod app; mod events; +mod logo_track; mod video_grid; mod video_renderer; diff --git a/examples/simple_room/src/moving-logo.png b/examples/simple_room/src/moving-logo.png new file mode 100644 index 0000000..5c47cc1 Binary files /dev/null and b/examples/simple_room/src/moving-logo.png differ diff --git a/examples/simple_room/src/video_renderer.rs b/examples/simple_room/src/video_renderer.rs index 587423e..5e24b9f 100644 --- a/examples/simple_room/src/video_renderer.rs +++ b/examples/simple_room/src/video_renderer.rs @@ -1,18 +1,17 @@ -use livekit::prelude::*; -use livekit::webrtc::video_frame_buffer::*; -use livekit::webrtc::yuv_helper; -use std::convert::TryInto; +use futures::StreamExt; +use livekit::webrtc::native::yuv_helper; +use livekit::webrtc::prelude::*; +use livekit::webrtc::video_stream::native::NativeVideoStream; use std::num::NonZeroU32; use std::{ ops::DerefMut, sync::{Arc, Mutex}, }; use tracing::debug_span; -use tracing::{error, warn}; pub struct VideoRenderer { internal: Arc>, - rtc_track: Arc, + rtc_track: RtcVideoTrack, } struct RendererInternal { @@ -86,7 +85,7 @@ impl RendererInternal { } impl VideoRenderer { - pub fn new(render_state: egui_wgpu::RenderState, rtc_track: Arc) -> Self { + pub fn new(render_state: egui_wgpu::RenderState, rtc_track: RtcVideoTrack) -> Self { let internal = Arc::new(Mutex::new(RendererInternal { render_state, width: 0, @@ -97,62 +96,73 @@ impl VideoRenderer { egui_texture: None, })); - rtc_track.on_frame({ + let mut video_sink = NativeVideoStream::new(rtc_track.clone()); + + tokio::spawn({ let internal = internal.clone(); + async move { + while let Some(frame) = video_sink.next().await { + let internal = internal.clone(); + // Process the frame + let _ = tokio::task::spawn_blocking(move || { + let span = debug_span!("texture_upload"); + let _enter = span.enter(); - Box::new(move |_frame, buffer| { - let span = debug_span!("texture_upload"); - let _enter = span.enter(); + let mut internal = internal.lock().unwrap(); + let buffer = frame.buffer.to_i420(); - let mut internal = internal.lock().unwrap(); - let buffer = buffer.to_i420(); + let width: u32 = buffer.width().try_into().unwrap(); + let height: u32 = buffer.height().try_into().unwrap(); - let width: u32 = buffer.width().try_into().unwrap(); - let height: u32 = buffer.height().try_into().unwrap(); + internal.ensure_texture_size(width, height); - internal.ensure_texture_size(width, height); + let rgba_ptr = internal.rgba_data.deref_mut(); + let rgba_stride = buffer.width() * 4; - let rgba_ptr = internal.rgba_data.deref_mut(); - let rgba_stride = buffer.width() * 4; + let (data_y, data_u, data_v) = buffer.data(); - yuv_helper::i420_to_abgr( - buffer.data_y(), - buffer.stride_y(), - buffer.data_u(), - buffer.stride_u(), - buffer.data_v(), - buffer.stride_v(), - rgba_ptr, - rgba_stride, - buffer.width(), - buffer.height(), - ); + yuv_helper::i420_to_abgr( + data_y, + buffer.stride_y(), + data_u, + buffer.stride_u(), + data_v, + buffer.stride_v(), + rgba_ptr, + rgba_stride, + buffer.width(), + buffer.height(), + ) + .unwrap(); - let copy_desc = wgpu::ImageCopyTexture { - texture: internal.texture.as_ref().unwrap(), - mip_level: 0, - origin: wgpu::Origin3d::default(), - aspect: wgpu::TextureAspect::default(), - }; + let copy_desc = wgpu::ImageCopyTexture { + texture: internal.texture.as_ref().unwrap(), + mip_level: 0, + origin: wgpu::Origin3d::default(), + aspect: wgpu::TextureAspect::default(), + }; - let copy_layout = wgpu::ImageDataLayout { - bytes_per_row: Some(NonZeroU32::new(width * 4).unwrap()), - ..Default::default() - }; + let copy_layout = wgpu::ImageDataLayout { + bytes_per_row: Some(NonZeroU32::new(width * 4).unwrap()), + ..Default::default() + }; - let copy_size = wgpu::Extent3d { - width, - height, - ..Default::default() - }; + let copy_size = wgpu::Extent3d { + width, + height, + ..Default::default() + }; - internal.render_state.queue.write_texture( - copy_desc, - &internal.rgba_data, - copy_layout, - copy_size, - ); - }) + internal.render_state.queue.write_texture( + copy_desc, + &internal.rgba_data, + copy_layout, + copy_size, + ); + }) + .await; + } + } }); Self { @@ -165,9 +175,3 @@ impl VideoRenderer { self.internal.lock().unwrap().egui_texture.clone() } } - -impl Drop for VideoRenderer { - fn drop(&mut self) { - self.rtc_track.on_frame(Box::new(|_, _| {})); - } -} diff --git a/livekit-ffi/Cargo.toml b/livekit-ffi/Cargo.toml index 3a1ee1d..45f7d02 100644 --- a/livekit-ffi/Cargo.toml +++ b/livekit-ffi/Cargo.toml @@ -14,10 +14,10 @@ prost = "0.11.0" prost-types = "0.11.1" lazy_static = "1.4.0" thiserror = "1.0.38" +futures-util = "0.3.23" [build-dependencies] prost-build = { version = "0.11.1" } - [lib] crate-type = ["cdylib", "staticlib"] diff --git a/livekit-ffi/protocol/ffi.proto b/livekit-ffi/protocol/ffi.proto index c8c4381..eb1a086 100644 --- a/livekit-ffi/protocol/ffi.proto +++ b/livekit-ffi/protocol/ffi.proto @@ -279,19 +279,12 @@ message TrackEvent { message FrameReceived { VideoFrameInfo frame = 1; - VideoFrameBufferInfo frame_buffer = 2; + VideoFrameBufferInfo buffer = 2; } message VideoFrameInfo { - int32 width = 1; - int32 height = 2; - uint32 size = 3; - uint32 id = 4; // uint16 - int64 timestamp_us = 5; - int64 ntp_time_ms = 6; - uint32 transport_frame_id = 7; - uint32 timestamp = 8; - VideoRotation rotation = 9; + int64 timestamp = 1; + VideoRotation rotation = 2; } message VideoFrameBufferInfo { @@ -312,11 +305,13 @@ message PlanarYuvBufferInfo { int32 stride_y = 3; int32 stride_u = 4; int32 stride_v = 5; + int32 stride_a = 6; // *const u8 or *const u16 - uint64 data_y_ptr = 6; - uint64 data_u_ptr = 7; - uint64 data_v_ptr = 8; + uint64 data_y_ptr = 7; + uint64 data_u_ptr = 8; + uint64 data_v_ptr = 9; + uint64 data_a_ptr = 10; // nullptr = no alpha } message BiplanarYuvBufferInfo { @@ -341,6 +336,7 @@ enum VideoFrameBufferType { I444 = 4; I010 = 5; NV12 = 6; + WEBGL = 7; } /// Participant Events diff --git a/livekit-ffi/src/server/conversion.rs b/livekit-ffi/src/server/conversion/mod.rs similarity index 53% rename from livekit-ffi/src/server/conversion.rs rename to livekit-ffi/src/server/conversion/mod.rs index 7257024..0a989ae 100644 --- a/livekit-ffi/src/server/conversion.rs +++ b/livekit-ffi/src/server/conversion/mod.rs @@ -1,12 +1,12 @@ use crate::{proto, server::FFIHandleId}; -use livekit::{ - prelude::*, - webrtc::video_frame_buffer::{ - BiplanarYuv8Buffer, BiplanarYuvBuffer, I010Buffer, I420ABuffer, I420Buffer, I422Buffer, - I444Buffer, NV12Buffer, PlanarYuv16BBuffer, PlanarYuv8Buffer, PlanarYuvBuffer, - }, -}; -use std::sync::Arc; +use livekit::prelude::*; +use livekit::webrtc::prelude::*; +use std::any::Any; + +pub mod participant; +pub mod publication; +pub mod room; +pub mod video_frame; impl From for proto::FfiHandleId { fn from(id: FFIHandleId) -> Self { @@ -30,8 +30,8 @@ macro_rules! impl_participant_into { }; } -impl_participant_into!(&Arc); -impl_participant_into!(&Arc); +impl_participant_into!(&LocalParticipant); +impl_participant_into!(&RemoteParticipant); impl_participant_into!(&Participant); impl From for proto::TrackSource { @@ -92,14 +92,13 @@ impl_track_into!(&LocalAudioTrack); impl_track_into!(&LocalVideoTrack); impl_track_into!(&RemoteAudioTrack); impl_track_into!(&RemoteVideoTrack); -impl_track_into!(&TrackHandle); -impl_track_into!(&LocalTrackHandle); -impl_track_into!(&RemoteTrackHandle); +impl_track_into!(&Track); +impl_track_into!(&LocalTrack); +impl_track_into!(&RemoteTrack); impl From for proto::TrackKind { fn from(kind: TrackKind) -> Self { match kind { - TrackKind::Unknown => proto::TrackKind::KindUnknown, TrackKind::Audio => proto::TrackKind::KindAudio, TrackKind::Video => proto::TrackKind::KindVideo, } @@ -109,7 +108,6 @@ impl From for proto::TrackKind { impl From for proto::StreamState { fn from(state: StreamState) -> Self { match state { - StreamState::Unknown => Self::StateUnknown, StreamState::Active => Self::StateActive, StreamState::Paused => Self::StatePaused, } @@ -182,136 +180,6 @@ impl proto::RoomEvent { } } -impl From for proto::VideoRotation { - fn from(rotation: VideoRotation) -> proto::VideoRotation { - match rotation { - VideoRotation::VideoRotation0 => Self::VideoRotation0, - VideoRotation::VideoRotation90 => Self::VideoRotation90, - VideoRotation::VideoRotation180 => Self::VideoRotation180, - VideoRotation::VideoRotation270 => Self::VideoRotation270, - } - } -} - -impl From for proto::VideoFrameInfo { - fn from(frame: VideoFrame) -> Self { - Self { - width: frame.width(), - height: frame.height(), - size: frame.size(), - id: frame.id() as u32, - timestamp_us: frame.timestamp_us(), - ntp_time_ms: frame.ntp_time_ms(), - transport_frame_id: frame.transport_frame_id(), - timestamp: frame.timestamp(), - rotation: proto::VideoRotation::from(frame.rotation()).into(), - } - } -} - -impl From for proto::VideoFrameBufferType { - fn from(buffer_type: VideoFrameBufferType) -> Self { - match buffer_type { - VideoFrameBufferType::Native => Self::Native, - VideoFrameBufferType::I420 => Self::I420, - VideoFrameBufferType::I420A => Self::I420a, - VideoFrameBufferType::I422 => Self::I422, - VideoFrameBufferType::I444 => Self::I444, - VideoFrameBufferType::I010 => Self::I010, - VideoFrameBufferType::NV12 => Self::Nv12, - } - } -} - -macro_rules! impl_yuv_into { - ($b:ty) => { - impl From<$b> for proto::PlanarYuvBufferInfo { - fn from(buffer: $b) -> Self { - Self { - chroma_width: buffer.chroma_width(), - chroma_height: buffer.chroma_height(), - stride_y: buffer.stride_y(), - stride_u: buffer.stride_u(), - stride_v: buffer.stride_v(), - data_y_ptr: buffer.data_y().as_ptr() as u64, - data_u_ptr: buffer.data_u().as_ptr() as u64, - data_v_ptr: buffer.data_v().as_ptr() as u64, - } - } - } - }; -} - -impl_yuv_into!(&I420Buffer); -impl_yuv_into!(&I420ABuffer); -impl_yuv_into!(&I422Buffer); -impl_yuv_into!(&I444Buffer); -impl_yuv_into!(&I010Buffer); - -macro_rules! impl_biyuv_into { - ($b:ty) => { - impl From<$b> for proto::BiplanarYuvBufferInfo { - fn from(buffer: $b) -> Self { - Self { - chroma_width: buffer.chroma_width(), - chroma_height: buffer.chroma_height(), - stride_y: buffer.stride_y(), - stride_uv: buffer.stride_uv(), - data_y_ptr: buffer.data_y().as_ptr() as u64, - data_uv_ptr: buffer.data_uv().as_ptr() as u64, - } - } - } - }; -} - -impl_biyuv_into!(&NV12Buffer); - -impl proto::VideoFrameBufferInfo { - pub fn from(handle_id: FFIHandleId, buffer: &VideoFrameBuffer) -> Self { - Self { - handle: Some(handle_id.into()), - buffer_type: proto::VideoFrameBufferType::from(buffer.buffer_type()).into(), - width: buffer.width(), - height: buffer.height(), - buffer: Some(match &buffer { - VideoFrameBuffer::Native(_) => { - proto::video_frame_buffer_info::Buffer::Native(proto::NativeBufferInfo {}) - } - VideoFrameBuffer::I420(i420) => { - proto::video_frame_buffer_info::Buffer::Yuv(i420.into()) - } - VideoFrameBuffer::I420A(i420a) => { - proto::video_frame_buffer_info::Buffer::Yuv(i420a.into()) - } - VideoFrameBuffer::I422(i422) => { - proto::video_frame_buffer_info::Buffer::Yuv(i422.into()) - } - VideoFrameBuffer::I444(i444) => { - proto::video_frame_buffer_info::Buffer::Yuv(i444.into()) - } - VideoFrameBuffer::I010(i010) => { - proto::video_frame_buffer_info::Buffer::Yuv(i010.into()) - } - VideoFrameBuffer::NV12(nv12) => { - proto::video_frame_buffer_info::Buffer::BiYuv(nv12.into()) - } - }), - } - } -} - -impl From for VideoFormatType { - fn from(format: proto::VideoFormatType) -> Self { - match format { - proto::VideoFormatType::FormatArgb => Self::ARGB, - proto::VideoFormatType::FormatBgra => Self::BGRA, - proto::VideoFormatType::FormatAbgr => Self::ABGR, - proto::VideoFormatType::FormatRgba => Self::RGBA, - } - } -} - impl From<&RoomSession> for proto::RoomInfo { fn from(session: &RoomSession) -> Self { Self { diff --git a/livekit-ffi/src/server/conversion/participant.rs b/livekit-ffi/src/server/conversion/participant.rs new file mode 100644 index 0000000..e69de29 diff --git a/livekit-ffi/src/server/conversion/publication.rs b/livekit-ffi/src/server/conversion/publication.rs new file mode 100644 index 0000000..e69de29 diff --git a/livekit-ffi/src/server/conversion/room.rs b/livekit-ffi/src/server/conversion/room.rs new file mode 100644 index 0000000..e69de29 diff --git a/livekit-ffi/src/server/conversion/video_frame.rs b/livekit-ffi/src/server/conversion/video_frame.rs new file mode 100644 index 0000000..0d03afa --- /dev/null +++ b/livekit-ffi/src/server/conversion/video_frame.rs @@ -0,0 +1,203 @@ +use crate::{proto, server::FFIHandleId}; +use livekit::webrtc::prelude::*; +use livekit::webrtc::video_frame; +use std::any::Any; + +macro_rules! impl_yuv_into { + (@fields, $buffer:ident, $data_y:ident, $data_u:ident, $data_v: ident) => { + Self { + chroma_width: $buffer.chroma_width(), + chroma_height: $buffer.chroma_height(), + stride_y: $buffer.stride_y(), + stride_u: $buffer.stride_u(), + stride_v: $buffer.stride_v(), + data_y_ptr: $data_y.as_ptr() as u64, + data_u_ptr: $data_u.as_ptr() as u64, + data_v_ptr: $data_v.as_ptr() as u64, + ..Default::default() + } + }; + ($buffer:ty, ALPHA) => { + impl From<$buffer> for proto::PlanarYuvBufferInfo { + fn from(buffer: $buffer) -> Self { + let (data_y, data_u, data_v, data_a) = buffer.data(); + let mut proto = impl_yuv_into!(@fields, buffer, data_y, data_u, data_v); + proto.stride_a = buffer.stride_a(); + proto.data_a_ptr = data_a.map(|data_a| data_a.as_ptr() as u64).unwrap_or(0); + proto + } + } + }; + ($buffer:ty) => { + impl From<$buffer> for proto::PlanarYuvBufferInfo { + fn from(buffer: $buffer) -> Self { + let (data_y, data_u, data_v) = buffer.data(); + impl_yuv_into!(@fields, buffer, data_y, data_u, data_v) + } + } + }; +} + +macro_rules! impl_biyuv_into { + ($b:ty) => { + impl From<$b> for proto::BiplanarYuvBufferInfo { + fn from(buffer: $b) -> Self { + let (data_y, data_uv) = buffer.data(); + Self { + chroma_width: buffer.chroma_width(), + chroma_height: buffer.chroma_height(), + stride_y: buffer.stride_y(), + stride_uv: buffer.stride_uv(), + data_y_ptr: data_y.as_ptr() as u64, + data_uv_ptr: data_uv.as_ptr() as u64, + } + } + } + }; +} + +impl_yuv_into!(&I420Buffer); +impl_yuv_into!(&I420ABuffer, ALPHA); +impl_yuv_into!(&I422Buffer); +impl_yuv_into!(&I444Buffer); +impl_yuv_into!(&I010Buffer); +impl_biyuv_into!(&NV12Buffer); + +impl proto::VideoFrameInfo { + pub fn from(frame: &VideoFrame) -> Self + where + T: VideoFrameBuffer, + { + Self { + timestamp: frame.timestamp, + rotation: proto::VideoRotation::from(frame.rotation).into(), + } + } +} + +impl proto::VideoFrameBufferInfo { + pub fn from(handle: FFIHandleId, buffer: &dyn VideoFrameBuffer) -> Self { + match &buffer.buffer_type() { + #[cfg(not(target_arch = "wasm32"))] + VideoFrameBufferType::Native => Self::from_native(handle, buffer.as_native().unwrap()), + VideoFrameBufferType::I420 => Self::from_i420(handle, buffer.as_i420().unwrap()), + VideoFrameBufferType::I420A => Self::from_i420a(handle, buffer.as_i420a().unwrap()), + VideoFrameBufferType::I422 => Self::from_i422(handle, buffer.as_i422().unwrap()), + VideoFrameBufferType::I444 => Self::from_i444(handle, buffer.as_i444().unwrap()), + VideoFrameBufferType::I010 => Self::from_i010(handle, buffer.as_i010().unwrap()), + VideoFrameBufferType::NV12 => Self::from_nv12(handle, buffer.as_nv12().unwrap()), + _ => panic!("unsupported buffer type on this platform"), + } + } + + #[cfg(not(target_arch = "wasm32"))] + pub fn from_native(handle_id: FFIHandleId, buffer: &video_frame::native::NativeBuffer) -> Self { + Self { + handle: Some(handle_id.into()), + buffer_type: proto::VideoFrameBufferType::Native.into(), + width: buffer.width(), + height: buffer.height(), + buffer: Some(proto::video_frame_buffer_info::Buffer::Native( + proto::NativeBufferInfo {}, + )), + } + } + + pub fn from_i420(handle_id: FFIHandleId, buffer: &I420Buffer) -> Self { + Self { + handle: Some(handle_id.into()), + buffer_type: proto::VideoFrameBufferType::I420.into(), + width: buffer.width(), + height: buffer.height(), + buffer: Some(proto::video_frame_buffer_info::Buffer::Yuv(buffer.into())), + } + } + + pub fn from_i420a(handle_id: FFIHandleId, buffer: &I420ABuffer) -> Self { + Self { + handle: Some(handle_id.into()), + buffer_type: proto::VideoFrameBufferType::I420a.into(), + width: buffer.width(), + height: buffer.height(), + buffer: Some(proto::video_frame_buffer_info::Buffer::Yuv(buffer.into())), + } + } + + pub fn from_i422(handle_id: FFIHandleId, buffer: &I422Buffer) -> Self { + Self { + handle: Some(handle_id.into()), + buffer_type: proto::VideoFrameBufferType::I422.into(), + width: buffer.width(), + height: buffer.height(), + buffer: Some(proto::video_frame_buffer_info::Buffer::Yuv(buffer.into())), + } + } + + pub fn from_i444(handle_id: FFIHandleId, buffer: &I444Buffer) -> Self { + Self { + handle: Some(handle_id.into()), + buffer_type: proto::VideoFrameBufferType::I444.into(), + width: buffer.width(), + height: buffer.height(), + buffer: Some(proto::video_frame_buffer_info::Buffer::Yuv(buffer.into())), + } + } + + pub fn from_i010(handle_id: FFIHandleId, buffer: &I010Buffer) -> Self { + Self { + handle: Some(handle_id.into()), + buffer_type: proto::VideoFrameBufferType::I010.into(), + width: buffer.width(), + height: buffer.height(), + buffer: Some(proto::video_frame_buffer_info::Buffer::Yuv(buffer.into())), + } + } + + pub fn from_nv12(handle_id: FFIHandleId, buffer: &NV12Buffer) -> Self { + Self { + handle: Some(handle_id.into()), + buffer_type: proto::VideoFrameBufferType::Nv12.into(), + width: buffer.width(), + height: buffer.height(), + buffer: Some(proto::video_frame_buffer_info::Buffer::BiYuv(buffer.into())), + } + } +} + +impl From for VideoFormatType { + fn from(format: proto::VideoFormatType) -> Self { + match format { + proto::VideoFormatType::FormatArgb => Self::ARGB, + proto::VideoFormatType::FormatBgra => Self::BGRA, + proto::VideoFormatType::FormatAbgr => Self::ABGR, + proto::VideoFormatType::FormatRgba => Self::RGBA, + } + } +} + +impl From for proto::VideoRotation { + fn from(rotation: VideoRotation) -> proto::VideoRotation { + match rotation { + VideoRotation::VideoRotation0 => Self::VideoRotation0, + VideoRotation::VideoRotation90 => Self::VideoRotation90, + VideoRotation::VideoRotation180 => Self::VideoRotation180, + VideoRotation::VideoRotation270 => Self::VideoRotation270, + } + } +} + +impl From for proto::VideoFrameBufferType { + fn from(buffer_type: VideoFrameBufferType) -> Self { + match buffer_type { + VideoFrameBufferType::Native => Self::Native, + VideoFrameBufferType::I420 => Self::I420, + VideoFrameBufferType::I420A => Self::I420a, + VideoFrameBufferType::I422 => Self::I422, + VideoFrameBufferType::I444 => Self::I444, + VideoFrameBufferType::I010 => Self::I010, + VideoFrameBufferType::NV12 => Self::Nv12, + VideoFrameBufferType::WebGl => Self::Webgl, + _ => panic!("unsupported buffer type on FFI server"), + } + } +} diff --git a/livekit-ffi/src/server/mod.rs b/livekit-ffi/src/server/mod.rs index af9ebbc..3d5f8f3 100644 --- a/livekit-ffi/src/server/mod.rs +++ b/livekit-ffi/src/server/mod.rs @@ -1,6 +1,7 @@ use crate::proto; use lazy_static::lazy_static; use livekit::prelude::*; +use livekit::webrtc::video_frame::{native::VideoFrameBufferExt, BoxVideoFrame, VideoFrameBuffer}; use parking_lot::{Mutex, RwLock}; use prost::Message; use std::any::Any; @@ -166,11 +167,11 @@ impl FFIServer { let buffer = self.release_handle(to_i420.buffer.unwrap().id as FFIHandleId); if let Some(buffer) = buffer { - if let Ok(buffer) = buffer.downcast::() { + if let Ok(buffer) = buffer.downcast::>() { let handle_id = self.next_handle_id(); - let i420 = VideoFrameBuffer::I420(buffer.to_i420()); - buffer_info = Some(proto::VideoFrameBufferInfo::from(handle_id, &i420)); - self.insert_handle(handle_id, Box::new(i420)); + let buffer = buffer.to_i420(); + buffer_info = Some(proto::VideoFrameBufferInfo::from(handle_id, &buffer)); + self.insert_handle(handle_id, Box::new(buffer)); } } @@ -188,7 +189,7 @@ impl FFIServer { let buffer = ffi_owned.get(&(to_argb.buffer.unwrap().id as FFIHandleId)); if let Some(buffer) = buffer { - if let Some(buffer) = buffer.downcast_ref::() { + if let Some(buffer) = buffer.downcast_ref::>() { let dst_buf = unsafe { slice::from_raw_parts_mut( to_argb.dst_ptr as *mut u8, diff --git a/livekit-ffi/src/server/room.rs b/livekit-ffi/src/server/room.rs index eb773c3..e9fc490 100644 --- a/livekit-ffi/src/server/room.rs +++ b/livekit-ffi/src/server/room.rs @@ -1,6 +1,8 @@ -use crate::proto::{self}; +use crate::proto; use crate::server::FFIServer; +use futures_util::stream::StreamExt; use livekit::prelude::*; +use livekit::webrtc::video_stream::native::NativeVideoStream; use tokio::sync::{mpsc, oneshot}; pub async fn create_room( @@ -67,9 +69,9 @@ async fn room_task( publication: _, participant: _, } => { - if let RemoteTrackHandle::Video(video_track) = track { - let rtc_track = video_track.rtc_track(); - rtc_track.on_frame(on_video_frame(server, video_track.sid())); + if let RemoteTrack::Video(video_track) = track { + let video_stream = NativeVideoStream::new(video_track.rtc_track()); + tokio::spawn(video_frame_task(server, video_track.sid(), video_stream)); } } _ => {} @@ -87,19 +89,20 @@ async fn room_task( async fn participant_task(participant: Participant) { let mut participant_events = participant.register_observer(); while let Some(event) = participant_events.recv().await { - // TODO convert event to proto + // TODO(theomonnom): convert event to proto } } -fn on_video_frame(server: &'static FFIServer, track_sid: TrackSid) -> OnFrameHandler { - // TODO(theomonnom): Should I use VideoSinkInfo here? (It'll help to have a more verbose - // lifetime) - - Box::new(move |frame, buffer| { - // Frame received, create a new FFIHandle from the video buffer. +async fn video_frame_task( + server: &'static FFIServer, + track_sid: TrackSid, + mut stream: NativeVideoStream, +) { + while let Some(frame) = stream.next().await { let handle_id = server.next_handle_id(); - let proto_buffer = proto::VideoFrameBufferInfo::from(handle_id, &buffer); - server.insert_handle(handle_id, Box::new(buffer)); + let frame_info = proto::VideoFrameInfo::from(&frame); + let buffer_info = proto::VideoFrameBufferInfo::from(handle_id, &frame.buffer); + server.insert_handle(handle_id, Box::new(frame.buffer)); // Send the received frame to the FFI language. let _ = server.send_event( @@ -107,12 +110,12 @@ fn on_video_frame(server: &'static FFIServer, track_sid: TrackSid) -> OnFrameHan track_sid: track_sid.to_string(), message: Some(proto::track_event::Message::FrameReceived( proto::FrameReceived { - frame: Some(frame.into()), - frame_buffer: Some(proto_buffer), + frame: Some(frame_info), + buffer: Some(buffer_info), }, )), }), None, ); - }) + } } diff --git a/livekit-utils/Cargo.toml b/livekit-utils/Cargo.toml index 2a9a5d4..bb5ad4d 100644 --- a/livekit-utils/Cargo.toml +++ b/livekit-utils/Cargo.toml @@ -6,8 +6,7 @@ license = "Apache-2.0" description = "Shared utilities for livekit" repository = "https://github.com/livekit/client-sdk-rust" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] -parking_lot = "0.12.1" -tokio = { version = "1", features = ["full"] } +tokio = { version = "1.26.0", features = ["full"] } +futures-util = "0.3" +parking_lot = "0.12" diff --git a/livekit-utils/src/enum_dispatch.rs b/livekit-utils/src/enum_dispatch.rs index d24eb4d..46cdfd4 100644 --- a/livekit-utils/src/enum_dispatch.rs +++ b/livekit-utils/src/enum_dispatch.rs @@ -1,11 +1,9 @@ - -// TODO(theomonnom): Match the complete function signature like: -// - pub(crate) fn update_info(&self, info: ParticipantInfo) -> (); +// TODO(theomonnom): Async methods #[macro_export] macro_rules! enum_dispatch { // This arm is used to avoid nested loops with the arguments - // The arguments are transformed to $combined_args TokenTree - (@match $self:ident $fnc:ident $combined_args:tt [$($variant:ident),+]) => { + // The arguments are transformed to $combined_args tt + (@match [$($variant:ident),+]: $fnc:ident, $self:ident, $combined_args:tt) => { match $self { $( Self::$variant(inner) => inner.$fnc$combined_args, @@ -13,15 +11,17 @@ macro_rules! enum_dispatch { } }; - ($vis:vis$(,)? $fnc:ident, $self:ty, [$($arg:ident: $t:ty),*], $ret:ty, [$($variant:ident),+]) => { - $vis fn $fnc(self: $self, $($arg: $t),*) -> $ret { - enum_dispatch!(@match self $fnc ($($arg,)*) [$($variant),+]) + // Create the function and extract self fron the $args tt (little hack) + (@fnc [$($variant:ident),+]: $vis:vis fn $fnc:ident($self:ident: $sty:ty $(, $arg:ident: $t:ty)*) -> $ret:ty) => { + #[inline] + $vis fn $fnc($self: $sty, $($arg: $t),*) -> $ret { + enum_dispatch!(@match [$($variant),+]: $fnc, $self, ($($arg,)*)) } }; - ($variants:tt $(fnc!($vis:vis$(,)? $fnc:ident, $self:ty, $args:tt, $ret:ty);)+) => { + ($variants:tt; $($vis:vis fn $fnc:ident$args:tt -> $ret:ty;)+) => { $( - enum_dispatch!($vis, $fnc, $self, $args, $ret, $variants); - )* + enum_dispatch!(@fnc $variants: $vis fn $fnc$args -> $ret); + )+ }; } diff --git a/livekit-utils/src/observer.rs b/livekit-utils/src/observer.rs index 14c26b2..12974eb 100644 --- a/livekit-utils/src/observer.rs +++ b/livekit-utils/src/observer.rs @@ -1,14 +1,16 @@ -// Really basic implementation of the observer pattern using mpsc channels. -// Currently unbounded channels - +use futures_util::sink::Sink; +use futures_util::task::{Context, Poll}; +use parking_lot::Mutex; +use std::pin::Pin; +use std::sync::Arc; use tokio::sync::mpsc; -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct Dispatcher where T: Clone, { - senders: Vec>, + senders: Arc>>>, } impl Default for Dispatcher @@ -26,14 +28,43 @@ impl Dispatcher where T: Clone, { - pub fn register(&mut self) -> mpsc::UnboundedReceiver { + pub fn register(&self) -> mpsc::UnboundedReceiver { let (tx, rx) = mpsc::unbounded_channel(); - self.senders.push(tx); + self.senders.lock().push(tx); rx } - pub fn dispatch(&mut self, msg: &T) { + pub fn dispatch(&self, msg: &T) { self.senders + .lock() .retain(|sender| sender.send(msg.clone()).is_ok()); } + + pub fn clear(&self) { + self.senders.lock().clear(); + } +} + +impl Sink for Dispatcher +where + T: Clone, +{ + type Error = (); + + fn poll_ready(self: Pin<&mut Self>, _: &mut Context) -> Poll> { + Poll::Ready(Ok(())) + } + + fn start_send(self: Pin<&mut Self>, item: T) -> Result<(), Self::Error> { + self.dispatch(&item); + Ok(()) + } + + fn poll_flush(self: Pin<&mut Self>, _: &mut Context) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_close(self: Pin<&mut Self>, _: &mut Context) -> Poll> { + Poll::Ready(Ok(())) + } } diff --git a/livekit-webrtc/Cargo.toml b/livekit-webrtc/Cargo.toml index 3a34979..e81f5c8 100644 --- a/livekit-webrtc/Cargo.toml +++ b/livekit-webrtc/Cargo.toml @@ -8,12 +8,35 @@ description = "Livekit safe bindings to libwebrtc" repository = "https://github.com/livekit/client-sdk-rust" [dependencies] -webrtc-sys = { path = "../webrtc-sys", version = "0.1.1" } livekit-utils = { path = "../livekit-utils", version = "0.1.1" } -tokio = { version = "1", features = ["full"] } -cxx = "1.0" log = "0.4" thiserror = "1.0" +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +webrtc-sys = { path = "../webrtc-sys", version = "0.1.1" } +futures = { version = "0.3" } +tokio = { version = "1.26.0", features = ["full"] } +cxx = "1.0" + +[target.'cfg(target_arch = "wasm32")'.dependencies] +wasm-bindgen = "0.2.84" +js-sys = "0.3" +wasm-bindgen-futures = "0.4.34" +web-sys = { version = "0.3.22", features=[ + "MessageEvent", + "RtcPeerConnection", + "RtcSignalingState", + "RtcSdpType", + "RtcSessionDescriptionInit", + "RtcPeerConnectionIceEvent", + "RtcIceCandidate", + "RtcDataChannel", + "RtcDataChannelEvent", + "RtcDataChannelState", + "EventTarget", + "WebGlRenderingContext", + "WebGlTexture", +] } + [dev-dependencies] env_logger = "0.9" diff --git a/livekit-webrtc/src/data_channel.rs b/livekit-webrtc/src/data_channel.rs index 5395b03..bcd6a90 100644 --- a/livekit-webrtc/src/data_channel.rs +++ b/livekit-webrtc/src/data_channel.rs @@ -1,152 +1,9 @@ -use std::error::Error; -use std::fmt::{Debug, Display, Formatter}; -use std::sync::Mutex; +use crate::{imp::data_channel as dc_imp, rtp_parameters::Priority}; +use std::{fmt::Debug, str::Utf8Error}; +use thiserror::Error; -use cxx::UniquePtr; -use log::trace; - -pub use sys_dc::ffi::{DataState, Priority}; -use webrtc_sys::data_channel as sys_dc; - -pub struct DataChannel { - cxx_handle: UniquePtr, - observer: Box, - - // Keep alive for C++ - native_observer: UniquePtr, -} - -impl Debug for DataChannel { - fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - f.debug_struct("DataChannel") - .field("label", &self.label()) - .finish() - } -} - -#[derive(Debug)] -pub struct DataSendError; - -impl Display for DataSendError { - fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - write!(f, "failed to send data to the DataChannel") - } -} - -impl Error for DataSendError {} - -impl DataChannel { - pub(crate) fn new(cxx_handle: UniquePtr) -> Self { - let mut observer = Box::new(InternalDataChannelObserver::default()); - - let mut dc = unsafe { - Self { - cxx_handle, - native_observer: sys_dc::ffi::create_native_data_channel_observer(Box::new( - sys_dc::DataChannelObserverWrapper::new(&mut *observer), - )), - observer, - } - }; - - unsafe { - dc.cxx_handle - .pin_mut() - .register_observer(dc.native_observer.pin_mut()); - } - - dc - } - - pub fn send(&self, data: &[u8], binary: bool) -> Result<(), DataSendError> { - let buffer = sys_dc::ffi::DataBuffer { - ptr: data.as_ptr(), - len: data.len(), - binary, - }; - - self.cxx_handle - .send(&buffer) - .then_some(()) - .ok_or(DataSendError {}) - } - - pub fn label(&self) -> String { - self.cxx_handle.label() - } - - pub fn state(&self) -> DataState { - self.cxx_handle.state() - } - - pub fn close(&self) { - self.cxx_handle.close(); - } - - pub fn on_state_change(&mut self, handler: OnStateChangeHandler) { - *self.observer.on_state_change_handler.lock().unwrap() = Some(handler); - } - - pub fn on_message(&mut self, handler: OnMessageHandler) { - *self.observer.on_message_handler.lock().unwrap() = Some(handler); - } - - pub fn on_buffered_amount_change(&mut self, handler: OnBufferedAmountChangeHandler) { - *self - .observer - .on_buffered_amount_change_handler - .lock() - .unwrap() = Some(handler); - } -} - -impl Drop for DataChannel { - fn drop(&mut self) { - self.cxx_handle.pin_mut().unregister_observer(); - } -} - -pub type OnStateChangeHandler = Box; -pub type OnMessageHandler = Box; -pub type OnBufferedAmountChangeHandler = Box; - -#[derive(Default)] -struct InternalDataChannelObserver { - on_state_change_handler: Mutex>, - on_message_handler: Mutex>, - on_buffered_amount_change_handler: Mutex>, -} - -impl sys_dc::DataChannelObserver for InternalDataChannelObserver { - fn on_state_change(&self) { - trace!("DataChannel: on_state_change"); - let mut handler = self.on_state_change_handler.lock().unwrap(); - if let Some(f) = handler.as_mut() { - f(); - } - } - - fn on_message(&self, data: &[u8], is_binary: bool) { - trace!("DataChannel: on_message"); - let mut handler = self.on_message_handler.lock().unwrap(); - if let Some(f) = handler.as_mut() { - f(data, is_binary); - } - } - - fn on_buffered_amount_change(&self, sent_data_size: u64) { - trace!("DataChannel: on_buffered_amount_change"); - let mut handler = self.on_buffered_amount_change_handler.lock().unwrap(); - if let Some(f) = handler.as_mut() { - f(sent_data_size); - } - } -} - -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct DataChannelInit { - #[deprecated] - pub reliable: bool, pub ordered: bool, pub max_retransmit_time: Option, pub max_retransmits: Option, @@ -159,11 +16,10 @@ pub struct DataChannelInit { impl Default for DataChannelInit { fn default() -> Self { Self { - reliable: false, ordered: true, max_retransmit_time: None, max_retransmits: None, - protocol: "".to_string(), + protocol: String::new(), negotiated: false, id: -1, priority: None, @@ -171,20 +27,72 @@ impl Default for DataChannelInit { } } -impl From for sys_dc::ffi::DataChannelInit { - fn from(init: DataChannelInit) -> Self { - Self { - reliable: init.reliable, - ordered: init.ordered, - has_max_retransmit_time: init.max_retransmit_time.is_some(), - max_retransmit_time: init.max_retransmit_time.unwrap_or_default(), - has_max_retransmits: init.max_retransmits.is_some(), - max_retransmits: init.max_retransmits.unwrap_or_default(), - protocol: init.protocol, - negotiated: init.negotiated, - id: init.id, - has_priority: init.priority.is_some(), - priority: init.priority.unwrap_or(Priority::Low), - } +#[derive(Debug, Error)] +pub enum DataChannelError { + #[error("failed to send data, dc not open? send buffer is full ?")] + Send, + #[error("only utf8 strings can be sent")] + Utf8(#[from] Utf8Error), +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum DataState { + Connecting, + Open, + Closing, + Closed, +} + +#[derive(Debug)] +pub struct DataBuffer<'a> { + pub data: &'a [u8], + pub binary: bool, +} + +pub type OnStateChange = Box; +pub type OnMessage = Box; +pub type OnBufferedAmountChange = Box; + +#[derive(Clone)] +pub struct DataChannel { + pub(crate) handle: dc_imp::DataChannel, +} + +impl DataChannel { + pub fn send(&self, data: &[u8], binary: bool) -> Result<(), DataChannelError> { + self.handle.send(data, binary) + } + + pub fn label(&self) -> String { + self.handle.label() + } + + pub fn state(&self) -> DataState { + self.handle.state() + } + + pub fn close(&self) { + self.handle.close() + } + + pub fn on_state_change(&self, callback: Option) { + self.handle.on_state_change(callback) + } + + pub fn on_message(&self, callback: Option) { + self.handle.on_message(callback) + } + + pub fn on_buffered_amount_change(&self, callback: Option) { + self.handle.on_buffered_amount_change(callback) + } +} + +impl Debug for DataChannel { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DataChannel") + .field("label", &self.label()) + .field("state", &self.state()) + .finish() } } diff --git a/livekit-webrtc/src/ice_candidate.rs b/livekit-webrtc/src/ice_candidate.rs new file mode 100644 index 0000000..6d2cc1f --- /dev/null +++ b/livekit-webrtc/src/ice_candidate.rs @@ -0,0 +1,40 @@ +use crate::{imp::ice_candidate as imp_ic, session_description::SdpParseError}; +use std::fmt::Debug; + +pub struct IceCandidate { + pub(crate) handle: imp_ic::IceCandidate, +} + +impl IceCandidate { + pub fn parse( + sdp_mid: &str, + sdp_mline_index: i32, + sdp: &str, + ) -> Result { + imp_ic::IceCandidate::parse(sdp_mid, sdp_mline_index, sdp) + } + + pub fn sdp_mid(&self) -> String { + self.handle.sdp_mid() + } + + pub fn sdp_mline_index(&self) -> i32 { + self.handle.sdp_mline_index() + } + + pub fn candidate(&self) -> String { + self.handle.candidate() + } +} + +impl ToString for IceCandidate { + fn to_string(&self) -> String { + self.handle.to_string() + } +} + +impl Debug for IceCandidate { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("IceCandidate").finish() + } +} diff --git a/livekit-webrtc/src/jsep.rs b/livekit-webrtc/src/jsep.rs deleted file mode 100644 index d9bba01..0000000 --- a/livekit-webrtc/src/jsep.rs +++ /dev/null @@ -1,103 +0,0 @@ -use std::fmt::{Debug, Formatter}; - -use cxx::{SharedPtr, UniquePtr}; - -pub use sys_jsep::ffi::{SdpParseError, SdpType}; -use webrtc_sys::jsep as sys_jsep; - -#[derive(Clone)] -pub struct IceCandidate { - cxx_handle: SharedPtr, -} - -impl Debug for IceCandidate { - fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - write!(f, "IceCandidate[{}]", self.to_string()) - } -} - -impl IceCandidate { - pub fn from( - sdp_mid: &str, - sdp_mline_index: i32, - sdp: &str, - ) -> Result { - let res = sys_jsep::ffi::create_ice_candidate( - sdp_mid.to_string(), - sdp_mline_index, - sdp.to_string(), - ); - - match res { - Ok(cxx_handle) => Ok(IceCandidate::new(cxx_handle)), - Err(e) => Err(unsafe { SdpParseError::from(e.what()) }), - } - } - - pub(crate) fn new(cxx_handle: SharedPtr) -> Self { - Self { cxx_handle } - } - - pub(crate) fn release(self) -> SharedPtr { - self.cxx_handle - } - - pub fn sdp_mid(&self) -> String { - self.cxx_handle.sdp_mid() - } - - pub fn sdp_mline_index(&self) -> i32 { - self.cxx_handle.sdp_mline_index() - } - - pub fn candidate(&self) -> String { - self.cxx_handle.candidate() - } -} - -impl ToString for IceCandidate { - fn to_string(&self) -> String { - self.cxx_handle.stringify() - } -} - -pub struct SessionDescription { - cxx_handle: UniquePtr, -} - -impl Debug for SessionDescription { - fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - write!(f, "SessionDescription[{}]", self.to_string()) - } -} - -impl SessionDescription { - pub fn from(sdp_type: SdpType, description: &str) -> Result { - let res = sys_jsep::ffi::create_session_description(sdp_type, description.to_string()); - - match res { - Ok(cxx_handle) => Ok(SessionDescription::new(cxx_handle)), - Err(e) => Err(unsafe { SdpParseError::from(e.what()) }), - } - } - - pub(crate) fn new(cxx_handle: UniquePtr) -> Self { - Self { cxx_handle } - } - - pub(crate) fn release(self) -> UniquePtr { - self.cxx_handle - } -} - -impl ToString for SessionDescription { - fn to_string(&self) -> String { - self.cxx_handle.stringify() - } -} - -impl Clone for SessionDescription { - fn clone(&self) -> Self { - SessionDescription::new(self.cxx_handle.clone()) - } -} diff --git a/livekit-webrtc/src/lib.rs b/livekit-webrtc/src/lib.rs index 1bdbe38..269ec6e 100644 --- a/livekit-webrtc/src/lib.rs +++ b/livekit-webrtc/src/lib.rs @@ -1,15 +1,48 @@ +use thiserror::Error; + +#[cfg_attr(target_arch = "wasm32", path = "web/mod.rs")] +#[cfg_attr(not(target_arch = "wasm32"), path = "native/mod.rs")] +mod imp; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum MediaType { + Audio, + Video, + Data, + Unsupported, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum RtcErrorType { + Internal, + InvalidSdp, + InvalidState, +} + +#[derive(Error, Debug)] +#[error("an RtcError occured: {error_type:?} - {message}")] +pub struct RtcError { + pub error_type: RtcErrorType, + pub message: String, +} + pub mod data_channel; -pub mod jsep; +pub mod ice_candidate; pub mod media_stream; pub mod peer_connection; pub mod peer_connection_factory; pub mod prelude; -pub mod rtc_error; pub mod rtp_parameters; pub mod rtp_receiver; pub mod rtp_sender; pub mod rtp_transceiver; +pub mod session_description; pub mod video_frame; -pub mod video_frame_buffer; -pub mod webrtc; -pub mod yuv_helper; +pub mod video_source; +pub mod video_stream; + +#[cfg(not(target_arch = "wasm32"))] +pub mod native { + pub use crate::imp::yuv_helper; + pub use webrtc_sys::webrtc::ffi::create_random_uuid; +} diff --git a/livekit-webrtc/src/media_stream.rs b/livekit-webrtc/src/media_stream.rs index ec5bd50..92ce669 100644 --- a/livekit-webrtc/src/media_stream.rs +++ b/livekit-webrtc/src/media_stream.rs @@ -1,254 +1,139 @@ -use cxx::{SharedPtr, UniquePtr}; +use crate::imp::media_stream as imp_ms; use livekit_utils::enum_dispatch; -use std::fmt::{Debug, Formatter}; -use std::sync::{Arc, Mutex}; -use webrtc_sys::media_stream as sys_ms; -use webrtc_sys::MEDIA_TYPE_VIDEO; +use std::fmt::Debug; -pub use sys_ms::ffi::ContentHint; -pub use sys_ms::ffi::TrackState; - -use crate::video_frame::VideoFrame; -use crate::video_frame_buffer::VideoFrameBuffer; - -pub trait MediaStreamTrackTrait { - fn kind(&self) -> String; - fn id(&self) -> String; - fn enabled(&self) -> bool; - fn set_enabled(&self, enabled: bool) -> bool; - fn state(&self) -> TrackState; +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum RtcTrackState { + Live, + Ended, } #[derive(Clone)] -pub enum MediaStreamTrackHandle { - Audio(Arc), - Video(Arc), +pub struct MediaStream { + pub(crate) handle: imp_ms::MediaStream, } -impl MediaStreamTrackHandle { - pub(crate) fn new(cxx_handle: SharedPtr) -> Self { - if cxx_handle.kind() == MEDIA_TYPE_VIDEO { - Self::Video(VideoTrack::new(cxx_handle)) - } else { - Self::Audio(AudioTrack::new(cxx_handle)) - } +impl MediaStream { + pub fn id(&self) -> String { + self.handle.id() } - // TODO(theomonnom): enum_dispatch with visibility support? - pub(crate) fn cxx_handle(&self) -> SharedPtr { - match self { - Self::Video(video) => video.cxx_handle(), - Self::Audio(audio) => audio.cxx_handle(), - } + pub fn audio_tracks(&self) -> Vec { + self.handle.audio_tracks() + } + + pub fn video_tracks(&self) -> Vec { + self.handle.video_tracks() } } -impl Debug for MediaStreamTrackHandle { - fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - f.debug_struct("MediaStreamTrack") +impl Debug for MediaStream { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MediaStream") + .field("id", &self.id()) + .field("audio_tracks", &self.audio_tracks()) + .field("video_tracks", &self.video_tracks()) + .finish() + } +} + +#[derive(Clone)] +pub struct RtcVideoTrack { + pub(crate) handle: imp_ms::RtcVideoTrack, +} + +#[derive(Clone)] +pub struct RtcAudioTrack { + pub(crate) handle: imp_ms::RtcAudioTrack, +} + +#[derive(Debug, Clone)] +pub enum MediaStreamTrack { + Video(RtcVideoTrack), + Audio(RtcAudioTrack), +} + +#[cfg(not(target_arch = "wasm32"))] +impl MediaStreamTrack { + enum_dispatch!( + [Video, Audio]; + pub(crate) fn sys_handle(self: &Self) -> cxx::SharedPtr; + ); +} + +impl MediaStreamTrack { + enum_dispatch!( + [Video, Audio]; + pub fn id(self: &Self) -> String; + pub fn enabled(self: &Self) -> bool; + pub fn set_enabled(self: &Self, enabled: bool) -> bool; + pub fn state(self: &Self) -> RtcTrackState; + ); +} + +macro_rules! media_stream_track { + () => { + pub fn id(&self) -> String { + self.handle.id() + } + + pub fn enabled(&self) -> bool { + self.handle.enabled() + } + + pub fn set_enabled(&self, enabled: bool) -> bool { + self.handle.set_enabled(enabled) + } + + pub fn state(&self) -> RtcTrackState { + self.handle.state().into() + } + + #[cfg(not(target_arch = "wasm32"))] + pub(crate) fn sys_handle( + &self, + ) -> cxx::SharedPtr { + self.handle.sys_handle() + } + }; +} + +impl RtcVideoTrack { + media_stream_track!(); +} + +impl RtcAudioTrack { + media_stream_track!(); +} + +impl Debug for RtcAudioTrack { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RtcAudioTrack") .field("id", &self.id()) - .field("kind", &self.kind()) .field("enabled", &self.enabled()) .field("state", &self.state()) .finish() } } -impl MediaStreamTrackTrait for MediaStreamTrackHandle { - enum_dispatch!( - [Audio, Video] - fnc!(kind, &Self, [], String); - fnc!(id, &Self, [], String); - fnc!(enabled, &Self, [], bool); - fnc!(state, &Self, [], TrackState); - fnc!(set_enabled, &Self, [enabled: bool], bool); - ); -} - -pub struct AudioTrack { - cxx_handle: SharedPtr, -} - -impl AudioTrack { - fn new(cxx_handle: SharedPtr) -> Arc { - Arc::new(Self { cxx_handle }) - } - - pub(crate) fn cxx_handle(&self) -> SharedPtr { - self.cxx_handle.clone() - } -} - -pub struct VideoTrack { - cxx_handle: SharedPtr, - observer: Box, - - // Keep alive for c++ - native_observer: UniquePtr, -} - -impl VideoTrack { - fn new(cxx_handle: SharedPtr) -> Arc { - let mut observer = Box::new(InternalVideoTrackSink::default()); - - let mut track = unsafe { - Self { - cxx_handle, - native_observer: { - sys_ms::ffi::create_native_video_frame_sink(Box::new( - sys_ms::VideoFrameSinkWrapper::new(&mut *observer), - )) - }, - observer, - } - }; - - unsafe { - (*track.video_handle()).add_sink(track.native_observer.pin_mut()); - } - - Arc::new(track) - } - - pub(crate) fn cxx_handle(&self) -> SharedPtr { - self.cxx_handle.clone() - } - - fn video_handle(&self) -> *const sys_ms::ffi::VideoTrack { - unsafe { sys_ms::ffi::media_to_video(&*self.cxx_handle) } - } - - pub fn set_should_receive(&self, should_receive: bool) { - unsafe { (*self.video_handle()).set_should_receive(should_receive) } - } - - pub fn set_content_hint(&self, hint: ContentHint) { - unsafe { (*self.video_handle()).set_content_hint(hint) } - } - - pub fn should_receive(&self) -> bool { - unsafe { (*self.video_handle()).should_receive() } - } - - pub fn content_hint(&self) -> ContentHint { - unsafe { (*self.video_handle()).content_hint() } - } - - pub fn on_frame(&self, handler: OnFrameHandler) { - *self.observer.on_frame_handler.lock().unwrap() = Some(handler); - } - - pub fn on_discarded_frame(&self, handler: OnDiscardedFrameHandler) { - *self.observer.on_discarded_frame_handler.lock().unwrap() = Some(handler); - } - - pub fn on_constraints_changed(&self, handler: OnConstraintsChangedHandler) { - *self.observer.on_constraints_changed_handler.lock().unwrap() = Some(handler); - } -} - -impl Drop for VideoTrack { - fn drop(&mut self) { - unsafe { - (*self.video_handle()).remove_sink(self.native_observer.pin_mut()); - } - } -} - -macro_rules! impl_media_stream_track_trait { - ($x:ty) => { - impl MediaStreamTrackTrait for $x { - fn kind(&self) -> String { - self.cxx_handle.kind() - } - - fn id(&self) -> String { - self.cxx_handle.id() - } - - fn enabled(&self) -> bool { - self.cxx_handle.enabled() - } - - fn set_enabled(&self, enabled: bool) -> bool { - self.cxx_handle.set_enabled(enabled) - } - - fn state(&self) -> TrackState { - self.cxx_handle.state() - } - } - }; -} - -impl_media_stream_track_trait!(VideoTrack); -impl_media_stream_track_trait!(AudioTrack); - -pub type OnFrameHandler = Box; -pub type OnDiscardedFrameHandler = Box; -pub type OnConstraintsChangedHandler = Box; - -#[derive(Default)] -struct InternalVideoTrackSink { - on_frame_handler: Mutex>, - on_discarded_frame_handler: Mutex>, - on_constraints_changed_handler: Mutex>, -} - -pub struct VideoTrackSourceConstraints { - pub min_fps: Option, - pub max_fps: Option, -} - -impl From for VideoTrackSourceConstraints { - fn from(cst: sys_ms::ffi::VideoTrackSourceConstraints) -> Self { - Self { - min_fps: (cst.min_fps != 1.0).then_some(cst.min_fps), - max_fps: (cst.max_fps != 1.0).then_some(cst.max_fps), - } - } -} - -impl sys_ms::VideoFrameSink for InternalVideoTrackSink { - fn on_frame(&self, frame: UniquePtr) { - if let Some(cb) = self.on_frame_handler.lock().unwrap().as_mut() { - let frame = VideoFrame::new(frame); - let video_frame_buffer = unsafe { frame.video_frame_buffer() }; - cb(frame, video_frame_buffer); - } - } - - fn on_discarded_frame(&self) { - if let Some(cb) = self.on_discarded_frame_handler.lock().unwrap().as_mut() { - cb(); - } - } - - fn on_constraints_changed(&self, constraints: sys_ms::ffi::VideoTrackSourceConstraints) { - if let Some(cb) = self.on_constraints_changed_handler.lock().unwrap().as_mut() { - cb(constraints.into()); - } - } -} - -pub struct MediaStream { - cxx_handle: SharedPtr, -} - -impl Debug for MediaStream { - fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - f.debug_struct("MediaStream") +impl Debug for RtcVideoTrack { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RtcVideoTrack") .field("id", &self.id()) + .field("enabled", &self.enabled()) + .field("state", &self.state()) .finish() } } -impl MediaStream { - pub(crate) fn new(cxx_handle: SharedPtr) -> Self { - Self { cxx_handle } - } - - pub fn id(&self) -> String { - self.cxx_handle.id() +impl From for MediaStreamTrack { + fn from(track: RtcAudioTrack) -> Self { + Self::Audio(track) + } +} + +impl From for MediaStreamTrack { + fn from(track: RtcVideoTrack) -> Self { + Self::Video(track) } } diff --git a/livekit-webrtc/src/native/data_channel.rs b/livekit-webrtc/src/native/data_channel.rs new file mode 100644 index 0000000..cbe2289 --- /dev/null +++ b/livekit-webrtc/src/native/data_channel.rs @@ -0,0 +1,137 @@ +use crate::data_channel::{ + DataBuffer, DataChannelError, DataChannelInit, DataState, OnBufferedAmountChange, OnMessage, + OnStateChange, +}; +use cxx::SharedPtr; +use std::str; +use std::sync::{Arc, Mutex}; +use webrtc_sys::data_channel as sys_dc; + +impl From for DataState { + fn from(value: sys_dc::ffi::DataState) -> Self { + match value { + sys_dc::ffi::DataState::Connecting => Self::Connecting, + sys_dc::ffi::DataState::Open => Self::Open, + sys_dc::ffi::DataState::Closing => Self::Closing, + sys_dc::ffi::DataState::Closed => Self::Closed, + _ => panic!("unknown data channel state"), + } + } +} + +impl From for sys_dc::ffi::DataChannelInit { + fn from(value: DataChannelInit) -> Self { + Self { + ordered: value.ordered, + has_max_retransmit_time: value.max_retransmit_time.is_some(), + max_retransmit_time: value.max_retransmit_time.unwrap_or_default(), + has_max_retransmits: value.max_retransmits.is_some(), + max_retransmits: value.max_retransmits.unwrap_or_default(), + protocol: value.protocol, + id: value.id, + has_priority: false, + priority: sys_dc::ffi::Priority::Medium, + negotiated: value.negotiated, + } + } +} + +#[derive(Clone)] +pub struct DataChannel { + #[allow(dead_code)] + native_observer: SharedPtr, + observer: Arc, + + pub(crate) sys_handle: SharedPtr, +} + +impl DataChannel { + pub fn configure(sys_handle: SharedPtr) -> Self { + unsafe { + let observer = Arc::new(DataChannelObserver::default()); + let dc = Self { + sys_handle: sys_handle.clone(), + native_observer: sys_dc::ffi::create_native_data_channel_observer( + Box::new(sys_dc::DataChannelObserverWrapper::new(observer.clone())), + &*sys_handle as *const _ as *mut _, + ), + observer, + }; + + dc.sys_handle + .register_observer(&*dc.native_observer as *const _ as *mut _); + dc + } + } + + pub fn send(&self, data: &[u8], binary: bool) -> Result<(), DataChannelError> { + if !binary { + str::from_utf8(data)?; + } + + let buffer = sys_dc::ffi::DataBuffer { + ptr: data.as_ptr(), + len: data.len(), + binary, + }; + + self.sys_handle + .send(&buffer) + .then_some(()) + .ok_or(DataChannelError::Send) + } + + pub fn label(&self) -> String { + self.sys_handle.label() + } + + pub fn state(&self) -> DataState { + self.sys_handle.state().into() + } + + pub fn close(&self) { + self.sys_handle.close(); + } + + pub fn on_state_change(&self, handler: Option) { + *self.observer.state_change_handler.lock().unwrap() = handler; + } + + pub fn on_message(&self, handler: Option) { + *self.observer.message_handler.lock().unwrap() = handler; + } + + pub fn on_buffered_amount_change(&self, handler: Option) { + *self.observer.buffered_amount_change_handler.lock().unwrap() = handler; + } +} + +#[derive(Default)] +struct DataChannelObserver { + state_change_handler: Mutex>, + message_handler: Mutex>, + buffered_amount_change_handler: Mutex>, +} + +impl sys_dc::DataChannelObserver for DataChannelObserver { + fn on_state_change(&self, state: sys_dc::ffi::DataState) { + let mut handler = self.state_change_handler.lock().unwrap(); + if let Some(f) = handler.as_mut() { + f(state.into()); + } + } + + fn on_message(&self, data: &[u8], binary: bool) { + let mut handler = self.message_handler.lock().unwrap(); + if let Some(f) = handler.as_mut() { + f(DataBuffer { data, binary }); + } + } + + fn on_buffered_amount_change(&self, sent_data_size: u64) { + let mut handler = self.buffered_amount_change_handler.lock().unwrap(); + if let Some(f) = handler.as_mut() { + f(sent_data_size); + } + } +} diff --git a/livekit-webrtc/src/native/ice_candidate.rs b/livekit-webrtc/src/native/ice_candidate.rs new file mode 100644 index 0000000..31635b8 --- /dev/null +++ b/livekit-webrtc/src/native/ice_candidate.rs @@ -0,0 +1,48 @@ +use crate::ice_candidate as ic; +use crate::session_description::SdpParseError; +use cxx::SharedPtr; +use webrtc_sys::jsep as sys_jsep; + +#[derive(Clone)] +pub struct IceCandidate { + pub(crate) sys_handle: SharedPtr, +} + +impl IceCandidate { + pub fn parse( + sdp_mid: &str, + sdp_mline_index: i32, + sdp: &str, + ) -> Result { + let res = sys_jsep::ffi::create_ice_candidate( + sdp_mid.to_string(), + sdp_mline_index, + sdp.to_string(), + ); + + match res { + Ok(sys_handle) => Ok(ic::IceCandidate { + handle: IceCandidate { sys_handle }, + }), + Err(e) => Err(unsafe { sys_jsep::ffi::SdpParseError::from(e.what()).into() }), + } + } + + pub fn sdp_mid(&self) -> String { + self.sys_handle.sdp_mid() + } + + pub fn sdp_mline_index(&self) -> i32 { + self.sys_handle.sdp_mline_index() + } + + pub fn candidate(&self) -> String { + self.sys_handle.candidate() + } +} + +impl ToString for IceCandidate { + fn to_string(&self) -> String { + self.sys_handle.stringify() + } +} diff --git a/livekit-webrtc/src/native/media_stream.rs b/livekit-webrtc/src/native/media_stream.rs new file mode 100644 index 0000000..069f6e8 --- /dev/null +++ b/livekit-webrtc/src/native/media_stream.rs @@ -0,0 +1,118 @@ +use crate::media_stream::{self, MediaStreamTrack, RtcTrackState}; +use cxx::SharedPtr; +use webrtc_sys::media_stream as sys_ms; +use webrtc_sys::media_stream::ffi::{ + audio_to_media, media_to_audio, media_to_video, video_to_media, +}; +use webrtc_sys::{MEDIA_TYPE_AUDIO, MEDIA_TYPE_VIDEO}; + +impl From for RtcTrackState { + fn from(state: sys_ms::ffi::TrackState) -> Self { + match state { + sys_ms::ffi::TrackState::Live => RtcTrackState::Live, + sys_ms::ffi::TrackState::Ended => RtcTrackState::Ended, + _ => panic!("unknown TrackState"), + } + } +} + +#[derive(Clone)] +pub struct MediaStream { + pub(crate) sys_handle: SharedPtr, +} + +impl MediaStream { + pub fn id(&self) -> String { + self.sys_handle.id() + } + + pub fn audio_tracks(&self) -> Vec { + self.sys_handle + .get_audio_tracks() + .into_iter() + .map(|t| media_stream::RtcAudioTrack { + handle: RtcAudioTrack { sys_handle: t.ptr }, + }) + .collect() + } + + pub fn video_tracks(&self) -> Vec { + self.sys_handle + .get_video_tracks() + .into_iter() + .map(|t| media_stream::RtcVideoTrack { + handle: RtcVideoTrack { sys_handle: t.ptr }, + }) + .collect() + } +} + +pub fn new_media_stream_track( + sys_handle: SharedPtr, +) -> MediaStreamTrack { + if sys_handle.kind() == MEDIA_TYPE_AUDIO { + MediaStreamTrack::Audio(media_stream::RtcAudioTrack { + handle: RtcAudioTrack { + sys_handle: media_to_audio(sys_handle), + }, + }) + } else if sys_handle.kind() == MEDIA_TYPE_VIDEO { + MediaStreamTrack::Video(media_stream::RtcVideoTrack { + handle: RtcVideoTrack { + sys_handle: media_to_video(sys_handle), + }, + }) + } else { + panic!("unknown track kind") + } +} + +macro_rules! impl_media_stream_track { + ($cast:ident) => { + pub fn id(&self) -> String { + let ptr = sys_ms::ffi::$cast(self.sys_handle.clone()); + ptr.id() + } + + pub fn enabled(&self) -> bool { + let ptr = sys_ms::ffi::$cast(self.sys_handle.clone()); + ptr.enabled() + } + + pub fn set_enabled(&self, enabled: bool) -> bool { + let ptr = sys_ms::ffi::$cast(self.sys_handle.clone()); + ptr.set_enabled(enabled) + } + + pub fn state(&self) -> RtcTrackState { + let ptr = sys_ms::ffi::$cast(self.sys_handle.clone()); + ptr.state().into() + } + }; +} + +#[derive(Clone)] +pub struct RtcVideoTrack { + pub(crate) sys_handle: SharedPtr, +} + +impl RtcVideoTrack { + impl_media_stream_track!(video_to_media); + + pub fn sys_handle(&self) -> SharedPtr { + video_to_media(self.sys_handle.clone()) + } +} + +#[derive(Clone)] +pub struct RtcAudioTrack { + pub(crate) sys_handle: SharedPtr, +} + +impl RtcAudioTrack { + impl_media_stream_track!(audio_to_media); + + pub fn sys_handle(&self) -> SharedPtr { + audio_to_media(self.sys_handle.clone()) + } +} diff --git a/livekit-webrtc/src/native/mod.rs b/livekit-webrtc/src/native/mod.rs new file mode 100644 index 0000000..e890949 --- /dev/null +++ b/livekit-webrtc/src/native/mod.rs @@ -0,0 +1,48 @@ +pub mod data_channel; +pub mod ice_candidate; +pub mod media_stream; +pub mod peer_connection; +pub mod peer_connection_factory; +pub mod rtp_parameters; +pub mod rtp_receiver; +pub mod rtp_sender; +pub mod rtp_transceiver; +pub mod session_description; +pub mod video_frame; +pub mod video_source; +pub mod video_stream; +pub mod yuv_helper; + +use crate::MediaType; +use crate::{RtcError, RtcErrorType}; +use webrtc_sys::rtc_error as sys_err; +use webrtc_sys::webrtc as sys_rtc; + +impl From for RtcErrorType { + fn from(value: sys_err::ffi::RTCErrorType) -> Self { + match value { + sys_err::ffi::RTCErrorType::InvalidState => Self::InvalidState, + _ => Self::Internal, + } + } +} + +impl From for RtcError { + fn from(value: sys_err::ffi::RTCError) -> Self { + Self { + error_type: value.error_type.into(), + message: value.message, + } + } +} + +impl From for sys_rtc::ffi::MediaType { + fn from(value: MediaType) -> Self { + match value { + MediaType::Audio => Self::Audio, + MediaType::Video => Self::Video, + MediaType::Data => Self::Data, + MediaType::Unsupported => Self::Unsupported, + } + } +} diff --git a/livekit-webrtc/src/native/peer_connection.rs b/livekit-webrtc/src/native/peer_connection.rs new file mode 100644 index 0000000..b1eabbc --- /dev/null +++ b/livekit-webrtc/src/native/peer_connection.rs @@ -0,0 +1,579 @@ +use crate::data_channel::DataChannel; +use crate::data_channel::DataChannelInit; +use crate::ice_candidate::IceCandidate; +use crate::imp::data_channel as imp_dc; +use crate::imp::ice_candidate as imp_ic; +use crate::imp::media_stream as imp_ms; +use crate::imp::rtp_receiver as imp_rr; +use crate::imp::rtp_sender as imp_rs; +use crate::imp::rtp_transceiver as imp_rt; +use crate::imp::session_description as imp_sdp; +use crate::media_stream::{MediaStream, MediaStreamTrack}; +use crate::peer_connection::{ + AnswerOptions, IceCandidateError, IceConnectionState, IceGatheringState, OfferOptions, + OnConnectionChange, OnDataChannel, OnIceCandidate, OnIceCandidateError, OnIceConnectionChange, + OnIceGatheringChange, OnNegotiationNeeded, OnSignalingChange, OnTrack, PeerConnectionState, + SignalingState, TrackEvent, +}; +use crate::rtp_receiver::RtpReceiver; +use crate::rtp_sender::RtpSender; +use crate::rtp_transceiver::RtpTransceiver; +use crate::rtp_transceiver::RtpTransceiverInit; +use crate::MediaType; +use crate::{session_description::SessionDescription, RtcError}; +use cxx::{SharedPtr, UniquePtr}; +use futures::channel::oneshot; +use std::mem::ManuallyDrop; +use std::sync::{Arc, Mutex}; +use webrtc_sys::data_channel as sys_dc; +use webrtc_sys::jsep as sys_jsep; +use webrtc_sys::peer_connection as sys_pc; +use webrtc_sys::rtc_error as sys_err; + +impl From for sys_pc::ffi::RTCOfferAnswerOptions { + fn from(options: OfferOptions) -> Self { + Self { + ice_restart: options.ice_restart, + offer_to_receive_audio: options.offer_to_receive_audio as i32, + offer_to_receive_video: options.offer_to_receive_video as i32, + ..Default::default() + } + } +} + +impl From for sys_pc::ffi::RTCOfferAnswerOptions { + fn from(_options: AnswerOptions) -> Self { + Self::default() + } +} + +impl From for PeerConnectionState { + fn from(state: sys_pc::ffi::PeerConnectionState) -> Self { + match state { + sys_pc::ffi::PeerConnectionState::New => PeerConnectionState::New, + sys_pc::ffi::PeerConnectionState::Connecting => PeerConnectionState::Connecting, + sys_pc::ffi::PeerConnectionState::Connected => PeerConnectionState::Connected, + sys_pc::ffi::PeerConnectionState::Disconnected => PeerConnectionState::Disconnected, + sys_pc::ffi::PeerConnectionState::Failed => PeerConnectionState::Failed, + sys_pc::ffi::PeerConnectionState::Closed => PeerConnectionState::Closed, + _ => panic!("unknown PeerConnectionState"), + } + } +} + +impl From for IceConnectionState { + fn from(state: sys_pc::ffi::IceConnectionState) -> Self { + match state { + sys_pc::ffi::IceConnectionState::IceConnectionNew => IceConnectionState::New, + sys_pc::ffi::IceConnectionState::IceConnectionChecking => IceConnectionState::Checking, + sys_pc::ffi::IceConnectionState::IceConnectionConnected => { + IceConnectionState::Connected + } + sys_pc::ffi::IceConnectionState::IceConnectionCompleted => { + IceConnectionState::Completed + } + sys_pc::ffi::IceConnectionState::IceConnectionFailed => IceConnectionState::Failed, + sys_pc::ffi::IceConnectionState::IceConnectionDisconnected => { + IceConnectionState::Disconnected + } + sys_pc::ffi::IceConnectionState::IceConnectionClosed => IceConnectionState::Closed, + sys_pc::ffi::IceConnectionState::IceConnectionMax => IceConnectionState::Max, + _ => panic!("unknown IceConnectionState"), + } + } +} + +impl From for IceGatheringState { + fn from(state: sys_pc::ffi::IceGatheringState) -> Self { + match state { + sys_pc::ffi::IceGatheringState::IceGatheringNew => IceGatheringState::New, + sys_pc::ffi::IceGatheringState::IceGatheringGathering => IceGatheringState::Gathering, + sys_pc::ffi::IceGatheringState::IceGatheringComplete => IceGatheringState::Complete, + _ => panic!("unknown IceGatheringState"), + } + } +} + +impl From for SignalingState { + fn from(state: sys_pc::ffi::SignalingState) -> Self { + match state { + sys_pc::ffi::SignalingState::Stable => SignalingState::Stable, + sys_pc::ffi::SignalingState::HaveLocalOffer => SignalingState::HaveLocalOffer, + sys_pc::ffi::SignalingState::HaveRemoteOffer => SignalingState::HaveRemoteOffer, + sys_pc::ffi::SignalingState::HaveLocalPrAnswer => SignalingState::HaveLocalPrAnswer, + sys_pc::ffi::SignalingState::HaveRemotePrAnswer => SignalingState::HaveRemotePrAnswer, + sys_pc::ffi::SignalingState::Closed => SignalingState::Closed, + _ => panic!("unknown SignalingState"), + } + } +} + +#[derive(Clone)] +pub struct PeerConnection { + native_observer: SharedPtr, + observer: Arc, + + pub(crate) sys_handle: SharedPtr, +} + +impl PeerConnection { + pub fn configure( + sys_handle: SharedPtr, + observer: Arc, + native_observer: SharedPtr, + ) -> Self { + Self { + sys_handle, + observer, + native_observer, + } + } + + pub async fn create_offer( + &self, + options: OfferOptions, + ) -> Result { + let (mut native_wrapper, mut sdp_rx, mut err_rx) = create_sdp_observer(); + + unsafe { + self.sys_handle + .create_offer(native_wrapper.pin_mut(), options.into()); + } + + futures::select! { + sdp = sdp_rx => Ok(sdp.unwrap()), + err = err_rx => Err(err.unwrap()), + } + } + + pub async fn create_answer( + &self, + options: AnswerOptions, + ) -> Result { + let (mut native_wrapper, mut sdp_rx, mut err_rx) = create_sdp_observer(); + + unsafe { + self.sys_handle + .create_answer(native_wrapper.pin_mut(), options.into()); + } + + futures::select! { + sdp = sdp_rx => Ok(sdp.unwrap()), + err = err_rx => Err(err.unwrap()), + } + } + + pub async fn set_local_description(&self, desc: SessionDescription) -> Result<(), RtcError> { + let (tx, rx) = oneshot::channel(); + let wrapper = + sys_jsep::SetLocalSdpObserverWrapper(ManuallyDrop::new(Box::new(move |error| { + let _ = tx.send(if error.ok() { Ok(()) } else { Err(error) }); + }))); + + let mut native_wrapper = + sys_jsep::ffi::create_native_set_local_sdp_observer(Box::new(wrapper)); + + unsafe { + self.sys_handle + .set_local_description(desc.handle.sys_handle, native_wrapper.pin_mut()); + } + + rx.await.unwrap().map_err(Into::into) + } + + pub async fn set_remote_description(&self, desc: SessionDescription) -> Result<(), RtcError> { + let (tx, rx) = oneshot::channel(); + let wrapper = + sys_jsep::SetRemoteSdpObserverWrapper(ManuallyDrop::new(Box::new(move |error| { + let _ = tx.send(if error.ok() { Ok(()) } else { Err(error) }); + }))); + + let mut native_wrapper = + sys_jsep::ffi::create_native_set_remote_sdp_observer(Box::new(wrapper)); + + unsafe { + self.sys_handle + .set_remote_description(desc.handle.sys_handle, native_wrapper.pin_mut()); + } + + rx.await.unwrap().map_err(Into::into) + } + + pub async fn add_ice_candidate(&self, candidate: IceCandidate) -> Result<(), RtcError> { + let (tx, rx) = oneshot::channel(); + let observer = + sys_pc::AddIceCandidateObserverWrapper(ManuallyDrop::new(Box::new(|error| { + let _ = tx.send(if error.ok() { Ok(()) } else { Err(error) }); + }))); + + let mut native_observer = + sys_pc::ffi::create_native_add_ice_candidate_observer(Box::new(observer)); + self.sys_handle + .add_ice_candidate(candidate.handle.sys_handle, native_observer.pin_mut()); + + rx.await.unwrap().map_err(Into::into) + } + + pub fn create_data_channel( + &self, + label: &str, + init: DataChannelInit, + ) -> Result { + let native_init = sys_dc::ffi::create_data_channel_init(init.into()); + let res = self + .sys_handle + .create_data_channel(label.to_string(), native_init); + + match res { + Ok(sys_handle) => Ok(DataChannel { + handle: imp_dc::DataChannel::configure(sys_handle), + }), + Err(e) => Err(unsafe { sys_err::ffi::RTCError::from(e.what()).into() }), + } + } + + pub fn add_track>( + &self, + track: MediaStreamTrack, + stream_ids: &[T], + ) -> Result { + let stream_ids = stream_ids.iter().map(|s| s.as_ref().to_owned()).collect(); + let res = self.sys_handle.add_track(track.sys_handle(), &stream_ids); + + match res { + Ok(sys_handle) => Ok(RtpSender { + handle: imp_rs::RtpSender { sys_handle }, + }), + Err(e) => unsafe { Err(sys_err::ffi::RTCError::from(e.what()).into()) }, + } + } + + pub fn add_transceiver( + &self, + track: MediaStreamTrack, + init: RtpTransceiverInit, + ) -> Result { + let res = self + .sys_handle + .add_transceiver(track.sys_handle(), init.into()); + + match res { + Ok(sys_handle) => Ok(RtpTransceiver { + handle: imp_rt::RtpTransceiver { + sys_handle: sys_handle, + }, + }), + Err(e) => unsafe { Err(sys_err::ffi::RTCError::from(e.what()).into()) }, + } + } + + pub fn add_transceiver_for_media( + &self, + media_type: MediaType, + init: RtpTransceiverInit, + ) -> Result { + let res = self + .sys_handle + .add_transceiver_for_media(media_type.into(), init.into()); + + match res { + Ok(cxx_handle) => Ok(RtpTransceiver { + handle: imp_rt::RtpTransceiver { + sys_handle: cxx_handle, + }, + }), + Err(e) => unsafe { Err(sys_err::ffi::RTCError::from(e.what()).into()) }, + } + } + + pub fn close(&self) { + self.sys_handle.close(); + } + + pub fn connection_state(&self) -> PeerConnectionState { + self.sys_handle.connection_state().into() + } + + pub fn ice_connection_state(&self) -> IceConnectionState { + self.sys_handle.ice_connection_state().into() + } + + pub fn ice_gathering_state(&self) -> IceGatheringState { + self.sys_handle.ice_gathering_state().into() + } + + pub fn signaling_state(&self) -> SignalingState { + self.sys_handle.signaling_state().into() + } + + pub fn current_local_description(&self) -> Option { + let sdp = self.sys_handle.current_local_description(); + if sdp.is_null() { + return None; + } + + Some(SessionDescription { + handle: imp_sdp::SessionDescription { sys_handle: sdp }, + }) + } + + pub fn current_remote_description(&self) -> Option { + let sdp = self.sys_handle.current_remote_description(); + if sdp.is_null() { + return None; + } + + Some(SessionDescription { + handle: imp_sdp::SessionDescription { sys_handle: sdp }, + }) + } + + pub fn remove_track(&self, sender: RtpSender) -> Result<(), RtcError> { + self.sys_handle + .remove_track(sender.handle.sys_handle) + .map_err(|e| unsafe { sys_err::ffi::RTCError::from(e.what()).into() }) + } + + pub fn senders(&self) -> Vec { + self.sys_handle + .get_senders() + .into_iter() + .map(|sender| RtpSender { + handle: imp_rs::RtpSender { + sys_handle: sender.ptr, + }, + }) + .collect() + } + + pub fn receivers(&self) -> Vec { + self.sys_handle + .get_receivers() + .into_iter() + .map(|receiver| RtpReceiver { + handle: imp_rr::RtpReceiver { + sys_handle: receiver.ptr, + }, + }) + .collect() + } + + pub fn transceivers(&self) -> Vec { + self.sys_handle + .get_transceivers() + .into_iter() + .map(|transceiver| RtpTransceiver { + handle: imp_rt::RtpTransceiver { + sys_handle: transceiver.ptr, + }, + }) + .collect() + } + + pub fn on_connection_state_change(&self, f: Option) { + *self.observer.connection_change_handler.lock().unwrap() = f; + } + + pub fn on_data_channel(&self, f: Option) { + *self.observer.data_channel_handler.lock().unwrap() = f; + } + + pub fn on_ice_candidate(&self, f: Option) { + *self.observer.ice_candidate_handler.lock().unwrap() = f; + } + + pub fn on_ice_candidate_error(&self, f: Option) { + *self.observer.ice_candidate_error_handler.lock().unwrap() = f; + } + + pub fn on_ice_connection_state_change(&self, f: Option) { + *self.observer.ice_connection_change_handler.lock().unwrap() = f; + } + + pub fn on_ice_gathering_state_change(&self, f: Option) { + *self.observer.ice_gathering_change_handler.lock().unwrap() = f; + } + + pub fn on_negotiation_needed(&self, f: Option) { + *self.observer.negotiation_needed_handler.lock().unwrap() = f; + } + + pub fn on_signaling_state_change(&self, f: Option) { + *self.observer.signaling_change_handler.lock().unwrap() = f; + } + + pub fn on_track(&self, f: Option) { + *self.observer.track_handler.lock().unwrap() = f; + } +} + +fn create_sdp_observer() -> ( + UniquePtr, + oneshot::Receiver, + oneshot::Receiver, +) { + let (sdp_tx, sdp_rx) = oneshot::channel(); + let (err_tx, err_rx) = oneshot::channel(); + + let wrapper = sys_jsep::CreateSdpObserverWrapper { + on_success: ManuallyDrop::new(Box::new(move |session_description| { + let _ = sdp_tx.send(SessionDescription { + handle: imp_sdp::SessionDescription { + sys_handle: session_description, + }, + }); + })), + on_failure: ManuallyDrop::new(Box::new(move |error| { + let _ = err_tx.send(error.into()); + })), + }; + + ( + sys_jsep::ffi::create_native_create_sdp_observer(Box::new(wrapper)), + sdp_rx, + err_rx, + ) +} + +#[derive(Default)] +pub struct PeerObserver { + pub connection_change_handler: Mutex>, + pub data_channel_handler: Mutex>, + pub ice_candidate_handler: Mutex>, + pub ice_candidate_error_handler: Mutex>, + pub ice_connection_change_handler: Mutex>, + pub ice_gathering_change_handler: Mutex>, + pub negotiation_needed_handler: Mutex>, + pub signaling_change_handler: Mutex>, + pub track_handler: Mutex>, +} + +impl sys_pc::PeerConnectionObserver for PeerObserver { + fn on_signaling_change(&self, new_state: sys_pc::ffi::SignalingState) { + if let Some(f) = self.signaling_change_handler.lock().unwrap().as_mut() { + f(new_state.into()); + } + } + + fn on_add_stream(&self, _stream: SharedPtr) {} + + fn on_remove_stream(&self, _stream: SharedPtr) {} + + fn on_data_channel(&self, data_channel: SharedPtr) { + if let Some(f) = self.data_channel_handler.lock().unwrap().as_mut() { + f(DataChannel { + handle: imp_dc::DataChannel::configure(data_channel), + }); + } + } + + fn on_renegotiation_needed(&self) {} + + fn on_negotiation_needed_event(&self, event: u32) { + if let Some(f) = self.negotiation_needed_handler.lock().unwrap().as_mut() { + f(event); + } + } + + fn on_ice_connection_change(&self, _new_state: sys_pc::ffi::IceConnectionState) {} + + fn on_standardized_ice_connection_change(&self, new_state: sys_pc::ffi::IceConnectionState) { + if let Some(f) = self.ice_connection_change_handler.lock().unwrap().as_mut() { + f(new_state.into()); + } + } + + fn on_connection_change(&self, new_state: sys_pc::ffi::PeerConnectionState) { + if let Some(f) = self.connection_change_handler.lock().unwrap().as_mut() { + f(new_state.into()); + } + } + + fn on_ice_gathering_change(&self, new_state: sys_pc::ffi::IceGatheringState) { + if let Some(f) = self.ice_gathering_change_handler.lock().unwrap().as_mut() { + f(new_state.into()); + } + } + + fn on_ice_candidate(&self, candidate: SharedPtr) { + if let Some(f) = self.ice_candidate_handler.lock().unwrap().as_mut() { + f(IceCandidate { + handle: imp_ic::IceCandidate { + sys_handle: candidate, + }, + }); + } + } + + fn on_ice_candidate_error( + &self, + address: String, + port: i32, + url: String, + error_code: i32, + error_text: String, + ) { + if let Some(f) = self.ice_candidate_error_handler.lock().unwrap().as_mut() { + f(IceCandidateError { + address, + port, + url, + error_code, + error_text, + }); + } + } + + fn on_ice_candidates_removed( + &self, + _removed: Vec>, + ) { + } + + fn on_ice_connection_receiving_change(&self, _receiving: bool) {} + + fn on_ice_selected_candidate_pair_changed( + &self, + _event: sys_pc::ffi::CandidatePairChangeEvent, + ) { + } + + fn on_add_track( + &self, + _receiver: SharedPtr, + _streams: Vec>, + ) { + } + + fn on_track(&self, transceiver: SharedPtr) { + if let Some(f) = self.track_handler.lock().unwrap().as_mut() { + let receiver = transceiver.receiver(); + let streams = receiver.streams(); + let track = receiver.track(); + + f(TrackEvent { + receiver: RtpReceiver { + handle: imp_rr::RtpReceiver { + sys_handle: receiver, + }, + }, + streams: streams + .into_iter() + .map(|s| MediaStream { + handle: imp_ms::MediaStream { sys_handle: s.ptr }, + }) + .collect(), + track: imp_ms::new_media_stream_track(track), + transceiver: RtpTransceiver { + handle: imp_rt::RtpTransceiver { + sys_handle: transceiver, + }, + }, + }); + } + } + + fn on_remove_track(&self, _receiver: SharedPtr) {} + + fn on_interesting_usage(&self, _usage_pattern: i32) {} +} diff --git a/livekit-webrtc/src/native/peer_connection_factory.rs b/livekit-webrtc/src/native/peer_connection_factory.rs new file mode 100644 index 0000000..13440a9 --- /dev/null +++ b/livekit-webrtc/src/native/peer_connection_factory.rs @@ -0,0 +1,146 @@ +use crate::imp::media_stream as imp_ms; +use crate::imp::peer_connection as imp_pc; +use crate::media_stream::RtcVideoTrack; +use crate::peer_connection::PeerConnection; +use crate::peer_connection_factory::{ + ContinualGatheringPolicy, IceServer, IceTransportsType, RtcConfiguration, +}; +use crate::rtp_parameters::RtpCapabilities; +use crate::video_source::native::NativeVideoSource; +use crate::MediaType; +use crate::RtcError; +use cxx::SharedPtr; +use std::sync::Arc; +use webrtc_sys::peer_connection as sys_pc; +use webrtc_sys::peer_connection_factory as sys_pcf; +use webrtc_sys::rtc_error as sys_err; +use webrtc_sys::webrtc as sys_webrtc; + +impl From for sys_pcf::ffi::ICEServer { + fn from(value: IceServer) -> Self { + sys_pcf::ffi::ICEServer { + urls: value.urls, + username: value.username, + password: value.password, + } + } +} + +impl From for sys_pcf::ffi::ContinualGatheringPolicy { + fn from(value: ContinualGatheringPolicy) -> Self { + match value { + ContinualGatheringPolicy::GatherOnce => { + sys_pcf::ffi::ContinualGatheringPolicy::GatherOnce + } + ContinualGatheringPolicy::GatherContinually => { + sys_pcf::ffi::ContinualGatheringPolicy::GatherContinually + } + } + } +} + +impl From for sys_pcf::ffi::IceTransportsType { + fn from(value: IceTransportsType) -> Self { + match value { + IceTransportsType::None => sys_pcf::ffi::IceTransportsType::None, + IceTransportsType::Relay => sys_pcf::ffi::IceTransportsType::Relay, + IceTransportsType::NoHost => sys_pcf::ffi::IceTransportsType::NoHost, + IceTransportsType::All => sys_pcf::ffi::IceTransportsType::All, + } + } +} + +impl From for sys_pcf::ffi::RTCConfiguration { + fn from(value: RtcConfiguration) -> Self { + Self { + ice_servers: value.ice_servers.into_iter().map(Into::into).collect(), + continual_gathering_policy: value.continual_gathering_policy.into(), + ice_transport_type: value.ice_transport_type.into(), + } + } +} + +#[derive(Clone)] +pub struct RTCRuntime { + pub(crate) sys_handle: SharedPtr, +} + +impl Default for RTCRuntime { + fn default() -> Self { + Self { + sys_handle: sys_webrtc::ffi::create_rtc_runtime(), + } + } +} + +#[derive(Clone)] +pub struct PeerConnectionFactory { + sys_handle: SharedPtr, + + #[allow(unused)] + runtime: RTCRuntime, +} + +impl Default for PeerConnectionFactory { + fn default() -> Self { + let runtime = RTCRuntime::default(); + Self { + sys_handle: sys_pcf::ffi::create_peer_connection_factory(runtime.sys_handle.clone()), + runtime, + } + } +} + +impl PeerConnectionFactory { + pub fn create_peer_connection( + &self, + config: RtcConfiguration, + ) -> Result { + let native_config = sys_pcf::ffi::create_rtc_configuration(config.into()); + + unsafe { + let observer = Arc::new(imp_pc::PeerObserver::default()); + let native_observer = sys_pc::ffi::create_native_peer_connection_observer( + self.runtime.clone().sys_handle, + Box::new(sys_pc::PeerConnectionObserverWrapper::new(observer.clone())), + ); + + let res = self + .sys_handle + .create_peer_connection(native_config, &*native_observer as *const _ as *mut _); + + match res { + Ok(sys_handle) => Ok(PeerConnection { + handle: imp_pc::PeerConnection::configure( + sys_handle, + observer, + native_observer, + ), + }), + Err(e) => Err(sys_err::ffi::RTCError::from(e.what()).into()), + } + } + } + + pub fn create_video_track(&self, label: &str, source: NativeVideoSource) -> RtcVideoTrack { + RtcVideoTrack { + handle: imp_ms::RtcVideoTrack { + sys_handle: self + .sys_handle + .create_video_track(label.to_string(), source.handle.sys_handle()), + }, + } + } + + pub fn get_rtp_sender_capabilities(&self, media_type: MediaType) -> RtpCapabilities { + self.sys_handle + .get_rtp_sender_capabilities(media_type.into()) + .into() + } + + pub fn get_rtp_receiver_capabilities(&self, media_type: MediaType) -> RtpCapabilities { + self.sys_handle + .get_rtp_receiver_capabilities(media_type.into()) + .into() + } +} diff --git a/livekit-webrtc/src/native/rtp_parameters.rs b/livekit-webrtc/src/native/rtp_parameters.rs new file mode 100644 index 0000000..e50e98d --- /dev/null +++ b/livekit-webrtc/src/native/rtp_parameters.rs @@ -0,0 +1,285 @@ +use crate::rtp_parameters::*; +use crate::MediaType; +use webrtc_sys::rtp_parameters as sys_rp; +use webrtc_sys::webrtc as sys_webrtc; + +impl From for Priority { + fn from(value: sys_webrtc::ffi::Priority) -> Self { + match value { + sys_webrtc::ffi::Priority::VeryLow => Self::VeryLow, + sys_webrtc::ffi::Priority::Low => Self::Low, + sys_webrtc::ffi::Priority::Medium => Self::Medium, + sys_webrtc::ffi::Priority::High => Self::High, + _ => panic!("unknown Priority"), + } + } +} + +impl From for RtpHeaderExtensionParameters { + fn from(value: sys_rp::ffi::RtpExtension) -> Self { + Self { + uri: value.uri, + id: value.id, + encrypted: value.encrypt, + } + } +} + +impl From for RtpParameters { + fn from(value: sys_rp::ffi::RtpParameters) -> Self { + Self { + codecs: value.codecs.into_iter().map(Into::into).collect(), + header_extensions: value + .header_extensions + .into_iter() + .map(Into::into) + .collect(), + rtcp: value.rtcp.into(), + } + } +} + +impl From for RtpCodecParameters { + fn from(value: sys_rp::ffi::RtpCodecParameters) -> Self { + Self { + mime_type: value.mime_type, + payload_type: value.payload_type as u8, + clock_rate: value.has_clock_rate.then_some(value.clock_rate as u64), + channels: value.has_num_channels.then_some(value.num_channels as u16), + } + } +} + +impl From for RtcpParameters { + fn from(value: sys_rp::ffi::RtcpParameters) -> Self { + Self { + cname: value.cname, + reduced_size: value.reduced_size, + } + } +} + +impl From for RtpEncodingParameters { + fn from(value: sys_rp::ffi::RtpEncodingParameters) -> Self { + Self { + active: value.active, + max_bitrate: value + .has_max_bitrate_bps + .then_some(value.max_bitrate_bps as u64), + max_framerate: value.has_max_framerate.then_some(value.max_framerate), + priority: value.network_priority.into(), + rid: value.rid, + scale_resolution_down_by: value + .has_scale_resolution_down_by + .then_some(value.scale_resolution_down_by), + } + } +} + +impl From for RtpCodecCapability { + fn from(value: sys_rp::ffi::RtpCodecCapability) -> Self { + Self { + channels: value.has_num_channels.then_some(value.num_channels as u16), + mime_type: value.mime_type, + clock_rate: value.has_clock_rate.then_some(value.clock_rate as u64), + sdp_fmtp_line: { + let parameters: Vec = value + .parameters + .into_iter() + .map(|key_value| { + if !key_value.key.is_empty() { + format!("{}={}", key_value.key, key_value.value) + } else { + key_value.value + } + }) + .collect(); + + if !parameters.is_empty() { + Some(parameters.join(";")) + } else { + None + } + }, + } + } +} + +impl From for RtpHeaderExtensionCapability { + fn from(value: sys_rp::ffi::RtpHeaderExtensionCapability) -> Self { + Self { + direction: value.direction.into(), + uri: value.uri, + } + } +} + +impl From for RtpCapabilities { + fn from(value: sys_rp::ffi::RtpCapabilities) -> Self { + Self { + codecs: value.codecs.into_iter().map(Into::into).collect(), + header_extensions: value + .header_extensions + .into_iter() + .map(Into::into) + .collect(), + } + } +} + +impl From for sys_webrtc::ffi::Priority { + fn from(value: Priority) -> Self { + match value { + Priority::VeryLow => Self::VeryLow, + Priority::Low => Self::Low, + Priority::Medium => Self::Medium, + Priority::High => Self::High, + } + } +} + +impl From for sys_rp::ffi::RtpExtension { + fn from(value: RtpHeaderExtensionParameters) -> Self { + Self { + uri: value.uri, + id: value.id, + encrypt: value.encrypted, + } + } +} + +impl From for sys_rp::ffi::RtpParameters { + fn from(value: RtpParameters) -> Self { + Self { + codecs: value.codecs.into_iter().map(Into::into).collect(), + header_extensions: value + .header_extensions + .into_iter() + .map(Into::into) + .collect(), + encodings: Vec::new(), + rtcp: value.rtcp.into(), + transaction_id: "".to_string(), + mid: "".to_string(), + has_degradation_preference: false, + degradation_preference: sys_rp::ffi::DegradationPreference::Balanced, + } + } +} + +impl From for sys_rp::ffi::RtpCodecParameters { + fn from(value: RtpCodecParameters) -> Self { + Self { + payload_type: value.payload_type as i32, + mime_type: value.mime_type, + has_clock_rate: value.clock_rate.is_some(), + clock_rate: value.clock_rate.unwrap_or_default() as i32, + has_num_channels: value.channels.is_some(), + num_channels: value.channels.unwrap_or_default() as i32, + name: "".to_string(), + kind: sys_rp::ffi::MediaType::Audio, + has_max_ptime: false, + max_ptime: 0, + has_ptime: false, + ptime: 0, + rtcp_feedback: Vec::new(), + parameters: Vec::new(), + } + } +} + +impl From for sys_rp::ffi::RtcpParameters { + fn from(value: RtcpParameters) -> Self { + Self { + cname: value.cname, + reduced_size: value.reduced_size, + has_ssrc: false, + ssrc: 0, + mux: false, + } + } +} + +impl From for sys_rp::ffi::RtpEncodingParameters { + fn from(value: RtpEncodingParameters) -> Self { + Self { + active: value.active, + has_max_bitrate_bps: value.max_bitrate.is_some(), + max_bitrate_bps: value.max_bitrate.unwrap_or_default() as i32, + has_max_framerate: value.max_framerate.is_some(), + max_framerate: value.max_framerate.unwrap_or_default(), + network_priority: value.priority.into(), + rid: value.rid, + has_scale_resolution_down_by: value.scale_resolution_down_by.is_some(), + scale_resolution_down_by: value.scale_resolution_down_by.unwrap_or_default(), + adaptive_ptime: false, + bitrate_priority: sys_rp::DEFAULT_BITRATE_PRIORITY, + has_min_bitrate_bps: false, + min_bitrate_bps: 0, + has_num_temporal_layers: false, + num_temporal_layers: 0, + has_scalability_mode: false, + scalability_mode: "".to_string(), + has_ssrc: false, + ssrc: 0, + } + } +} + +impl From for sys_rp::ffi::RtpCodecCapability { + fn from(value: RtpCodecCapability) -> Self { + let mime_type: Vec<&str> = value.mime_type.split('/').collect(); + let kind = match mime_type[0] { + "audio" => sys_webrtc::ffi::MediaType::Audio, + "video" => sys_webrtc::ffi::MediaType::Video, + _ => panic!("invalid media type"), + }; + let name = mime_type[1].to_string(); + + Self { + name, + kind, + has_clock_rate: value.clock_rate.is_some(), + clock_rate: value.clock_rate.unwrap_or_default() as i32, + has_num_channels: value.channels.is_some(), + num_channels: value.channels.unwrap_or_default() as i32, + parameters: { + value + .sdp_fmtp_line + .map(|sdp_fmtp_line| { + sdp_fmtp_line + .split(';') + .map(|v| { + let key_value: Vec<&str> = v.split('=').collect(); + if key_value.len() == 2 { + sys_rp::ffi::StringKeyValue { + key: key_value[0].to_string(), + value: key_value[1].to_string(), + } + } else { + sys_rp::ffi::StringKeyValue { + key: "".to_string(), + value: key_value[0].to_string(), + } + } + }) + .collect() + }) + .unwrap_or_default() + }, + // Ignore + mime_type: String::default(), // !! + has_preferred_payload_type: false, + preferred_payload_type: 0, + has_max_ptime: false, + max_ptime: 0, + has_ptime: false, + ptime: 0, + rtcp_feedback: Vec::default(), + options: Vec::default(), + max_temporal_layer_extensions: 0, + max_spatial_layer_extensions: 0, + svc_multi_stream_support: false, + } + } +} diff --git a/livekit-webrtc/src/native/rtp_receiver.rs b/livekit-webrtc/src/native/rtp_receiver.rs new file mode 100644 index 0000000..478eb4f --- /dev/null +++ b/livekit-webrtc/src/native/rtp_receiver.rs @@ -0,0 +1,24 @@ +use super::media_stream::new_media_stream_track; +use crate::{media_stream::MediaStreamTrack, rtp_parameters::RtpParameters}; +use cxx::SharedPtr; +use webrtc_sys::rtp_receiver as sys_rr; + +#[derive(Clone)] +pub struct RtpReceiver { + pub(crate) sys_handle: SharedPtr, +} + +impl RtpReceiver { + pub fn track(&self) -> Option { + let track_handle = self.sys_handle.track(); + if track_handle.is_null() { + return None; + } + + Some(new_media_stream_track(track_handle)) + } + + pub fn parameters(&self) -> RtpParameters { + self.sys_handle.get_parameters().into() + } +} diff --git a/livekit-webrtc/src/native/rtp_sender.rs b/livekit-webrtc/src/native/rtp_sender.rs new file mode 100644 index 0000000..d806f6b --- /dev/null +++ b/livekit-webrtc/src/native/rtp_sender.rs @@ -0,0 +1,46 @@ +use super::media_stream::new_media_stream_track; +use crate::{ + media_stream::MediaStreamTrack, rtp_parameters::RtpParameters, RtcError, RtcErrorType, +}; +use cxx::SharedPtr; +use webrtc_sys::{rtc_error::ffi::RTCError, rtp_sender as sys_rs}; + +#[derive(Clone)] +pub struct RtpSender { + pub(crate) sys_handle: SharedPtr, +} + +impl RtpSender { + pub fn track(&self) -> Option { + let track_handle = self.sys_handle.track(); + if track_handle.is_null() { + return None; + } + + Some(new_media_stream_track(track_handle)) + } + + pub fn set_track(&self, track: Option) -> Result<(), RtcError> { + if !self + .sys_handle + .set_track(track.map_or(SharedPtr::null(), |t| t.sys_handle())) + { + return Err(RtcError { + error_type: RtcErrorType::InvalidState, + message: "Failed to set track".to_string(), + }); + } + + Ok(()) + } + + pub fn parameters(&self) -> RtpParameters { + self.sys_handle.get_parameters().into() + } + + pub fn set_parameters(&self, parameters: RtpParameters) -> Result<(), RtcError> { + self.sys_handle + .set_parameters(parameters.into()) + .map_err(|e| unsafe { RTCError::from(e.what()).into() }) + } +} diff --git a/livekit-webrtc/src/native/rtp_transceiver.rs b/livekit-webrtc/src/native/rtp_transceiver.rs new file mode 100644 index 0000000..a3119d5 --- /dev/null +++ b/livekit-webrtc/src/native/rtp_transceiver.rs @@ -0,0 +1,94 @@ +use crate::imp::rtp_receiver::RtpReceiver; +use crate::imp::rtp_sender::RtpSender; +use crate::rtp_parameters::RtpCodecCapability; +use crate::rtp_receiver; +use crate::rtp_sender; +use crate::rtp_transceiver::RtpTransceiverDirection; +use crate::rtp_transceiver::RtpTransceiverInit; +use crate::MediaType; +use crate::RtcError; +use cxx::SharedPtr; +use webrtc_sys::rtc_error as sys_err; +use webrtc_sys::rtp_transceiver as sys_rt; +use webrtc_sys::webrtc as sys_webrtc; + +impl From for RtpTransceiverDirection { + fn from(value: sys_webrtc::ffi::RtpTransceiverDirection) -> Self { + match value { + sys_webrtc::ffi::RtpTransceiverDirection::SendRecv => Self::SendRecv, + sys_webrtc::ffi::RtpTransceiverDirection::SendOnly => Self::SendOnly, + sys_webrtc::ffi::RtpTransceiverDirection::RecvOnly => Self::RecvOnly, + sys_webrtc::ffi::RtpTransceiverDirection::Inactive => Self::Inactive, + _ => panic!("unknown RtpTransceiverDirection"), + } + } +} + +impl From for sys_webrtc::ffi::RtpTransceiverDirection { + fn from(value: RtpTransceiverDirection) -> Self { + match value { + RtpTransceiverDirection::SendRecv => Self::SendRecv, + RtpTransceiverDirection::SendOnly => Self::SendOnly, + RtpTransceiverDirection::RecvOnly => Self::RecvOnly, + RtpTransceiverDirection::Inactive => Self::Inactive, + _ => panic!("unknown RtpTransceiverDirection"), + } + } +} + +impl From for sys_rt::ffi::RtpTransceiverInit { + fn from(value: RtpTransceiverInit) -> Self { + Self { + direction: value.direction.into(), + stream_ids: value.stream_ids, + send_encodings: value.send_encodings.into_iter().map(Into::into).collect(), + } + } +} + +#[derive(Clone)] +pub struct RtpTransceiver { + pub(crate) sys_handle: SharedPtr, +} + +impl RtpTransceiver { + pub fn mid(&self) -> Option { + self.sys_handle.mid().ok() + } + + pub fn current_direction(&self) -> Option { + self.sys_handle.current_direction().ok().map(Into::into) + } + + pub fn direction(&self) -> RtpTransceiverDirection { + self.sys_handle.direction().into() + } + + pub fn sender(&self) -> rtp_sender::RtpSender { + rtp_sender::RtpSender { + handle: RtpSender { + sys_handle: self.sys_handle.sender(), + }, + } + } + + pub fn receiver(&self) -> rtp_receiver::RtpReceiver { + rtp_receiver::RtpReceiver { + handle: RtpReceiver { + sys_handle: self.sys_handle.receiver(), + }, + } + } + + pub fn set_codec_preferences(&self, codecs: Vec) -> Result<(), RtcError> { + self.sys_handle + .set_codec_preferences(codecs.into_iter().map(Into::into).collect()) + .map_err(|e| unsafe { sys_err::ffi::RTCError::from(e.what()).into() }) + } + + pub fn stop(&self) -> Result<(), RtcError> { + self.sys_handle + .stop_standard() + .map_err(|e| unsafe { sys_err::ffi::RTCError::from(e.what()).into() }) + } +} diff --git a/livekit-webrtc/src/native/session_description.rs b/livekit-webrtc/src/native/session_description.rs new file mode 100644 index 0000000..00a150b --- /dev/null +++ b/livekit-webrtc/src/native/session_description.rs @@ -0,0 +1,72 @@ +use crate::session_description::{self, SdpParseError, SdpType}; +use cxx::UniquePtr; +use webrtc_sys::jsep as sys_jsep; + +impl From for SdpType { + fn from(sdp_type: sys_jsep::ffi::SdpType) -> Self { + match sdp_type { + sys_jsep::ffi::SdpType::Offer => SdpType::Offer, + sys_jsep::ffi::SdpType::PrAnswer => SdpType::PrAnswer, + sys_jsep::ffi::SdpType::Answer => SdpType::Answer, + sys_jsep::ffi::SdpType::Rollback => SdpType::Rollback, + _ => panic!("unknown SdpType"), + } + } +} + +impl From for sys_jsep::ffi::SdpType { + fn from(sdp_type: SdpType) -> Self { + match sdp_type { + SdpType::Offer => sys_jsep::ffi::SdpType::Offer, + SdpType::PrAnswer => sys_jsep::ffi::SdpType::PrAnswer, + SdpType::Answer => sys_jsep::ffi::SdpType::Answer, + SdpType::Rollback => sys_jsep::ffi::SdpType::Rollback, + } + } +} + +impl From for SdpParseError { + fn from(e: sys_jsep::ffi::SdpParseError) -> Self { + Self { + line: e.line, + description: e.description, + } + } +} + +pub struct SessionDescription { + pub(crate) sys_handle: UniquePtr, +} + +impl SessionDescription { + pub fn parse( + sdp: &str, + sdp_type: SdpType, + ) -> Result { + let res = sys_jsep::ffi::create_session_description(sdp_type.into(), sdp.to_owned()); + match res { + Ok(sys_handle) => Ok(session_description::SessionDescription { + handle: SessionDescription { sys_handle }, + }), + Err(e) => Err(unsafe { sys_jsep::ffi::SdpParseError::from(e.what()).into() }), + } + } + + pub fn sdp_type(&self) -> SdpType { + self.sys_handle.sdp_type().into() + } +} + +impl ToString for SessionDescription { + fn to_string(&self) -> String { + self.sys_handle.stringify() + } +} + +impl Clone for SessionDescription { + fn clone(&self) -> Self { + SessionDescription { + sys_handle: self.sys_handle.clone(), + } + } +} diff --git a/livekit-webrtc/src/native/video_frame.rs b/livekit-webrtc/src/native/video_frame.rs new file mode 100644 index 0000000..c4bbb92 --- /dev/null +++ b/livekit-webrtc/src/native/video_frame.rs @@ -0,0 +1,764 @@ +use super::yuv_helper::{self, ConvertError}; +use crate::video_frame::VideoRotation; +use crate::video_frame::{self as vf, VideoFormatType}; +use cxx::UniquePtr; +use std::slice; +use webrtc_sys::video_frame as vf_sys; +use webrtc_sys::video_frame_buffer as vfb_sys; + +/// We don't use vf::VideoFrameBuffer trait for the types inside this module to avoid confusion +/// because irectly using platform specific types is not valid (e.g user callback) +/// All the types inside this module are only used internally. For public types, see the top level video_frame.rs + +pub fn new_video_frame_buffer( + mut sys_handle: UniquePtr, +) -> Box { + unsafe { + match sys_handle.buffer_type().into() { + vfb_sys::ffi::VideoFrameBufferType::Native => Box::new(vf::native::NativeBuffer { + handle: NativeBuffer { sys_handle }, + }), + vfb_sys::ffi::VideoFrameBufferType::I420 => Box::new(vf::I420Buffer { + handle: I420Buffer { + sys_handle: sys_handle.pin_mut().get_i420(), + }, + }), + vfb_sys::ffi::VideoFrameBufferType::I420A => Box::new(vf::I420ABuffer { + handle: I420ABuffer { + sys_handle: sys_handle.pin_mut().get_i420a(), + }, + }), + vfb_sys::ffi::VideoFrameBufferType::I422 => Box::new(vf::I422Buffer { + handle: I422Buffer { + sys_handle: sys_handle.pin_mut().get_i422(), + }, + }), + vfb_sys::ffi::VideoFrameBufferType::I444 => Box::new(vf::I444Buffer { + handle: I444Buffer { + sys_handle: sys_handle.pin_mut().get_i444(), + }, + }), + vfb_sys::ffi::VideoFrameBufferType::I010 => Box::new(vf::I010Buffer { + handle: I010Buffer { + sys_handle: sys_handle.pin_mut().get_i010(), + }, + }), + vfb_sys::ffi::VideoFrameBufferType::NV12 => Box::new(vf::NV12Buffer { + handle: NV12Buffer { + sys_handle: sys_handle.pin_mut().get_nv12(), + }, + }), + _ => unreachable!(), + } + } +} + +impl From for VideoRotation { + fn from(rotation: vf_sys::ffi::VideoRotation) -> Self { + match rotation { + vf_sys::ffi::VideoRotation::VideoRotation0 => Self::VideoRotation0, + vf_sys::ffi::VideoRotation::VideoRotation90 => Self::VideoRotation90, + vf_sys::ffi::VideoRotation::VideoRotation180 => Self::VideoRotation180, + vf_sys::ffi::VideoRotation::VideoRotation270 => Self::VideoRotation270, + _ => panic!("invalid VideoRotation"), + } + } +} + +impl From for vf_sys::ffi::VideoRotation { + fn from(rotation: VideoRotation) -> Self { + match rotation { + VideoRotation::VideoRotation0 => Self::VideoRotation0, + VideoRotation::VideoRotation90 => Self::VideoRotation90, + VideoRotation::VideoRotation180 => Self::VideoRotation180, + VideoRotation::VideoRotation270 => Self::VideoRotation270, + } + } +} + +macro_rules! recursive_cast { + ($ptr:expr $(, $fnc:ident)*) => { + { + let ptr = $ptr; + $( + let ptr = vfb_sys::ffi::$fnc(ptr); + )* + ptr + } + }; +} + +pub struct NativeBuffer { + sys_handle: UniquePtr, +} + +pub struct I420Buffer { + sys_handle: UniquePtr, +} + +pub struct I420ABuffer { + sys_handle: UniquePtr, +} + +pub struct I422Buffer { + sys_handle: UniquePtr, +} + +pub struct I444Buffer { + sys_handle: UniquePtr, +} + +pub struct I010Buffer { + sys_handle: UniquePtr, +} + +pub struct NV12Buffer { + sys_handle: UniquePtr, +} + +macro_rules! impl_to_argb { + (I420Buffer [$($variant:ident: $fnc:ident),+], $format:ident, $self:ident, $dst:ident, $dst_stride:ident, $dst_width:ident, $dst_height:ident) => { + match $format { + $( + VideoFormatType::$variant => { + let (data_y, data_u, data_v) = $self.data(); + yuv_helper::$fnc( + data_y, + $self.stride_y(), + data_u, + $self.stride_u(), + data_v, + $self.stride_v(), + $dst, + $dst_stride, + $dst_width, + $dst_height, + ) + } + )+ + } + }; + (I420ABuffer) => { + todo!(); + } +} + +#[allow(unused_unsafe)] +impl NativeBuffer { + pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer { + &*self.sys_handle + } + + pub fn width(&self) -> i32 { + self.sys_handle.width() + } + + pub fn height(&self) -> i32 { + self.sys_handle.height() + } + + pub fn to_i420(&self) -> I420Buffer { + I420Buffer { + sys_handle: unsafe { self.sys_handle.to_i420() }, + } + } + + pub fn to_argb( + &self, + format: VideoFormatType, + dst: &mut [u8], + dst_stride: i32, + dst_width: i32, + dst_height: i32, + ) -> Result<(), ConvertError> { + self.to_i420() + .to_argb(format, dst, dst_stride, dst_width, dst_height) + } +} + +impl I420Buffer { + pub fn new(width: u32, height: u32) -> vf::I420Buffer { + vf::I420Buffer { + handle: I420Buffer { + sys_handle: vfb_sys::ffi::new_i420_buffer( + width.try_into().unwrap(), + height.try_into().unwrap(), + ), + }, + } + } + + pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer { + unsafe { &*recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb) } + } + + pub fn width(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb); + (*ptr).width() + } + } + + pub fn height(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb); + (*ptr).height() + } + } + + pub fn chroma_width(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv); + (*ptr).chroma_width() + } + } + + pub fn chroma_height(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv); + (*ptr).chroma_height() + } + } + + pub fn stride_y(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv); + (*ptr).stride_y() + } + } + + pub fn stride_u(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv); + (*ptr).stride_u() + } + } + + pub fn stride_v(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv); + (*ptr).stride_v() + } + } + + pub fn to_i420(&self) -> I420Buffer { + I420Buffer { + sys_handle: unsafe { + // We make a copy of the buffer because internally, when calling ToI420() + // if the buffer is of type I420, libwebrtc will reuse the same underlying pointer + // for the new created type + let copy = vfb_sys::ffi::copy_i420_buffer(&self.sys_handle); + let ptr = recursive_cast!(&*copy, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb); + (*ptr).to_i420() + }, + } + } + + pub fn to_argb( + &self, + format: VideoFormatType, + dst: &mut [u8], + dst_stride: i32, + dst_width: i32, + dst_height: i32, + ) -> Result<(), ConvertError> { + impl_to_argb!( + I420Buffer + [ + ARGB: i420_to_argb, + BGRA: i420_to_bgra, + ABGR: i420_to_abgr, + RGBA: i420_to_rgba + ], + format, self, dst, dst_stride, dst_width, dst_height + ) + } + + pub fn data(&self) -> (&[u8], &[u8], &[u8]) { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8); + let chroma_height = (self.height() + 1) / 2; + ( + slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize), + slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * chroma_height) as usize), + slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * chroma_height) as usize), + ) + } + } +} + +impl I420ABuffer { + pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer { + unsafe { &*recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb) } + } + + pub fn width(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb); + (*ptr).width() + } + } + + pub fn height(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb); + (*ptr).height() + } + } + + pub fn chroma_width(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv); + (*ptr).chroma_width() + } + } + + pub fn chroma_height(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv); + (*ptr).chroma_height() + } + } + + pub fn stride_y(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv); + (*ptr).stride_y() + } + } + + pub fn stride_u(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv); + (*ptr).stride_u() + } + } + + pub fn stride_v(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv); + (*ptr).stride_v() + } + } + + pub fn stride_a(&self) -> i32 { + self.sys_handle.stride_a() + } + + pub fn to_i420(&self) -> I420Buffer { + I420Buffer { + sys_handle: unsafe { + let ptr = + recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb); + (*ptr).to_i420() + }, + } + } + + pub fn to_argb( + &self, + format: VideoFormatType, + dst: &mut [u8], + dst_stride: i32, + dst_width: i32, + dst_height: i32, + ) -> Result<(), ConvertError> { + self.to_i420() + .to_argb(format, dst, dst_stride, dst_width, dst_height) + } + + pub fn data(&self) -> (&[u8], &[u8], &[u8], Option<&[u8]>) { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8); + let chroma_height = (self.height() + 1) / 2; + let data_a = self.sys_handle.data_a(); + let has_data_a = !data_a.is_null(); + ( + slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize), + slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * chroma_height) as usize), + slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * chroma_height) as usize), + has_data_a.then_some(slice::from_raw_parts( + data_a, + (self.stride_a() * self.height()) as usize, + )), + ) + } + } +} + +impl I422Buffer { + pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer { + unsafe { &*recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb) } + } + + pub fn width(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb); + (*ptr).width() + } + } + + pub fn height(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb); + (*ptr).height() + } + } + + pub fn chroma_width(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv); + (*ptr).chroma_width() + } + } + + pub fn chroma_height(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv); + (*ptr).chroma_height() + } + } + + pub fn stride_y(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv); + (*ptr).stride_y() + } + } + + pub fn stride_u(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv); + (*ptr).stride_u() + } + } + + pub fn stride_v(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv); + (*ptr).stride_v() + } + } + + pub fn to_i420(&self) -> I420Buffer { + I420Buffer { + sys_handle: unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb); + (*ptr).to_i420() + }, + } + } + + pub fn to_argb( + &self, + format: VideoFormatType, + dst: &mut [u8], + dst_stride: i32, + dst_width: i32, + dst_height: i32, + ) -> Result<(), ConvertError> { + self.to_i420() + .to_argb(format, dst, dst_stride, dst_width, dst_height) + } + + pub fn data(&self) -> (&[u8], &[u8], &[u8]) { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8); + ( + slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize), + slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * self.height()) as usize), + slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * self.height()) as usize), + ) + } + } +} +impl I444Buffer { + pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer { + unsafe { &*recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb) } + } + + pub fn width(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb); + (*ptr).width() + } + } + + pub fn height(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb); + (*ptr).height() + } + } + + pub fn chroma_width(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv); + (*ptr).chroma_width() + } + } + + pub fn chroma_height(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv); + (*ptr).chroma_height() + } + } + + pub fn stride_y(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv); + (*ptr).stride_y() + } + } + + pub fn stride_u(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv); + (*ptr).stride_u() + } + } + + pub fn stride_v(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv); + (*ptr).stride_v() + } + } + + pub fn to_i420(&self) -> I420Buffer { + I420Buffer { + sys_handle: unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb); + (*ptr).to_i420() + }, + } + } + + pub fn to_argb( + &self, + format: VideoFormatType, + dst: &mut [u8], + dst_stride: i32, + dst_width: i32, + dst_height: i32, + ) -> Result<(), ConvertError> { + self.to_i420() + .to_argb(format, dst, dst_stride, dst_width, dst_height) + } + + pub fn data(&self) -> (&[u8], &[u8], &[u8]) { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8); + ( + slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize), + slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * self.height()) as usize), + slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * self.height()) as usize), + ) + } + } +} + +impl I010Buffer { + pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer { + unsafe { &*recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb) } + } + + pub fn width(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb); + (*ptr).width() + } + } + + pub fn height(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb); + (*ptr).height() + } + } + + pub fn chroma_width(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv); + (*ptr).chroma_width() + } + } + + pub fn chroma_height(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv); + (*ptr).chroma_height() + } + } + + pub fn stride_y(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv); + (*ptr).stride_y() + } + } + + pub fn stride_u(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv); + (*ptr).stride_u() + } + } + + pub fn stride_v(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv); + (*ptr).stride_v() + } + } + + pub fn to_i420(&self) -> I420Buffer { + I420Buffer { + sys_handle: unsafe { + let ptr = + recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb); + (*ptr).to_i420() + }, + } + } + + pub fn to_argb( + &self, + format: VideoFormatType, + dst: &mut [u8], + dst_stride: i32, + dst_width: i32, + dst_height: i32, + ) -> Result<(), ConvertError> { + self.to_i420() + .to_argb(format, dst, dst_stride, dst_width, dst_height) + } + + pub fn data(&self) -> (&[u16], &[u16], &[u16]) { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b); + let chroma_height = (self.height() + 1) / 2; + ( + slice::from_raw_parts( + (*ptr).data_y(), + (self.stride_y() * self.height()) as usize / 2, + ), + slice::from_raw_parts( + (*ptr).data_u(), + (self.stride_u() * chroma_height) as usize / 2, + ), + slice::from_raw_parts( + (*ptr).data_v(), + (self.stride_v() * chroma_height) as usize / 2, + ), + ) + } + } +} + +impl NV12Buffer { + pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer { + unsafe { + &*recursive_cast!( + &*self.sys_handle, + nv12_to_biyuv8, + biyuv8_to_biyuv, + biyuv_to_vfb + ) + } + } + + pub fn width(&self) -> i32 { + unsafe { + let ptr = recursive_cast!( + &*self.sys_handle, + nv12_to_biyuv8, + biyuv8_to_biyuv, + biyuv_to_vfb + ); + (*ptr).width() + } + } + + pub fn height(&self) -> i32 { + unsafe { + let ptr = recursive_cast!( + &*self.sys_handle, + nv12_to_biyuv8, + biyuv8_to_biyuv, + biyuv_to_vfb + ); + (*ptr).height() + } + } + + pub fn chroma_width(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv); + (*ptr).chroma_width() + } + } + + pub fn chroma_height(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv); + (*ptr).chroma_height() + } + } + + pub fn stride_y(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv); + (*ptr).stride_y() + } + } + + pub fn stride_uv(&self) -> i32 { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv); + (*ptr).stride_uv() + } + } + + pub fn to_i420(&self) -> I420Buffer { + I420Buffer { + sys_handle: unsafe { + let ptr = recursive_cast!( + &*self.sys_handle, + nv12_to_biyuv8, + biyuv8_to_biyuv, + biyuv_to_vfb + ); + (*ptr).to_i420() + }, + } + } + + pub fn to_argb( + &self, + format: VideoFormatType, + dst: &mut [u8], + dst_stride: i32, + dst_width: i32, + dst_height: i32, + ) -> Result<(), ConvertError> { + self.to_i420() + .to_argb(format, dst, dst_stride, dst_width, dst_height) + } + + pub fn data(&self) -> (&[u8], &[u8]) { + unsafe { + let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8); + let chroma_height = (self.height() + 1) / 2; + + ( + slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize), + slice::from_raw_parts( + (*ptr).data_uv(), + (self.stride_uv() * chroma_height) as usize, + ), + ) + } + } +} diff --git a/livekit-webrtc/src/native/video_source.rs b/livekit-webrtc/src/native/video_source.rs new file mode 100644 index 0000000..a6c2d86 --- /dev/null +++ b/livekit-webrtc/src/native/video_source.rs @@ -0,0 +1,34 @@ +use crate::video_frame::{VideoFrame, VideoFrameBuffer}; +use cxx::SharedPtr; +use webrtc_sys::media_stream as ms_sys; +use webrtc_sys::video_frame as vf_sys; + +#[derive(Clone)] +pub struct NativeVideoSource { + sys_handle: SharedPtr, +} + +impl Default for NativeVideoSource { + fn default() -> Self { + Self { + sys_handle: ms_sys::ffi::new_adapted_video_track_source(), + } + } +} + +impl NativeVideoSource { + pub fn sys_handle(&self) -> SharedPtr { + self.sys_handle.clone() + } + + pub fn capture_frame(&self, frame: &VideoFrame) { + let mut builder = vf_sys::ffi::new_video_frame_builder(); + builder.pin_mut().set_rotation(frame.rotation.into()); + builder + .pin_mut() + .set_video_frame_buffer(frame.buffer.sys_handle()); + + let frame = builder.pin_mut().build(); + self.sys_handle.on_captured_frame(&frame); + } +} diff --git a/livekit-webrtc/src/native/video_stream.rs b/livekit-webrtc/src/native/video_stream.rs new file mode 100644 index 0000000..08ef3c4 --- /dev/null +++ b/livekit-webrtc/src/native/video_stream.rs @@ -0,0 +1,84 @@ +use super::video_frame::new_video_frame_buffer; +use crate::media_stream::RtcVideoTrack; +use crate::video_frame::{BoxVideoFrame, VideoFrame}; +use cxx::UniquePtr; +use futures::stream::Stream; +use std::pin::Pin; +use std::task::{Context, Poll}; +use tokio::sync::mpsc; +use webrtc_sys::media_stream as sys_ms; + +pub struct NativeVideoStream { + native_observer: UniquePtr, + _observer: Box, + video_track: RtcVideoTrack, + frame_rx: mpsc::UnboundedReceiver, +} + +impl NativeVideoStream { + pub fn new(video_track: RtcVideoTrack) -> Self { + let (frame_tx, frame_rx) = mpsc::unbounded_channel(); + let mut observer = Box::new(VideoTrackObserver { frame_tx }); + let mut native_observer = unsafe { + sys_ms::ffi::new_native_video_frame_sink(Box::new(sys_ms::VideoFrameSinkWrapper::new( + &mut *observer, + ))) + }; + + unsafe { + sys_ms::ffi::media_to_video(video_track.sys_handle()) + .add_sink(native_observer.pin_mut()); + } + + Self { + native_observer, + _observer: observer, + video_track, + frame_rx, + } + } + + pub fn track(&self) -> RtcVideoTrack { + self.video_track.clone() + } + + pub fn close(&mut self) { + self.frame_rx.close(); + unsafe { + sys_ms::ffi::media_to_video(self.video_track.sys_handle()) + .remove_sink(self.native_observer.pin_mut()); + } + } +} + +impl Drop for NativeVideoStream { + fn drop(&mut self) { + self.close(); + } +} + +impl Stream for NativeVideoStream { + type Item = BoxVideoFrame; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + self.frame_rx.poll_recv(cx) + } +} + +struct VideoTrackObserver { + frame_tx: mpsc::UnboundedSender, +} + +impl sys_ms::VideoFrameSink for VideoTrackObserver { + fn on_frame(&self, frame: UniquePtr) { + let _ = self.frame_tx.send(VideoFrame { + rotation: frame.rotation().into(), + timestamp: frame.timestamp_us(), + buffer: new_video_frame_buffer(unsafe { frame.video_frame_buffer() }), + }); + } + + fn on_discarded_frame(&self) {} + + fn on_constraints_changed(&self, _constraints: sys_ms::ffi::VideoTrackSourceConstraints) {} +} diff --git a/livekit-webrtc/src/native/yuv_helper.rs b/livekit-webrtc/src/native/yuv_helper.rs new file mode 100644 index 0000000..df9f556 --- /dev/null +++ b/livekit-webrtc/src/native/yuv_helper.rs @@ -0,0 +1,183 @@ +use thiserror::Error; +use webrtc_sys::yuv_helper as yuv_sys; + +#[derive(Error, Debug)] +pub enum ConvertError { + #[error("conversion failed: {0}")] + Convert(&'static str), +} + +#[inline] +fn argb_assert_safety( + src: &[u8], + src_stride: i32, + _width: i32, + height: i32, +) -> Result<(), ConvertError> { + let min = (src_stride * height) as usize; + + if src.len() < min { + return Err(ConvertError::Convert("dst isn't large enough")); + } + + Ok(()) +} + +#[inline] +fn i420_assert_safety( + src_y: &[u8], + src_stride_y: i32, + src_u: &[u8], + src_stride_u: i32, + src_v: &[u8], + src_stride_v: i32, + _width: i32, + height: i32, +) -> Result<(), ConvertError> { + let chroma_height = (height + 1) / 2; + let min_y = (src_stride_y * height) as usize; + let min_u = (src_stride_u * chroma_height) as usize; + let min_v = (src_stride_v * chroma_height) as usize; + + if src_y.len() < min_y { + return Err(ConvertError::Convert("src_y isn't large enough")); + } + + if src_u.len() < min_u { + return Err(ConvertError::Convert("src_u isn't large enough")); + } + + if src_v.len() < min_v { + return Err(ConvertError::Convert("src_v isn't large enough")); + } + + Ok(()) +} + +macro_rules! i420_to_x { + ($x:ident) => { + pub fn $x( + src_y: &[u8], + src_stride_y: i32, + src_u: &[u8], + src_stride_u: i32, + src_v: &[u8], + src_stride_v: i32, + dst: &mut [u8], + dst_stride: i32, + width: i32, + height: i32, + ) -> Result<(), ConvertError> { + argb_assert_safety(dst, dst_stride, width, height)?; + i420_assert_safety( + src_y, + src_stride_y, + src_u, + src_stride_u, + src_v, + src_stride_v, + width, + height, + )?; + + unsafe { + yuv_sys::ffi::$x( + src_y.as_ptr(), + src_stride_y, + src_u.as_ptr(), + src_stride_u, + src_v.as_ptr(), + src_stride_v, + dst.as_mut_ptr(), + dst_stride, + width, + height, + ) + .unwrap(); + } + + Ok(()) + } + }; +} + +macro_rules! x_to_i420 { + ($x:ident) => { + pub fn $x( + src_argb: &[u8], + src_stride_argb: i32, + dst_y: &mut [u8], + dst_stride_y: i32, + dst_u: &mut [u8], + dst_stride_u: i32, + dst_v: &mut [u8], + dst_stride_v: i32, + width: i32, + height: i32, + ) -> Result<(), ConvertError> { + argb_assert_safety(src_argb, src_stride_argb, width, height)?; + i420_assert_safety( + dst_y, + dst_stride_y, + dst_u, + dst_stride_u, + dst_v, + dst_stride_v, + width, + height, + )?; + + unsafe { + yuv_sys::ffi::$x( + src_argb.as_ptr(), + src_stride_argb, + dst_y.as_mut_ptr(), + dst_stride_y, + dst_u.as_mut_ptr(), + dst_stride_u, + dst_v.as_mut_ptr(), + dst_stride_v, + width, + height, + ) + .unwrap(); + } + + Ok(()) + } + }; +} + +pub fn argb_to_rgb24( + src_argb: &[u8], + src_stride_argb: i32, + dst_rgb24: &mut [u8], + dst_stride_rgb24: i32, + width: i32, + height: i32, +) -> Result<(), ConvertError> { + argb_assert_safety(src_argb, src_stride_argb, width, height)?; + argb_assert_safety(dst_rgb24, dst_stride_rgb24, width, height)?; + + unsafe { + yuv_sys::ffi::argb_to_rgb24( + src_argb.as_ptr(), + src_stride_argb, + dst_rgb24.as_mut_ptr(), + dst_stride_rgb24, + width, + height, + ) + .unwrap(); + } + + Ok(()) +} + +x_to_i420!(argb_to_i420); +x_to_i420!(abgr_to_i420); + +i420_to_x!(i420_to_argb); +i420_to_x!(i420_to_bgra); +i420_to_x!(i420_to_abgr); +i420_to_x!(i420_to_rgba); diff --git a/livekit-webrtc/src/peer_connection.rs b/livekit-webrtc/src/peer_connection.rs index 796bc15..aa6ed72 100644 --- a/livekit-webrtc/src/peer_connection.rs +++ b/livekit-webrtc/src/peer_connection.rs @@ -1,737 +1,240 @@ -use crate::prelude::*; -use cxx::{SharedPtr, UniquePtr}; -use log::trace; -use std::fmt::{Debug, Formatter}; -use std::mem::ManuallyDrop; -use std::sync::{Arc, Mutex}; -use tokio::sync::{mpsc, oneshot}; +use std::fmt::Debug; -use webrtc_sys::candidate as sys_ca; -use webrtc_sys::data_channel as sys_dc; -use webrtc_sys::jsep as sys_jsep; -use webrtc_sys::media_stream as sys_ms; -use webrtc_sys::peer_connection as sys_pc; -use webrtc_sys::rtp_receiver as sys_rr; -use webrtc_sys::rtp_sender as sys_rs; -use webrtc_sys::rtp_transceiver as sys_rt; +use crate::data_channel::{DataChannel, DataChannelInit}; +use crate::ice_candidate::IceCandidate; +use crate::imp::peer_connection as imp_pc; +use crate::media_stream::{MediaStream, MediaStreamTrack}; +use crate::rtp_receiver::RtpReceiver; +use crate::rtp_sender::RtpSender; +use crate::rtp_transceiver::{RtpTransceiver, RtpTransceiverInit}; +use crate::session_description::SessionDescription; +use crate::{MediaType, RtcError}; -pub use webrtc_sys::peer_connection::ffi::IceConnectionState; -pub use webrtc_sys::peer_connection::ffi::IceGatheringState; -pub use webrtc_sys::peer_connection::ffi::PeerConnectionState; -pub use webrtc_sys::peer_connection::ffi::RTCOfferAnswerOptions; -pub use webrtc_sys::peer_connection::ffi::SignalingState; - -pub struct PeerConnection { - cxx_handle: UniquePtr, - observer: Box, - - // Keep alive for C++ - #[allow(unused)] - native_observer: UniquePtr, +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum PeerConnectionState { + New, + Connecting, + Connected, + Disconnected, + Failed, + Closed, } -impl Debug for PeerConnection { - fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - f.debug_struct("PeerConnection") - .field("signaling_state", &self.signaling_state()) - .field("ice_connection_state", &self.ice_connection_state()) - .field("ice_gathering_state", &self.ice_gathering_state()) - .finish() - } +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum IceConnectionState { + New, + Checking, + Connected, + Completed, + Failed, + Disconnected, + Closed, + Max, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum IceGatheringState { + New, + Gathering, + Complete, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum SignalingState { + Stable, + HaveLocalOffer, + HaveLocalPrAnswer, + HaveRemoteOffer, + HaveRemotePrAnswer, + Closed, +} + +#[derive(Debug, Clone, Default)] +pub struct OfferOptions { + pub ice_restart: bool, + pub offer_to_receive_audio: bool, + pub offer_to_receive_video: bool, +} + +#[derive(Debug, Clone, Default)] +pub struct AnswerOptions {} + +#[derive(Debug, Clone)] +pub struct IceCandidateError { + pub address: String, + pub port: i32, + pub url: String, + pub error_code: i32, + pub error_text: String, +} + +#[derive(Debug, Clone)] +pub struct TrackEvent { + pub receiver: RtpReceiver, + pub streams: Vec, + pub track: MediaStreamTrack, + pub transceiver: RtpTransceiver, +} + +pub type OnConnectionChange = Box; +pub type OnDataChannel = Box; +pub type OnIceCandidate = Box; +pub type OnIceCandidateError = Box; +pub type OnIceConnectionChange = Box; +pub type OnIceGatheringChange = Box; +pub type OnNegotiationNeeded = Box; +pub type OnSignalingChange = Box; +pub type OnTrack = Box; + +#[derive(Clone)] +pub struct PeerConnection { + pub(crate) handle: imp_pc::PeerConnection, } impl PeerConnection { - pub(crate) fn new( - cxx_handle: UniquePtr, - observer: Box, - native_observer: UniquePtr, - ) -> Self { - Self { - cxx_handle, - observer, - native_observer, - } - } - - fn create_sdp_observer() -> ( - UniquePtr, - mpsc::Receiver>, - ) { - let (tx, rx) = mpsc::channel(1); - let wrapper = sys_jsep::CreateSdpObserverWrapper { - on_success: ManuallyDrop::new(Box::new({ - let tx = tx.clone(); - move |session_description| { - let _ = tx.blocking_send(Ok(SessionDescription::new(session_description))); - } - })), - on_failure: ManuallyDrop::new(Box::new(move |error| { - let _ = tx.blocking_send(Err(error)); - })), - }; - - ( - sys_jsep::ffi::create_native_create_sdp_observer(Box::new(wrapper)), - rx, - ) - } - pub async fn create_offer( &self, - options: RTCOfferAnswerOptions, - ) -> Result { - let (mut native_wrapper, mut rx) = Self::create_sdp_observer(); - - unsafe { - self.cxx_handle - .create_offer(native_wrapper.pin_mut(), options); - } - - rx.recv().await.unwrap() + options: OfferOptions, + ) -> Result { + self.handle.create_offer(options).await } pub async fn create_answer( &self, - options: RTCOfferAnswerOptions, - ) -> Result { - let (mut native_wrapper, mut rx) = Self::create_sdp_observer(); - - unsafe { - self.cxx_handle - .create_answer(native_wrapper.pin_mut(), options); - } - - rx.recv().await.unwrap() + options: AnswerOptions, + ) -> Result { + self.handle.create_answer(options).await } - pub async fn set_local_description(&self, desc: SessionDescription) -> Result<(), RTCError> { - let (tx, rx) = oneshot::channel(); - let wrapper = - sys_jsep::SetLocalSdpObserverWrapper(ManuallyDrop::new(Box::new(move |error| { - let _ = tx.send(if error.ok() { Ok(()) } else { Err(error) }); - }))); - let mut native_wrapper = - sys_jsep::ffi::create_native_set_local_sdp_observer(Box::new(wrapper)); - - unsafe { - self.cxx_handle - .set_local_description(desc.release(), native_wrapper.pin_mut()); - } - - rx.await.unwrap() + pub async fn set_local_description(&self, desc: SessionDescription) -> Result<(), RtcError> { + self.handle.set_local_description(desc).await } - pub async fn set_remote_description(&self, desc: SessionDescription) -> Result<(), RTCError> { - let (tx, rx) = oneshot::channel(); - let wrapper = - sys_jsep::SetRemoteSdpObserverWrapper(ManuallyDrop::new(Box::new(move |error| { - let _ = tx.send(if error.ok() { Ok(()) } else { Err(error) }); - }))); - let mut native_wrapper = - sys_jsep::ffi::create_native_set_remote_sdp_observer(Box::new(wrapper)); - - unsafe { - self.cxx_handle - .set_remote_description(desc.release(), native_wrapper.pin_mut()); - } - - rx.await.unwrap() + pub async fn set_remote_description(&self, desc: SessionDescription) -> Result<(), RtcError> { + self.handle.set_remote_description(desc).await } - pub fn add_track( - &self, - track: MediaStreamTrackHandle, - stream_ids: &Vec, - ) -> Result { - let res = self.cxx_handle.add_track(track.cxx_handle(), stream_ids); - match res { - Ok(cxx_handle) => Ok(RtpSender::new(cxx_handle)), - Err(e) => unsafe { Err(RTCError::from(e.what())) }, - } - } - - pub fn remove_track(&self, sender: RtpSender) -> Result<(), RTCError> { - self.cxx_handle - .remove_track(sender.cxx_handle()) - .map_err(|e| unsafe { RTCError::from(e.what()) }) - } - - pub fn add_transceiver( - &self, - track: MediaStreamTrackHandle, - init: RtpTransceiverInit, - ) -> Result { - let res = self - .cxx_handle - .add_transceiver(track.cxx_handle(), init.into()); - - match res { - Ok(cxx_handle) => Ok(RtpTransceiver::new(cxx_handle)), - Err(e) => unsafe { Err(RTCError::from(e.what())) }, - } - } - - pub fn add_transceiver_for_media( - &self, - media_type: MediaType, - init: RtpTransceiverInit, - ) -> Result { - let res = self - .cxx_handle - .add_transceiver_for_media(media_type, init.into()); - - match res { - Ok(cxx_handle) => Ok(RtpTransceiver::new(cxx_handle)), - Err(e) => unsafe { Err(RTCError::from(e.what())) }, - } - } - - pub fn senders(&self) -> Vec { - self.cxx_handle - .get_senders() - .into_iter() - .map(|sender| RtpSender::new(sender.ptr)) - .collect() - } - - pub fn receivers(&self) -> Vec { - self.cxx_handle - .get_receivers() - .into_iter() - .map(|receiver| RtpReceiver::new(receiver.ptr)) - .collect() - } - - pub fn transceivers(&self) -> Vec { - self.cxx_handle - .get_transceivers() - .into_iter() - .map(|transceiver| RtpTransceiver::new(transceiver.ptr)) - .collect() + pub async fn add_ice_candidate(&self, candidate: IceCandidate) -> Result<(), RtcError> { + self.handle.add_ice_candidate(candidate).await } pub fn create_data_channel( &self, label: &str, init: DataChannelInit, - ) -> Result { - let native_init = sys_dc::ffi::create_data_channel_init(init.into()); - let res = self - .cxx_handle - .create_data_channel(label.to_string(), native_init); - - match res { - Ok(cxx_handle) => Ok(DataChannel::new(cxx_handle)), - Err(e) => Err(unsafe { RTCError::from(e.what()) }), - } + ) -> Result { + self.handle.create_data_channel(label, init) } - // TODO(theomonnom) Use IceCandidateInit instead of IceCandidate - pub async fn add_ice_candidate(&self, candidate: IceCandidate) -> Result<(), RTCError> { - let (tx, rx) = oneshot::channel(); - let observer = - sys_pc::AddIceCandidateObserverWrapper(ManuallyDrop::new(Box::new(|error| { - let _ = tx.send(if error.ok() { Ok(()) } else { Err(error) }); - }))); - - let mut native_observer = - sys_pc::ffi::create_native_add_ice_candidate_observer(Box::new(observer)); - self.cxx_handle - .add_ice_candidate(candidate.release(), native_observer.pin_mut()); - - rx.await.unwrap() + pub fn add_track>( + &self, + track: MediaStreamTrack, + streams_ids: &[T], + ) -> Result { + self.handle.add_track(track, streams_ids) } - pub fn local_description(&self) -> Option { - let local_description = self.cxx_handle.local_description(); - if local_description.is_null() { - None - } else { - Some(SessionDescription::new(local_description)) - } + pub fn remove_track(&self, sender: RtpSender) -> Result<(), RtcError> { + self.handle.remove_track(sender) } - pub fn remote_description(&self) -> Option { - let remote_description = self.cxx_handle.remote_description(); - if remote_description.is_null() { - None - } else { - Some(SessionDescription::new(remote_description)) - } + pub fn add_transceiver( + &self, + track: MediaStreamTrack, + init: RtpTransceiverInit, + ) -> Result { + self.handle.add_transceiver(track, init) } - pub fn signaling_state(&self) -> SignalingState { - self.cxx_handle.signaling_state() + pub fn add_transceiver_for_media( + &self, + media_type: MediaType, + init: RtpTransceiverInit, + ) -> Result { + self.handle.add_transceiver_for_media(media_type, init) + } + pub fn close(&self) { + self.handle.close() } - pub fn ice_gathering_state(&self) -> IceGatheringState { - self.cxx_handle.ice_gathering_state() + pub fn connection_state(&self) -> PeerConnectionState { + self.handle.connection_state() } pub fn ice_connection_state(&self) -> IceConnectionState { - self.cxx_handle.ice_connection_state() + self.handle.ice_connection_state() } - pub fn close(&mut self) { - self.cxx_handle.pin_mut().close(); + pub fn ice_gathering_state(&self) -> IceGatheringState { + self.handle.ice_gathering_state() } - pub fn on_signaling_change(&mut self, handler: OnSignalingChangeHandler) { - *self.observer.on_signaling_change_handler.lock().unwrap() = Some(handler); + pub fn signaling_state(&self) -> SignalingState { + self.handle.signaling_state() } - pub fn on_add_stream(&mut self, handler: OnAddStreamHandler) { - *self.observer.on_add_stream_handler.lock().unwrap() = Some(handler); + pub fn current_local_description(&self) -> Option { + self.handle.current_local_description() } - pub fn on_remove_stream(&mut self, handler: OnRemoveStreamHandler) { - *self.observer.on_remove_stream_handler.lock().unwrap() = Some(handler); + pub fn current_remote_description(&self) -> Option { + self.handle.current_remote_description() } - pub fn on_data_channel(&mut self, handler: OnDataChannelHandler) { - *self.observer.on_data_channel_handler.lock().unwrap() = Some(handler); + pub fn senders(&self) -> Vec { + self.handle.senders() } - pub fn on_renegotiation_needed(&mut self, handler: OnRenegotiationNeededHandler) { - *self - .observer - .on_renegotiation_needed_handler - .lock() - .unwrap() = Some(handler); + pub fn receivers(&self) -> Vec { + self.handle.receivers() } - pub fn on_ice_connection_change(&mut self, handler: OnIceConnectionChangeHandler) { - *self - .observer - .on_ice_connection_change_handler - .lock() - .unwrap() = Some(handler); + pub fn transceivers(&self) -> Vec { + self.handle.transceivers() } - pub fn on_standardized_ice_connection_change( - &mut self, - handler: OnStandardizedIceConnectionChangeHandler, - ) { - *self - .observer - .on_standardized_ice_connection_change_handler - .lock() - .unwrap() = Some(handler); + pub fn on_connection_state_change(&self, f: Option) { + self.handle.on_connection_state_change(f) } - pub fn on_connection_change(&mut self, handler: OnConnectionChangeHandler) { - *self.observer.on_connection_change_handler.lock().unwrap() = Some(handler); + pub fn on_data_channel(&self, f: Option) { + self.handle.on_data_channel(f) } - pub fn on_ice_gathering_change(&mut self, handler: OnIceGatheringChangeHandler) { - *self - .observer - .on_ice_gathering_change_handler - .lock() - .unwrap() = Some(handler); + pub fn on_ice_candidate(&self, f: Option) { + self.handle.on_ice_candidate(f) } - pub fn on_ice_candidate(&mut self, handler: OnIceCandidateHandler) { - *self.observer.on_ice_candidate_handler.lock().unwrap() = Some(handler); + pub fn on_ice_candidate_error(&self, f: Option) { + self.handle.on_ice_candidate_error(f) } - pub fn on_ice_candidate_error(&mut self, handler: OnIceCandidateErrorHandler) { - *self.observer.on_ice_candidate_error_handler.lock().unwrap() = Some(handler); + pub fn on_ice_connection_state_change(&self, f: Option) { + self.handle.on_ice_connection_state_change(f) } - pub fn on_ice_candidates_removed(&mut self, handler: OnIceCandidatesRemovedHandler) { - *self - .observer - .on_ice_candidates_removed_handler - .lock() - .unwrap() = Some(handler); + pub fn on_ice_gathering_state_change(&self, f: Option) { + self.handle.on_ice_gathering_state_change(f) } - pub fn on_ice_connection_receiving_change( - &mut self, - handler: OnIceConnectionReceivingChangeHandler, - ) { - *self - .observer - .on_ice_connection_receiving_change_handler - .lock() - .unwrap() = Some(handler); + pub fn on_negotiation_needed(&self, f: Option) { + self.handle.on_negotiation_needed(f) } - pub fn on_ice_selected_candidate_pair_changed( - &mut self, - handler: OnIceSelectedCandidatePairChangedHandler, - ) { - *self - .observer - .on_ice_selected_candidate_pair_changed_handler - .lock() - .unwrap() = Some(handler); + pub fn on_signaling_state_change(&self, f: Option) { + self.handle.on_signaling_state_change(f) } - pub fn on_add_track(&mut self, handler: OnAddTrackHandler) { - *self.observer.on_add_track_handler.lock().unwrap() = Some(handler); - } - - pub fn on_track(&mut self, handler: OnTrackHandler) { - *self.observer.on_track_handler.lock().unwrap() = Some(handler); - } - - pub fn on_remove_track(&mut self, handler: OnRemoveTrackHandler) { - *self.observer.on_remove_track_handler.lock().unwrap() = Some(handler); - } - - pub fn on_interesting_usage(&mut self, handler: OnInterestingUsageHandler) { - *self.observer.on_interesting_usage_handler.lock().unwrap() = Some(handler); + pub fn on_track(&self, f: Option) { + self.handle.on_track(f) } } -// TODO(theomonnom) Should we return futures? -pub type OnSignalingChangeHandler = Box; -pub type OnAddStreamHandler = Box; -pub type OnRemoveStreamHandler = Box; -pub type OnDataChannelHandler = Box; -pub type OnRenegotiationNeededHandler = Box; -pub type OnNegotiationNeededEventHandler = Box; -pub type OnIceConnectionChangeHandler = Box; -pub type OnStandardizedIceConnectionChangeHandler = - Box; -pub type OnConnectionChangeHandler = Box; -pub type OnIceGatheringChangeHandler = Box; -pub type OnIceCandidateHandler = Box; -pub type OnIceCandidateErrorHandler = - Box; -pub type OnIceCandidatesRemovedHandler = Box) + Send + Sync>; -pub type OnIceConnectionReceivingChangeHandler = Box; -pub type OnIceSelectedCandidatePairChangedHandler = - Box; -pub type OnAddTrackHandler = Box) + Send + Sync>; -pub type OnTrackHandler = Box; -pub type OnRemoveTrackHandler = Box; -pub type OnInterestingUsageHandler = Box; - -pub(crate) struct InternalObserver { - on_signaling_change_handler: Arc>>, - on_add_stream_handler: Arc>>, - on_remove_stream_handler: Arc>>, - on_data_channel_handler: Arc>>, - on_renegotiation_needed_handler: Arc>>, - on_negotiation_needed_event_handler: Arc>>, - on_ice_connection_change_handler: Arc>>, - on_standardized_ice_connection_change_handler: - Arc>>, - on_connection_change_handler: Arc>>, - on_ice_gathering_change_handler: Arc>>, - on_ice_candidate_handler: Arc>>, - on_ice_candidate_error_handler: Arc>>, - on_ice_candidates_removed_handler: Arc>>, - on_ice_connection_receiving_change_handler: - Arc>>, - on_ice_selected_candidate_pair_changed_handler: - Arc>>, - on_add_track_handler: Arc>>, - on_track_handler: Arc>>, - on_remove_track_handler: Arc>>, - on_interesting_usage_handler: Arc>>, -} - -impl Default for InternalObserver { - fn default() -> Self { - Self { - on_signaling_change_handler: Arc::new(Default::default()), - on_add_stream_handler: Arc::new(Default::default()), - on_remove_stream_handler: Arc::new(Default::default()), - on_data_channel_handler: Arc::new(Default::default()), - on_renegotiation_needed_handler: Arc::new(Default::default()), - on_negotiation_needed_event_handler: Arc::new(Default::default()), - on_ice_connection_change_handler: Arc::new(Default::default()), - on_standardized_ice_connection_change_handler: Arc::new(Default::default()), - on_connection_change_handler: Arc::new(Default::default()), - on_ice_gathering_change_handler: Arc::new(Default::default()), - on_ice_candidate_handler: Arc::new(Default::default()), - on_ice_candidate_error_handler: Arc::new(Default::default()), - on_ice_candidates_removed_handler: Arc::new(Default::default()), - on_ice_connection_receiving_change_handler: Arc::new(Default::default()), - on_ice_selected_candidate_pair_changed_handler: Arc::new(Default::default()), - on_add_track_handler: Arc::new(Default::default()), - on_track_handler: Arc::new(Default::default()), - on_remove_track_handler: Arc::new(Default::default()), - on_interesting_usage_handler: Arc::new(Default::default()), - } - } -} - -// Observers are being called on the Signaling Thread -impl sys_pc::PeerConnectionObserver for InternalObserver { - fn on_signaling_change(&self, new_state: SignalingState) { - trace!("on_signaling_change, {:?}", new_state); - let mut handler = self.on_signaling_change_handler.lock().unwrap(); - if let Some(f) = handler.as_mut() { - f(new_state); - } - } - - fn on_add_stream(&self, stream: SharedPtr) { - trace!("on_add_stream"); - let mut handler = self.on_add_stream_handler.lock().unwrap(); - if let Some(f) = handler.as_mut() { - // TODO(theomonnom) - } - } - - fn on_remove_stream(&self, stream: SharedPtr) { - trace!("on_remove_stream"); - let mut handler = self.on_remove_stream_handler.lock().unwrap(); - if let Some(f) = handler.as_mut() { - // TODO(theomonnom) - } - } - - fn on_data_channel(&self, data_channel: UniquePtr) { - trace!("on_data_channel"); - let mut handler = self.on_data_channel_handler.lock().unwrap(); - if let Some(f) = handler.as_mut() { - f(DataChannel::new(data_channel)); - } - } - - fn on_renegotiation_needed(&self) { - trace!("on_renegotiation_needed"); - let mut handler = self.on_renegotiation_needed_handler.lock().unwrap(); - if let Some(f) = handler.as_mut() { - f(); - } - } - - fn on_negotiation_needed_event(&self, event: u32) { - trace!("on_negotiation_needed_event"); - let mut handler = self.on_negotiation_needed_event_handler.lock().unwrap(); - if let Some(f) = handler.as_mut() { - f(event); - } - } - - fn on_ice_connection_change(&self, new_state: IceConnectionState) { - trace!("on_ice_connection_change (new_state: {:?})", new_state); - let mut handler = self.on_ice_connection_change_handler.lock().unwrap(); - if let Some(f) = handler.as_mut() { - f(new_state); - } - } - - fn on_standardized_ice_connection_change(&self, new_state: IceConnectionState) { - trace!( - "on_standardized_ice_connection_change (new_state: {:?}", - new_state - ); - let mut handler = self - .on_standardized_ice_connection_change_handler - .lock() - .unwrap(); - if let Some(f) = handler.as_mut() { - f(new_state); - } - } - - fn on_connection_change(&self, new_state: PeerConnectionState) { - trace!("on_connection_change (new_state: {:?})", new_state); - let mut handler = self.on_connection_change_handler.lock().unwrap(); - if let Some(f) = handler.as_mut() { - f(new_state); - } - } - - fn on_ice_gathering_change(&self, new_state: IceGatheringState) { - trace!("on_ice_gathering_change (new_state: {:?}", new_state); - let mut handler = self.on_ice_gathering_change_handler.lock().unwrap(); - if let Some(f) = handler.as_mut() { - f(new_state); - } - } - - fn on_ice_candidate(&self, candidate: SharedPtr) { - trace!("on_ice_candidate"); - let mut handler = self.on_ice_candidate_handler.lock().unwrap(); - if let Some(f) = handler.as_mut() { - f(IceCandidate::new(candidate)); - } - } - - fn on_ice_candidate_error( - &self, - address: String, - port: i32, - url: String, - error_code: i32, - error_text: String, - ) { - trace!("on_ice_candidate_error"); - let mut handler = self.on_ice_candidate_error_handler.lock().unwrap(); - if let Some(f) = handler.as_mut() { - f(address, port, url, error_code, error_text); - } - } - - fn on_ice_candidates_removed(&self, removed: Vec>) { - trace!("on_ice_candidates_removed"); - let mut handler = self.on_ice_candidates_removed_handler.lock().unwrap(); - if let Some(f) = handler.as_mut() { - // TODO(theomonnom) - } - } - - fn on_ice_connection_receiving_change(&self, receiving: bool) { - trace!("on_ice_connection_receiving_change"); - let mut handler = self - .on_ice_connection_receiving_change_handler - .lock() - .unwrap(); - if let Some(f) = handler.as_mut() { - f(receiving); - } - } - - fn on_ice_selected_candidate_pair_changed(&self, event: sys_pc::ffi::CandidatePairChangeEvent) { - trace!("on_ice_selected_candidate_pair_changed"); - let mut handler = self - .on_ice_selected_candidate_pair_changed_handler - .lock() - .unwrap(); - if let Some(f) = handler.as_mut() { - f(event); - } - } - - fn on_add_track( - &self, - receiver: SharedPtr, - streams: Vec>, - ) { - trace!("on_add_track"); - let mut handler = self.on_add_track_handler.lock().unwrap(); - if let Some(f) = handler.as_mut() { - let streams = streams.into_iter().map(MediaStream::new).collect(); - f(RtpReceiver::new(receiver), streams) - } - } - - fn on_track(&self, transceiver: SharedPtr) { - trace!("on_track"); - let mut handler = self.on_track_handler.lock().unwrap(); - if let Some(f) = handler.as_mut() { - // TODO(theomonnom) - } - } - - fn on_remove_track(&self, receiver: SharedPtr) { - trace!("on_remove_track"); - let mut handler = self.on_remove_track_handler.lock().unwrap(); - if let Some(f) = handler.as_mut() { - // TODO(theomonnom) - } - } - - fn on_interesting_usage(&self, usage_pattern: i32) { - trace!("on_interesting_usage"); - let mut handler = self.on_interesting_usage_handler.lock().unwrap(); - if let Some(f) = handler.as_mut() { - f(usage_pattern); - } - } -} - -#[cfg(test)] -mod tests { - use log::trace; - use tokio::sync::mpsc; - - use webrtc_sys::peer_connection::ffi::RTCOfferAnswerOptions; - use webrtc_sys::peer_connection_factory::ffi::{ContinualGatheringPolicy, IceTransportsType}; - - use crate::data_channel::{DataChannel, DataChannelInit}; - use crate::jsep::IceCandidate; - use crate::peer_connection_factory::{ICEServer, PeerConnectionFactory, RTCConfiguration}; - use crate::webrtc::RTCRuntime; - - fn init_log() { - let _ = env_logger::builder().is_test(true).try_init(); - } - - #[tokio::test] - async fn create_pc() { - init_log(); - - let rtc_runtime = RTCRuntime::new(); - - let factory = PeerConnectionFactory::new(rtc_runtime); - let config = RTCConfiguration { - ice_servers: vec![ICEServer { - urls: vec!["stun:stun1.l.google.com:19302".to_string()], - username: "".into(), - password: "".into(), - }], - continual_gathering_policy: ContinualGatheringPolicy::GatherOnce, - ice_transport_type: IceTransportsType::All, - }; - - let mut bob = factory.create_peer_connection(config.clone()).unwrap(); - let mut alice = factory.create_peer_connection(config.clone()).unwrap(); - - let (bob_ice_tx, mut bob_ice_rx) = mpsc::channel::(16); - let (alice_ice_tx, mut alice_ice_rx) = mpsc::channel::(16); - let (alice_dc_tx, mut alice_dc_rx) = mpsc::channel::(16); - - bob.on_ice_candidate(Box::new(move |candidate| { - bob_ice_tx.blocking_send(candidate).unwrap(); - })); - - alice.on_ice_candidate(Box::new(move |candidate| { - alice_ice_tx.blocking_send(candidate).unwrap(); - })); - - alice.on_data_channel(Box::new(move |dc| { - alice_dc_tx.blocking_send(dc).unwrap(); - })); - - let mut bob_dc = bob - .create_data_channel("test_dc", DataChannelInit::default()) - .unwrap(); - - let offer = bob - .create_offer(RTCOfferAnswerOptions::default()) - .await - .unwrap(); - trace!("Bob offer: {:?}", offer); - bob.set_local_description(offer.clone()).await.unwrap(); - alice.set_remote_description(offer).await.unwrap(); - - let answer = alice - .create_answer(RTCOfferAnswerOptions::default()) - .await - .unwrap(); - - trace!("Alice answer: {:?}", answer); - alice.set_local_description(answer.clone()).await.unwrap(); - bob.set_remote_description(answer).await.unwrap(); - - let bob_ice = bob_ice_rx.recv().await.unwrap(); - let alice_ice = alice_ice_rx.recv().await.unwrap(); - - bob.add_ice_candidate(alice_ice).await.unwrap(); - alice.add_ice_candidate(bob_ice).await.unwrap(); - - let (data_tx, mut data_rx) = mpsc::channel::(1); - let mut alice_dc = alice_dc_rx.recv().await.unwrap(); - alice_dc.on_message(Box::new(move |data, _| { - data_tx - .blocking_send(String::from_utf8_lossy(data).to_string()) - .unwrap(); - })); - - bob_dc.send(b"This is a test", true).unwrap(); - assert_eq!(data_rx.recv().await.unwrap(), "This is a test"); - - alice.close(); - bob.close(); +impl Debug for PeerConnection { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PeerConnection") + .field("state", &self.connection_state()) + .field("ice_state", &self.ice_connection_state()) + .finish() } } diff --git a/livekit-webrtc/src/peer_connection_factory.rs b/livekit-webrtc/src/peer_connection_factory.rs index 1e41f15..7fedb86 100644 --- a/livekit-webrtc/src/peer_connection_factory.rs +++ b/livekit-webrtc/src/peer_connection_factory.rs @@ -1,51 +1,78 @@ -use cxx::UniquePtr; +use crate::imp::peer_connection_factory as imp_pcf; +use crate::peer_connection::PeerConnection; +use crate::rtp_parameters::RtpCapabilities; +use crate::MediaType; +use crate::RtcError; +use std::fmt::Debug; -pub use sys_factory::ffi::{ - ContinualGatheringPolicy, ICEServer, IceTransportsType, RTCConfiguration, -}; -use webrtc_sys::peer_connection as sys_pc; -use webrtc_sys::peer_connection_factory as sys_factory; +#[derive(Debug, Clone)] +pub struct IceServer { + pub urls: Vec, + pub username: String, + pub password: String, +} -use crate::peer_connection::{InternalObserver, PeerConnection}; -use crate::rtc_error::RTCError; -use crate::webrtc::RTCRuntime; +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum ContinualGatheringPolicy { + GatherOnce, + GatherContinually, +} +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum IceTransportsType { + None, + Relay, + NoHost, + All, +} + +#[derive(Debug, Clone)] +pub struct RtcConfiguration { + pub ice_servers: Vec, + pub continual_gathering_policy: ContinualGatheringPolicy, + pub ice_transport_type: IceTransportsType, +} + +#[derive(Clone, Default)] pub struct PeerConnectionFactory { - cxx_handle: UniquePtr, - rtc_runtime: RTCRuntime, + pub(crate) handle: imp_pcf::PeerConnectionFactory, +} + +impl Debug for PeerConnectionFactory { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.debug_struct("PeerConnectionFactory").finish() + } } impl PeerConnectionFactory { - pub fn new(rtc_runtime: RTCRuntime) -> Self { - Self { - cxx_handle: sys_factory::ffi::create_peer_connection_factory( - rtc_runtime.clone().release(), - ), - rtc_runtime, - } - } - pub fn create_peer_connection( &self, - config: RTCConfiguration, - ) -> Result { - let native_config = sys_factory::ffi::create_rtc_configuration(config); + config: RtcConfiguration, + ) -> Result { + self.handle.create_peer_connection(config) + } - unsafe { - let mut observer = Box::new(InternalObserver::default()); - let mut native_observer = sys_pc::ffi::create_native_peer_connection_observer( - self.rtc_runtime.clone().release(), - Box::new(sys_pc::PeerConnectionObserverWrapper::new(&mut *observer)), - ); + pub fn get_rtp_sender_capabilities(&self, media_type: MediaType) -> RtpCapabilities { + self.handle.get_rtp_sender_capabilities(media_type) + } - let res = self - .cxx_handle - .create_peer_connection(native_config, native_observer.pin_mut()); + pub fn get_rtp_receiver_capabilities(&self, media_type: MediaType) -> RtpCapabilities { + self.handle.get_rtp_receiver_capabilities(media_type) + } +} - match res { - Ok(cxx_handle) => Ok(PeerConnection::new(cxx_handle, observer, native_observer)), - Err(e) => Err(RTCError::from(e.what())), - } +pub mod native { + use super::PeerConnectionFactory; + use crate::media_stream::RtcVideoTrack; + use crate::video_source::native::NativeVideoSource; + + pub trait PeerConnectionFactoryExt { + fn create_video_track(&self, label: &str, source: NativeVideoSource) -> RtcVideoTrack; + } + + impl PeerConnectionFactoryExt for PeerConnectionFactory { + fn create_video_track(&self, label: &str, source: NativeVideoSource) -> RtcVideoTrack { + self.handle.create_video_track(label, source) } } } diff --git a/livekit-webrtc/src/prelude.rs b/livekit-webrtc/src/prelude.rs index 65a5ede..d2d62d3 100644 --- a/livekit-webrtc/src/prelude.rs +++ b/livekit-webrtc/src/prelude.rs @@ -1,22 +1,24 @@ -pub use crate::data_channel::{DataChannel, DataChannelInit, DataState}; -pub use crate::jsep::{IceCandidate, SessionDescription}; +pub use crate::data_channel::{ + DataBuffer, DataChannel, DataChannelError, DataChannelInit, DataState, +}; +pub use crate::ice_candidate::IceCandidate; pub use crate::media_stream::{ - AudioTrack, MediaStream, MediaStreamTrackHandle, MediaStreamTrackTrait, - OnConstraintsChangedHandler, OnDiscardedFrameHandler, OnFrameHandler, VideoTrack, + MediaStream, MediaStreamTrack, RtcAudioTrack, RtcTrackState, RtcVideoTrack, }; pub use crate::peer_connection::{ - IceConnectionState, IceGatheringState, PeerConnection, PeerConnectionState, - RTCOfferAnswerOptions, SignalingState, + AnswerOptions, IceConnectionState, IceGatheringState, OfferOptions, PeerConnection, + PeerConnectionState, SignalingState, }; pub use crate::peer_connection_factory::{ - ContinualGatheringPolicy, ICEServer, IceTransportsType, PeerConnectionFactory, RTCConfiguration, + ContinualGatheringPolicy, IceServer, IceTransportsType, PeerConnectionFactory, RtcConfiguration, }; -pub use crate::rtc_error::RTCError; pub use crate::rtp_parameters::*; pub use crate::rtp_receiver::RtpReceiver; pub use crate::rtp_sender::RtpSender; -pub use crate::rtp_transceiver::{RtpTransceiver, RtpTransceiverInit}; -pub use crate::video_frame::{VideoFrame, VideoRotation}; -pub use crate::video_frame_buffer::*; -pub use crate::webrtc::*; -pub use crate::yuv_helper::ConvertError; +pub use crate::rtp_transceiver::{RtpTransceiver, RtpTransceiverDirection, RtpTransceiverInit}; +pub use crate::session_description::{SdpType, SessionDescription}; +pub use crate::video_frame::{ + BoxVideoFrame, I010Buffer, I420ABuffer, I420Buffer, I422Buffer, I444Buffer, NV12Buffer, + VideoFormatType, VideoFrame, VideoFrameBuffer, VideoFrameBufferType, VideoRotation, +}; +pub use crate::{RtcError, RtcErrorType}; diff --git a/livekit-webrtc/src/rtc_error.rs b/livekit-webrtc/src/rtc_error.rs deleted file mode 100644 index e92b49d..0000000 --- a/livekit-webrtc/src/rtc_error.rs +++ /dev/null @@ -1,2 +0,0 @@ -// TODO(theomonnom) Wrap the RTCError ffi so we can use Option(u16) -pub use webrtc_sys::rtc_error::ffi::RTCError; diff --git a/livekit-webrtc/src/rtp_parameters.rs b/livekit-webrtc/src/rtp_parameters.rs index fff79d7..5140a79 100644 --- a/livekit-webrtc/src/rtp_parameters.rs +++ b/livekit-webrtc/src/rtp_parameters.rs @@ -1,462 +1,91 @@ -use crate::prelude::*; -use std::collections::HashMap; -use std::vec::Vec; -use webrtc_sys::rtp_parameters as ps_sys; +use crate::rtp_transceiver::RtpTransceiverDirection; -// Don't exporting structs here (only enum), cxx doesn't support Option and HashMap -pub use ps_sys::ffi::{ - DegradationPreference, FecMechanism, RtcpFeedbackMessageType, RtcpFeedbackType, - RtpExtensionFilter, -}; +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum Priority { + VeryLow, + Low, + Medium, + High, +} #[derive(Debug, Clone)] -pub struct RtcpFeedback { - pub feedback_type: RtcpFeedbackType, - pub message_type: Option, +pub struct RtpHeaderExtensionParameters { + pub uri: String, + pub id: i32, + pub encrypted: bool, +} + +#[derive(Debug, Clone, Default)] +pub struct RtpParameters { + pub codecs: Vec, + pub header_extensions: Vec, + pub rtcp: RtcpParameters, +} + +#[derive(Debug, Clone)] +pub struct RtpCodecParameters { + pub payload_type: u8, + pub mime_type: String, // read-only + pub clock_rate: Option, + pub channels: Option, +} + +#[derive(Debug, Clone, Default)] +pub struct RtcpParameters { + pub cname: String, + pub reduced_size: bool, +} + +#[derive(Debug, Clone)] +pub struct RtpEncodingParameters { + pub active: bool, + pub max_bitrate: Option, + pub max_framerate: Option, + pub priority: Priority, + pub rid: String, + pub scale_resolution_down_by: Option, } #[derive(Debug, Clone)] pub struct RtpCodecCapability { + pub channels: Option, + pub clock_rate: Option, pub mime_type: String, - pub name: String, - pub kind: MediaType, - pub clock_rate: Option, - pub preferred_payload_type: Option, - pub max_ptime: Option, - pub ptime: Option, - pub num_channels: Option, - pub rtcp_feedback: Vec, - pub parameters: HashMap, - pub options: HashMap, - pub max_temporal_layer_extensions: i32, - pub max_spatial_layer_extensions: i32, - pub svc_multi_stream_support: bool, + pub sdp_fmtp_line: Option, } #[derive(Debug, Clone)] pub struct RtpHeaderExtensionCapability { pub uri: String, - pub preferred_id: Option, - pub preferred_encrypt: bool, pub direction: RtpTransceiverDirection, } -#[derive(Debug, Clone)] -pub struct RtpExtension { - pub uri: String, - pub id: i32, - pub encrypt: bool, -} - -#[derive(Debug, Clone)] -pub struct RtpFecParameters { - pub ssrc: Option, - pub mechanism: FecMechanism, -} - -#[derive(Debug, Clone)] -pub struct RtpRtxParameters { - pub ssrc: Option, -} - -#[derive(Debug, Clone)] -pub struct RtpEncodingParameters { - pub ssrc: Option, - pub bitrate_priority: f64, - pub network_priority: Priority, - pub max_bitrate_bps: Option, - pub min_bitrate_bps: Option, - pub max_framerate: Option, - pub num_temporal_layers: Option, - pub scale_resolution_down_by: Option, - pub scalability_mode: Option, - pub active: bool, - pub rid: String, - pub adaptive_ptime: bool, -} - -#[derive(Debug, Clone)] -pub struct RtpCodecParameters { - pub mime_type: String, - pub name: String, - pub kind: MediaType, - pub payload_type: i32, - pub clock_rate: Option, - pub num_channels: Option, - pub max_ptime: Option, - pub ptime: Option, - pub rtcp_feedback: Vec, - pub parameters: HashMap, -} - #[derive(Debug, Clone)] pub struct RtpCapabilities { pub codecs: Vec, pub header_extensions: Vec, - pub fec: Vec, } -#[derive(Debug, Clone)] -pub struct RtcpParameters { - pub ssrc: Option, - pub cname: String, - pub reduced_size: bool, - pub mux: bool, -} - -#[derive(Debug, Clone)] -pub struct RtpParameters { - pub transaction_id: String, - pub mid: String, - pub codecs: Vec, - pub header_extensions: Vec, - pub encodings: Vec, - pub rtcp: RtcpParameters, - pub degradation_preference: Option, -} - -fn into_map(vec: Vec) -> HashMap { - let mut map = HashMap::with_capacity(vec.len()); - for pair in vec { - map.insert(pair.key, pair.value); - } - map -} - -impl From for RtcpFeedback { - fn from(value: ps_sys::ffi::RtcpFeedback) -> Self { +impl Default for RtpCodecParameters { + fn default() -> Self { Self { - feedback_type: value.feedback_type, - message_type: value.has_message_type.then_some(value.message_type), + payload_type: 0, + mime_type: String::default(), + clock_rate: None, + channels: None, } } } -impl From for RtpCodecCapability { - fn from(value: ps_sys::ffi::RtpCodecCapability) -> Self { +impl Default for RtpEncodingParameters { + fn default() -> Self { Self { - mime_type: value.mime_type, - name: value.name, - kind: value.kind, - clock_rate: value.has_clock_rate.then_some(value.clock_rate), - preferred_payload_type: value - .has_preferred_payload_type - .then_some(value.preferred_payload_type), - max_ptime: value.has_max_ptime.then_some(value.max_ptime), - ptime: value.has_ptime.then_some(value.ptime), - num_channels: value.has_num_channels.then_some(value.num_channels), - rtcp_feedback: value.rtcp_feedback.into_iter().map(Into::into).collect(), - parameters: into_map(value.parameters), - options: into_map(value.options), - max_temporal_layer_extensions: value.max_temporal_layer_extensions, - max_spatial_layer_extensions: value.max_spatial_layer_extensions, - svc_multi_stream_support: value.svc_multi_stream_support, - } - } -} - -impl From for RtpHeaderExtensionCapability { - fn from(value: ps_sys::ffi::RtpHeaderExtensionCapability) -> Self { - Self { - uri: value.uri, - preferred_id: value.has_preferred_id.then_some(value.preferred_id), - preferred_encrypt: value.preferred_encrypt, - direction: value.direction, - } - } -} - -impl From for RtpExtension { - fn from(value: ps_sys::ffi::RtpExtension) -> Self { - Self { - uri: value.uri, - id: value.id, - encrypt: value.encrypt, - } - } -} - -impl From for RtpFecParameters { - fn from(value: ps_sys::ffi::RtpFecParameters) -> Self { - Self { - ssrc: value.has_ssrc.then_some(value.ssrc), - mechanism: value.mechanism, - } - } -} - -impl From for RtpRtxParameters { - fn from(value: ps_sys::ffi::RtpRtxParameters) -> Self { - Self { - ssrc: value.has_ssrc.then_some(value.ssrc), - } - } -} - -impl From for RtpEncodingParameters { - fn from(value: ps_sys::ffi::RtpEncodingParameters) -> Self { - Self { - ssrc: value.has_ssrc.then_some(value.ssrc), - bitrate_priority: value.bitrate_priority, - network_priority: value.network_priority, - max_bitrate_bps: value.has_max_bitrate_bps.then_some(value.max_bitrate_bps), - min_bitrate_bps: value.has_min_bitrate_bps.then_some(value.min_bitrate_bps), - max_framerate: value.has_max_framerate.then_some(value.max_framerate), - num_temporal_layers: value - .has_num_temporal_layers - .then_some(value.num_temporal_layers), - scale_resolution_down_by: value - .has_scale_resolution_down_by - .then_some(value.scale_resolution_down_by), - scalability_mode: value.has_scalability_mode.then_some(value.scalability_mode), - active: value.active, - rid: value.rid, - adaptive_ptime: value.adaptive_ptime, - } - } -} - -impl From for RtpCodecParameters { - fn from(value: ps_sys::ffi::RtpCodecParameters) -> Self { - Self { - mime_type: value.mime_type, - name: value.name, - kind: value.kind, - payload_type: value.payload_type, - clock_rate: value.has_clock_rate.then_some(value.clock_rate), - num_channels: value.has_num_channels.then_some(value.num_channels), - max_ptime: value.has_max_ptime.then_some(value.max_ptime), - ptime: value.has_ptime.then_some(value.ptime), - rtcp_feedback: value.rtcp_feedback.into_iter().map(Into::into).collect(), - parameters: into_map(value.parameters), - } - } -} - -impl From for RtpCapabilities { - fn from(value: ps_sys::ffi::RtpCapabilities) -> Self { - Self { - codecs: value.codecs.into_iter().map(Into::into).collect(), - header_extensions: value - .header_extensions - .into_iter() - .map(Into::into) - .collect(), - fec: value.fec.into_iter().map(Into::into).collect(), - } - } -} - -impl From for RtcpParameters { - fn from(value: ps_sys::ffi::RtcpParameters) -> Self { - Self { - ssrc: value.has_ssrc.then_some(value.ssrc), - cname: value.cname, - reduced_size: value.reduced_size, - mux: value.mux, - } - } -} - -impl From for RtpParameters { - fn from(value: ps_sys::ffi::RtpParameters) -> Self { - Self { - transaction_id: value.transaction_id, - mid: value.mid, - codecs: value.codecs.into_iter().map(Into::into).collect(), - header_extensions: value - .header_extensions - .into_iter() - .map(Into::into) - .collect(), - encodings: value.encodings.into_iter().map(Into::into).collect(), - rtcp: value.rtcp.into(), - degradation_preference: value - .has_degradation_preference - .then_some(value.degradation_preference), - } - } -} - -// Ignore the value inside unwrap_or for the following implementations - -fn into_vec(map: HashMap) -> Vec { - let mut vec = Vec::with_capacity(map.len()); - for (key, value) in map { - vec.push(ps_sys::ffi::StringKeyValue { key, value }) - } - vec -} - -impl From for ps_sys::ffi::RtcpFeedback { - fn from(value: RtcpFeedback) -> Self { - Self { - feedback_type: value.feedback_type, - has_message_type: value.message_type.is_some(), - message_type: value - .message_type - .unwrap_or(RtcpFeedbackMessageType::GenericNACK), - } - } -} - -impl From for ps_sys::ffi::RtpCodecCapability { - fn from(value: RtpCodecCapability) -> Self { - Self { - mime_type: value.mime_type, - name: value.name, - kind: value.kind, - has_clock_rate: value.clock_rate.is_some(), - clock_rate: value.clock_rate.unwrap_or(0), - has_preferred_payload_type: value.preferred_payload_type.is_some(), - preferred_payload_type: value.preferred_payload_type.unwrap_or(0), - has_max_ptime: value.max_ptime.is_some(), - max_ptime: value.max_ptime.unwrap_or(0), - has_ptime: value.ptime.is_some(), - ptime: value.ptime.unwrap_or(0), - has_num_channels: value.num_channels.is_some(), - num_channels: value.num_channels.unwrap_or(0), - rtcp_feedback: value.rtcp_feedback.into_iter().map(Into::into).collect(), - parameters: into_vec(value.parameters), - options: into_vec(value.options), - max_temporal_layer_extensions: value.max_temporal_layer_extensions, - max_spatial_layer_extensions: value.max_spatial_layer_extensions, - svc_multi_stream_support: value.svc_multi_stream_support, - } - } -} - -impl From for ps_sys::ffi::RtpHeaderExtensionCapability { - fn from(value: RtpHeaderExtensionCapability) -> Self { - Self { - uri: value.uri, - has_preferred_id: value.preferred_id.is_some(), - preferred_id: value.preferred_id.unwrap_or(0), - preferred_encrypt: value.preferred_encrypt, - direction: value.direction, - } - } -} - -impl From for ps_sys::ffi::RtpExtension { - fn from(value: RtpExtension) -> Self { - Self { - uri: value.uri, - id: value.id, - encrypt: value.encrypt, - } - } -} - -impl From for ps_sys::ffi::RtpFecParameters { - fn from(value: RtpFecParameters) -> Self { - Self { - has_ssrc: value.ssrc.is_some(), - ssrc: value.ssrc.unwrap_or(0), - mechanism: value.mechanism, - } - } -} - -impl From for ps_sys::ffi::RtpRtxParameters { - fn from(value: RtpRtxParameters) -> Self { - Self { - has_ssrc: value.ssrc.is_some(), - ssrc: value.ssrc.unwrap_or(0), - } - } -} - -impl From for ps_sys::ffi::RtpEncodingParameters { - fn from(value: RtpEncodingParameters) -> Self { - Self { - has_ssrc: value.ssrc.is_some(), - ssrc: value.ssrc.unwrap_or(0), - bitrate_priority: value.bitrate_priority, - network_priority: value.network_priority, - has_max_bitrate_bps: value.max_bitrate_bps.is_some(), - max_bitrate_bps: value.max_bitrate_bps.unwrap_or(0), - has_min_bitrate_bps: value.min_bitrate_bps.is_some(), - min_bitrate_bps: value.min_bitrate_bps.unwrap_or(0), - has_max_framerate: value.max_framerate.is_some(), - max_framerate: value.max_framerate.unwrap_or(0.0), - has_num_temporal_layers: value.num_temporal_layers.is_some(), - num_temporal_layers: value.num_temporal_layers.unwrap_or(0), - has_scale_resolution_down_by: value.scale_resolution_down_by.is_some(), - scale_resolution_down_by: value.scale_resolution_down_by.unwrap_or(0.0), - has_scalability_mode: value.scalability_mode.is_some(), - scalability_mode: value.scalability_mode.unwrap_or(String::new()), - active: value.active, - rid: value.rid, - adaptive_ptime: value.adaptive_ptime, - } - } -} - -impl From for ps_sys::ffi::RtpCodecParameters { - fn from(value: RtpCodecParameters) -> Self { - Self { - mime_type: value.mime_type, - name: value.name, - kind: value.kind, - payload_type: value.payload_type, - has_clock_rate: value.clock_rate.is_some(), - clock_rate: value.clock_rate.unwrap_or(0), - has_num_channels: value.num_channels.is_some(), - num_channels: value.num_channels.unwrap_or(0), - has_max_ptime: value.max_ptime.is_some(), - max_ptime: value.max_ptime.unwrap_or(0), - has_ptime: value.ptime.is_some(), - ptime: value.ptime.unwrap_or(0), - rtcp_feedback: value.rtcp_feedback.into_iter().map(Into::into).collect(), - parameters: into_vec(value.parameters), - } - } -} - -impl From for ps_sys::ffi::RtpCapabilities { - fn from(value: RtpCapabilities) -> Self { - Self { - codecs: value.codecs.into_iter().map(Into::into).collect(), - header_extensions: value - .header_extensions - .into_iter() - .map(Into::into) - .collect(), - fec: value.fec.into_iter().map(Into::into).collect(), - } - } -} - -impl From for ps_sys::ffi::RtcpParameters { - fn from(value: RtcpParameters) -> Self { - Self { - has_ssrc: value.ssrc.is_some(), - ssrc: value.ssrc.unwrap_or(0), - cname: value.cname, - reduced_size: value.reduced_size, - mux: value.mux, - } - } -} - -impl From for ps_sys::ffi::RtpParameters { - fn from(value: RtpParameters) -> Self { - Self { - transaction_id: value.transaction_id, - mid: value.mid, - codecs: value.codecs.into_iter().map(Into::into).collect(), - header_extensions: value - .header_extensions - .into_iter() - .map(Into::into) - .collect(), - encodings: value.encodings.into_iter().map(Into::into).collect(), - rtcp: value.rtcp.into(), - has_degradation_preference: value.degradation_preference.is_some(), - degradation_preference: value - .degradation_preference - .unwrap_or(DegradationPreference::Balanced), + active: true, + max_bitrate: None, + max_framerate: None, + priority: Priority::Low, + rid: String::default(), + scale_resolution_down_by: None, } } } diff --git a/livekit-webrtc/src/rtp_receiver.rs b/livekit-webrtc/src/rtp_receiver.rs index f7d3ad0..acaf71a 100644 --- a/livekit-webrtc/src/rtp_receiver.rs +++ b/livekit-webrtc/src/rtp_receiver.rs @@ -1,67 +1,29 @@ -use crate::media_stream::{MediaStream, MediaStreamTrackHandle}; -use crate::rtp_parameters::RtpParameters; -use cxx::SharedPtr; -use std::fmt::{Debug, Formatter}; -use webrtc_sys::rtp_receiver as sys_rec; -use webrtc_sys::webrtc as sys_webrtc; +use std::fmt::Debug; -pub use sys_webrtc::ffi::MediaType; +use crate::{ + imp::rtp_receiver as imp_rr, media_stream::MediaStreamTrack, rtp_parameters::RtpParameters, +}; #[derive(Clone)] pub struct RtpReceiver { - cxx_handle: SharedPtr, -} - -impl Debug for RtpReceiver { - fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - f.debug_struct("RtpReceiver") - .field("track", &self.track()) - .field("media_type", &self.media_type()) - .field("id", &self.id()) - .finish() - } + pub(crate) handle: imp_rr::RtpReceiver, } impl RtpReceiver { - pub(crate) fn new(cxx_handle: SharedPtr) -> Self { - Self { cxx_handle } - } - - pub(crate) fn cxx_handle(&self) -> SharedPtr { - self.cxx_handle.clone() - } - - pub fn track(&self) -> MediaStreamTrackHandle { - MediaStreamTrackHandle::new(self.cxx_handle.track()) - } - - pub fn stream_ids(&self) -> Vec { - self.cxx_handle.stream_ids() - } - - pub fn streams(&self) -> Vec { - let ptrs = self.cxx_handle.streams(); - let mut vec = Vec::with_capacity(ptrs.len()); - for stream in ptrs { - vec.push(MediaStream::new(stream.ptr)); - } - vec - } - - pub fn media_type(&self) -> MediaType { - self.cxx_handle.media_type() - } - - pub fn id(&self) -> String { - self.cxx_handle.id() + pub fn track(&self) -> Option { + self.handle.track() } pub fn parameters(&self) -> RtpParameters { - self.cxx_handle.get_parameters().into() - } - - pub fn set_jitter_buffer_minimum_delay(&self, delay_seconds: Option) { - self.cxx_handle - .set_jitter_buffer_minimum_delay(delay_seconds.is_some(), delay_seconds.unwrap_or(0.0)); + self.handle.parameters() + } +} + +impl Debug for RtpReceiver { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RtpReceiver") + .field("track", &self.track()) + .field("cname", &self.parameters().rtcp.cname) + .finish() } } diff --git a/livekit-webrtc/src/rtp_sender.rs b/livekit-webrtc/src/rtp_sender.rs index 0c5eb66..6d15392 100644 --- a/livekit-webrtc/src/rtp_sender.rs +++ b/livekit-webrtc/src/rtp_sender.rs @@ -1,81 +1,37 @@ -use crate::media_stream::{MediaStream, MediaStreamTrackHandle}; -use crate::prelude::*; -use crate::rtp_parameters::{RtpEncodingParameters, RtpParameters}; -use cxx::SharedPtr; -use std::fmt::{Debug, Formatter}; -use webrtc_sys::rtp_sender as sys_rs; -use webrtc_sys::webrtc as sys_webrtc; +use std::fmt::Debug; -pub use sys_webrtc::ffi::MediaType; +use crate::{ + imp::rtp_sender as imp_rs, media_stream::MediaStreamTrack, rtp_parameters::RtpParameters, + RtcError, +}; #[derive(Clone)] pub struct RtpSender { - cxx_handle: SharedPtr, -} - -impl Debug for RtpSender { - fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - f.debug_struct("RtpSender") - .field("track", &self.track()) - .field("media_type", &self.media_type()) - .field("ssrc", &self.ssrc()) - .field("id", &self.id()) - .finish() - } + pub(crate) handle: imp_rs::RtpSender, } impl RtpSender { - pub(crate) fn new(cxx_handle: SharedPtr) -> Self { - Self { cxx_handle } + pub fn track(&self) -> Option { + self.handle.track() } - pub(crate) fn cxx_handle(&self) -> SharedPtr { - self.cxx_handle.clone() - } - - pub fn set_track(&self, track: MediaStreamTrackHandle) -> bool { - self.cxx_handle.set_track(track.cxx_handle()) - } - - pub fn track(&self) -> MediaStreamTrackHandle { - MediaStreamTrackHandle::new(self.cxx_handle.track()) - } - - pub fn ssrc(&self) -> u32 { - self.cxx_handle.ssrc() - } - - pub fn media_type(&self) -> MediaType { - self.cxx_handle.media_type() - } - - pub fn id(&self) -> String { - self.cxx_handle.id() - } - - pub fn stream_ids(&self) -> Vec { - self.cxx_handle.stream_ids() - } - - pub fn set_streams(&self, stream_ids: &Vec) { - self.cxx_handle.set_streams(stream_ids); - } - - pub fn init_send_encodings(&self) -> Vec { - self.cxx_handle - .init_send_encodings() - .into_iter() - .map(Into::into) - .collect() + pub fn set_track(&self, track: Option) -> Result<(), RtcError> { + self.handle.set_track(track) } pub fn parameters(&self) -> RtpParameters { - self.cxx_handle.get_parameters().into() + self.handle.parameters() } - pub fn set_parameters(&self, params: RtpParameters) -> Result<(), RTCError> { - self.cxx_handle - .set_parameters(params.into()) - .map_err(|e| unsafe { RTCError::from(e.what()) }) + pub fn set_parameters(&self, parameters: RtpParameters) -> Result<(), RtcError> { + self.handle.set_parameters(parameters) + } +} + +impl Debug for RtpSender { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RtpReceiver") + .field("cname", &self.parameters().rtcp.cname) + .finish() } } diff --git a/livekit-webrtc/src/rtp_transceiver.rs b/livekit-webrtc/src/rtp_transceiver.rs index d6f6853..fbdf26d 100644 --- a/livekit-webrtc/src/rtp_transceiver.rs +++ b/livekit-webrtc/src/rtp_transceiver.rs @@ -1,119 +1,68 @@ -use crate::prelude::*; -use cxx::SharedPtr; -use std::fmt::{Debug, Formatter}; -use webrtc_sys::rtp_transceiver as sys_rt; +use crate::imp::rtp_transceiver as imp_rt; +use crate::rtp_parameters::{RtpCodecCapability, RtpEncodingParameters}; +use crate::rtp_receiver::RtpReceiver; +use crate::rtp_sender::RtpSender; +use crate::RtcError; +use std::fmt::Debug; -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct RtpTransceiverInit { pub direction: RtpTransceiverDirection, pub stream_ids: Vec, pub send_encodings: Vec, } -impl From for sys_rt::ffi::RtpTransceiverInit { - fn from(value: RtpTransceiverInit) -> Self { - Self { - direction: value.direction, - stream_ids: value.stream_ids, - send_encodings: value.send_encodings.into_iter().map(Into::into).collect() - } - } +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum RtpTransceiverDirection { + SendRecv, + SendOnly, + RecvOnly, + Inactive, + Stopped, } #[derive(Clone)] pub struct RtpTransceiver { - cxx_handle: SharedPtr, -} - -impl Debug for RtpTransceiver { - fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - f.debug_struct("RtpTransceiver") - .field("media_type", &self.media_type()) - .field("mid", &self.mid()) - .field("direction", &self.direction()) - .field("stopped", &self.stopped()) - .field("stopping", &self.stopping()) - .finish() - } + pub(crate) handle: imp_rt::RtpTransceiver, } impl RtpTransceiver { - pub(crate) fn new(cxx_handle: SharedPtr) -> Self { - Self { cxx_handle } - } - - pub(crate) fn cxx_handle(&self) -> SharedPtr { - self.cxx_handle.clone() - } - - pub fn media_type(&self) -> MediaType { - self.cxx_handle.media_type() - } - pub fn mid(&self) -> Option { - self.cxx_handle.mid().ok() - } - - pub fn sender(&self) -> RtpSender { - RtpSender::new(self.cxx_handle.sender()) - } - - pub fn receiver(&self) -> RtpReceiver { - RtpReceiver::new(self.cxx_handle.receiver()) - } - - pub fn stopped(&self) -> bool { - self.cxx_handle.stopped() - } - - pub fn stopping(&self) -> bool { - self.cxx_handle.stopping() - } - - pub fn direction(&self) -> RtpTransceiverDirection { - self.cxx_handle.direction() - } - - pub fn set_direction(&self, direction: RtpTransceiverDirection) -> Result<(), RTCError> { - self.cxx_handle.set_direction(direction) - .map_err(|e| unsafe { RTCError::from(e.what()) }) + self.handle.mid() } pub fn current_direction(&self) -> Option { - self.cxx_handle.current_direction().ok() + self.handle.current_direction() } - pub fn fired_direction(&self) -> Option { - self.cxx_handle.fired_direction().ok() + pub fn direction(&self) -> RtpTransceiverDirection { + self.handle.direction() } - pub fn stop_standard(&self) -> Result<(), RTCError> { - self.cxx_handle.stop_standard() - .map_err(|e| unsafe { RTCError::from(e.what()) }) + pub fn sender(&self) -> RtpSender { + self.handle.sender() } - pub fn set_codec_preferences(&self, codecs: Vec) -> Result<(), RTCError> { - let ffi_codecs = codecs.into_iter().map(Into::into).collect(); - self.cxx_handle.set_codec_preferences(ffi_codecs) - .map_err(|e| unsafe { RTCError::from(e.what()) }) + pub fn receiver(&self) -> RtpReceiver { + self.handle.receiver() } - pub fn codec_preferences(&self) -> Vec { - self.cxx_handle.codec_preferences().into_iter().map(Into::into).collect() + pub fn set_codec_preferences(&self, codecs: Vec) -> Result<(), RtcError> { + self.handle.set_codec_preferences(codecs) } - pub fn header_extensions_to_offer(&self) -> Vec { - self.cxx_handle.header_extensions_to_offer().into_iter().map(Into::into).collect() + pub fn stop(&self) -> Result<(), RtcError> { + self.handle.stop() + } +} + +impl Debug for RtpTransceiver { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RtpTransceiver") + .field("mid", &self.mid()) + .field("direction", &self.direction()) + .field("sender", &self.sender()) + .field("receiver", &self.receiver()) + .finish() } - - pub fn header_extensions_negotiated(&self) -> Vec { - self.cxx_handle.header_extensions_negotiated().into_iter().map(Into::into).collect() - } - - pub fn set_offered_rtp_header_extensions(&self, headers: Vec) -> Result<(), RTCError> { - let ffi_headers = headers.into_iter().map(Into::into).collect(); - self.cxx_handle.set_offered_rtp_header_extensions(ffi_headers) - .map_err(|e| unsafe { RTCError::from(e.what()) }) - } - } diff --git a/livekit-webrtc/src/session_description.rs b/livekit-webrtc/src/session_description.rs new file mode 100644 index 0000000..22e24b9 --- /dev/null +++ b/livekit-webrtc/src/session_description.rs @@ -0,0 +1,61 @@ +use crate::imp::session_description as sd_imp; +use std::{fmt::Debug, str::FromStr}; +use thiserror::Error; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum SdpType { + Offer, + PrAnswer, + Answer, + Rollback, +} + +impl FromStr for SdpType { + type Err = &'static str; + + fn from_str(sdp_type: &str) -> Result { + match sdp_type { + "offer" => Ok(Self::Offer), + "pranswer" => Ok(Self::PrAnswer), + "answer" => Ok(Self::Answer), + "rollback" => Ok(Self::Rollback), + _ => Err("invalid SdpType"), + } + } +} + +#[derive(Clone)] +pub struct SessionDescription { + pub(crate) handle: sd_imp::SessionDescription, +} + +#[derive(Clone, Error, Debug)] +#[error("Failed to parse sdp: {line} - {description}")] +pub struct SdpParseError { + pub line: String, + pub description: String, +} + +impl SessionDescription { + pub fn parse(sdp: &str, sdp_type: SdpType) -> Result { + sd_imp::SessionDescription::parse(sdp, sdp_type) + } + + pub fn sdp_type(&self) -> SdpType { + self.handle.sdp_type() + } +} + +impl ToString for SessionDescription { + fn to_string(&self) -> String { + self.handle.to_string() + } +} + +impl Debug for SessionDescription { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SessionDescription") + .field("sdp_type", &self.sdp_type()) + .finish() + } +} diff --git a/livekit-webrtc/src/video_frame.rs b/livekit-webrtc/src/video_frame.rs index 9ef0329..8ef3a37 100644 --- a/livekit-webrtc/src/video_frame.rs +++ b/livekit-webrtc/src/video_frame.rs @@ -1,9 +1,14 @@ -use crate::video_frame_buffer::VideoFrameBuffer; -use cxx::UniquePtr; -use std::fmt::{Debug, Formatter}; -use webrtc_sys::video_frame as vf_sys; +use crate::imp::video_frame as vf_imp; +use std::fmt::Debug; +use thiserror::Error; -#[derive(Debug)] +#[derive(Debug, Error)] +pub enum SinkError { + #[error("platform error: {0}")] + Platform(String), +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] pub enum VideoRotation { VideoRotation0 = 0, VideoRotation90 = 90, @@ -11,141 +16,479 @@ pub enum VideoRotation { VideoRotation270 = 270, } -impl From for VideoRotation { - fn from(rotation: vf_sys::ffi::VideoRotation) -> Self { - match rotation { - vf_sys::ffi::VideoRotation::VideoRotation0 => Self::VideoRotation0, - vf_sys::ffi::VideoRotation::VideoRotation90 => Self::VideoRotation90, - vf_sys::ffi::VideoRotation::VideoRotation180 => Self::VideoRotation180, - vf_sys::ffi::VideoRotation::VideoRotation270 => Self::VideoRotation270, - _ => unreachable!(), +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum VideoFormatType { + ARGB, + BGRA, + ABGR, + RGBA, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum VideoFrameBufferType { + Native, + I420, + I420A, + I422, + I444, + I010, + NV12, + WebGl, +} + +#[derive(Debug)] +pub struct VideoFrame +where + T: VideoFrameBuffer, +{ + pub rotation: VideoRotation, + pub timestamp: i64, // When the frame was captured + pub buffer: T, +} + +pub type BoxVideoFrame = VideoFrame>; + +macro_rules! new_buffer_type { + ($type:ident, $variant:ident, $as:ident) => { + pub struct $type { + pub(crate) handle: vf_imp::$type, + } + + impl $crate::video_frame::internal::BufferInternal for $type { + #[cfg(not(target_arch = "wasm32"))] + fn sys_handle(&self) -> &webrtc_sys::video_frame_buffer::ffi::VideoFrameBuffer { + self.handle.sys_handle() + } + + #[cfg(not(target_arch = "wasm32"))] + fn to_i420(&self) -> I420Buffer { + I420Buffer { + handle: self.handle.to_i420(), + } + } + + #[cfg(not(target_arch = "wasm32"))] + fn to_argb( + &self, + format: VideoFormatType, + dst: &mut [u8], + stride: i32, + width: i32, + height: i32, + ) -> Result<(), $crate::video_frame::native::ConvertError> { + self.handle.to_argb(format, dst, stride, width, height) + } + } + + impl VideoFrameBuffer for $type { + fn width(&self) -> i32 { + self.handle.width() + } + + fn height(&self) -> i32 { + self.handle.height() + } + + fn buffer_type(&self) -> VideoFrameBufferType { + VideoFrameBufferType::$variant + } + + fn $as(&self) -> Option<&$type> { + Some(self) + } + } + + impl Debug for $type { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct(stringify!($type)) + .field("width", &self.width()) + .field("height", &self.height()) + .finish() + } + } + }; +} + +pub(crate) mod internal { + use super::{I420Buffer, VideoFormatType}; + + pub trait BufferInternal { + #[cfg(not(target_arch = "wasm32"))] + fn sys_handle(&self) -> &webrtc_sys::video_frame_buffer::ffi::VideoFrameBuffer; + + #[cfg(not(target_arch = "wasm32"))] + fn to_i420(&self) -> I420Buffer; + + #[cfg(not(target_arch = "wasm32"))] + fn to_argb( + &self, + format: VideoFormatType, + dst: &mut [u8], + dst_stride: i32, + dst_width: i32, + dst_height: i32, + ) -> Result<(), super::native::ConvertError>; + } +} + +pub trait VideoFrameBuffer: internal::BufferInternal + Debug { + fn width(&self) -> i32; + fn height(&self) -> i32; + fn buffer_type(&self) -> VideoFrameBufferType; + + #[cfg(not(target_arch = "wasm32"))] + fn as_native(&self) -> Option<&native::NativeBuffer> { + None + } + + fn as_i420(&self) -> Option<&I420Buffer> { + None + } + + fn as_i420a(&self) -> Option<&I420ABuffer> { + None + } + + fn as_i422(&self) -> Option<&I422Buffer> { + None + } + + fn as_i444(&self) -> Option<&I444Buffer> { + None + } + + fn as_i010(&self) -> Option<&I010Buffer> { + None + } + + fn as_nv12(&self) -> Option<&NV12Buffer> { + None + } +} + +new_buffer_type!(I420Buffer, I420, as_i420); +new_buffer_type!(I420ABuffer, I420A, as_i420a); +new_buffer_type!(I422Buffer, I422, as_i422); +new_buffer_type!(I444Buffer, I444, as_i444); +new_buffer_type!(I010Buffer, I010, as_i010); +new_buffer_type!(NV12Buffer, NV12, as_nv12); + +impl I420Buffer { + pub fn chroma_width(&self) -> i32 { + self.handle.chroma_width() + } + + pub fn chroma_height(&self) -> i32 { + self.handle.chroma_height() + } + + pub fn stride_y(&self) -> i32 { + self.handle.stride_y() + } + + pub fn stride_u(&self) -> i32 { + self.handle.stride_u() + } + + pub fn stride_v(&self) -> i32 { + self.handle.stride_v() + } + + pub fn data(&self) -> (&[u8], &[u8], &[u8]) { + self.handle.data() + } + + pub fn data_mut(&mut self) -> (&mut [u8], &mut [u8], &mut [u8]) { + let (data_y, data_u, data_v) = self.handle.data(); + unsafe { + ( + std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()), + std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u8, data_u.len()), + std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u8, data_v.len()), + ) } } } -impl From for vf_sys::ffi::VideoRotation { - fn from(rotation: VideoRotation) -> Self { - match rotation { - VideoRotation::VideoRotation0 => Self::VideoRotation0, - VideoRotation::VideoRotation90 => Self::VideoRotation90, - VideoRotation::VideoRotation180 => Self::VideoRotation180, - VideoRotation::VideoRotation270 => Self::VideoRotation270, +impl I420ABuffer { + pub fn chroma_width(&self) -> i32 { + self.handle.chroma_width() + } + + pub fn chroma_height(&self) -> i32 { + self.handle.chroma_height() + } + + pub fn stride_y(&self) -> i32 { + self.handle.stride_y() + } + + pub fn stride_u(&self) -> i32 { + self.handle.stride_u() + } + + pub fn stride_v(&self) -> i32 { + self.handle.stride_v() + } + + pub fn stride_a(&self) -> i32 { + self.handle.stride_a() + } + + pub fn data(&self) -> (&[u8], &[u8], &[u8], Option<&[u8]>) { + self.handle.data() + } + + pub fn data_mut(&self) -> (&mut [u8], &mut [u8], &mut [u8], Option<&mut [u8]>) { + let (data_y, data_u, data_v, data_a) = self.handle.data(); + unsafe { + ( + std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()), + std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u8, data_u.len()), + std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u8, data_v.len()), + data_a.map(|data_a| { + std::slice::from_raw_parts_mut(data_a.as_ptr() as *mut u8, data_a.len()) + }), + ) } } } -pub struct VideoFrame { - cxx_handle: UniquePtr, -} - -impl Debug for VideoFrame { - fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - f.debug_struct("VideoFrame") - .field("width", &self.width()) - .field("height", &self.height()) - .field("id", &self.id()) - .field("rotation", &self.rotation()) - .field("timestamp", &self.timestamp()) - .finish() - } -} - -impl VideoFrame { - pub(crate) fn new(cxx_handle: UniquePtr) -> Self { - Self { cxx_handle } +impl I422Buffer { + pub fn chroma_width(&self) -> i32 { + self.handle.chroma_width() } - pub fn width(&self) -> i32 { - self.cxx_handle.width() + pub fn chroma_height(&self) -> i32 { + self.handle.chroma_height() } - pub fn height(&self) -> i32 { - self.cxx_handle.height() + pub fn stride_y(&self) -> i32 { + self.handle.stride_y() } - pub fn size(&self) -> u32 { - self.cxx_handle.size() + pub fn stride_u(&self) -> i32 { + self.handle.stride_u() } - pub fn id(&self) -> u16 { - self.cxx_handle.id() + pub fn stride_v(&self) -> i32 { + self.handle.stride_v() } - pub fn timestamp_us(&self) -> i64 { - self.cxx_handle.timestamp_us() + pub fn data(&self) -> (&[u8], &[u8], &[u8]) { + self.handle.data() } - pub fn ntp_time_ms(&self) -> i64 { - self.cxx_handle.ntp_time_ms() - } - - pub fn transport_frame_id(&self) -> u32 { - self.cxx_handle.transport_frame_id() - } - - pub fn timestamp(&self) -> u32 { - self.cxx_handle.timestamp() - } - - pub fn rotation(&self) -> VideoRotation { - self.cxx_handle.rotation().into() - } - - /// # Safety - /// Must be called only once, this function create the safe Rust - /// wrapper around a VideoFrameBuffer. - /// Only one wrapper musts exist at a time. - pub(crate) unsafe fn video_frame_buffer(&self) -> VideoFrameBuffer { - VideoFrameBuffer::new(self.cxx_handle.video_frame_buffer()) - } - - pub fn builder() -> VideoFrameBuilder { - VideoFrameBuilder::default() - } -} - -pub struct VideoFrameBuilder { - cxx_handle: UniquePtr, -} - -impl Debug for VideoFrameBuilder { - fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - f.debug_struct("VideoFrameBuilder").finish() - } -} - -impl Default for VideoFrameBuilder { - fn default() -> Self { - Self { - cxx_handle: vf_sys::ffi::create_video_frame_builder(), + pub fn data_mut(&mut self) -> (&mut [u8], &mut [u8], &mut [u8]) { + let (data_y, data_u, data_v) = self.handle.data(); + unsafe { + ( + std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()), + std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u8, data_u.len()), + std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u8, data_v.len()), + ) } } } -impl VideoFrameBuilder { - pub fn set_video_frame_buffer(mut self, buffer: VideoFrameBuffer) -> Self { - self.cxx_handle - .pin_mut() - .set_video_frame_buffer(buffer.release()); - self +impl I444Buffer { + pub fn chroma_width(&self) -> i32 { + self.handle.chroma_width() } - pub fn set_timestamp_us(mut self, ts_us: i64) -> Self { - self.cxx_handle.pin_mut().set_timestamp_us(ts_us); - self + pub fn chroma_height(&self) -> i32 { + self.handle.chroma_height() } - pub fn set_rotation(mut self, rotation: VideoRotation) -> Self { - self.cxx_handle.pin_mut().set_rotation(rotation.into()); - self + pub fn stride_y(&self) -> i32 { + self.handle.stride_y() } - pub fn set_id(mut self, id: u16) -> Self { - self.cxx_handle.pin_mut().set_id(id); - self + pub fn stride_u(&self) -> i32 { + self.handle.stride_u() } - pub fn build(mut self) -> VideoFrame { - VideoFrame::new(self.cxx_handle.pin_mut().build()) + pub fn stride_v(&self) -> i32 { + self.handle.stride_v() + } + + pub fn data(&self) -> (&[u8], &[u8], &[u8]) { + self.handle.data() + } + + pub fn data_mut(&mut self) -> (&mut [u8], &mut [u8], &mut [u8]) { + let (data_y, data_u, data_v) = self.handle.data(); + unsafe { + ( + std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()), + std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u8, data_u.len()), + std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u8, data_v.len()), + ) + } } } + +impl I010Buffer { + pub fn chroma_width(&self) -> i32 { + self.handle.chroma_width() + } + + pub fn chroma_height(&self) -> i32 { + self.handle.chroma_height() + } + + pub fn stride_y(&self) -> i32 { + self.handle.stride_y() + } + + pub fn stride_u(&self) -> i32 { + self.handle.stride_u() + } + + pub fn stride_v(&self) -> i32 { + self.handle.stride_v() + } + + pub fn data(&self) -> (&[u16], &[u16], &[u16]) { + self.handle.data() + } + + pub fn data_mut(&mut self) -> (&mut [u16], &mut [u16], &mut [u16]) { + let (data_y, data_u, data_v) = self.handle.data(); + unsafe { + ( + std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u16, data_y.len()), + std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u16, data_u.len()), + std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u16, data_v.len()), + ) + } + } +} + +impl NV12Buffer { + pub fn chroma_width(&self) -> i32 { + self.handle.chroma_width() + } + + pub fn chroma_height(&self) -> i32 { + self.handle.chroma_height() + } + + pub fn stride_y(&self) -> i32 { + self.handle.stride_y() + } + + pub fn stride_uv(&self) -> i32 { + self.handle.stride_uv() + } + + pub fn data(&self) -> (&[u8], &[u8]) { + self.handle.data() + } + + pub fn data_mut(&mut self) -> (&mut [u8], &mut [u8]) { + let (data_y, data_uv) = self.handle.data(); + unsafe { + ( + std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()), + std::slice::from_raw_parts_mut(data_uv.as_ptr() as *mut u8, data_uv.len()), + ) + } + } +} + +#[cfg(not(target_arch = "wasm32"))] +pub mod native { + use super::{vf_imp, I420Buffer, VideoFormatType, VideoFrameBuffer, VideoFrameBufferType}; + use std::fmt::Debug; + + pub use crate::imp::yuv_helper::ConvertError; + + new_buffer_type!(NativeBuffer, Native, as_native); + + pub trait I420BufferExt { + fn new(width: u32, height: u32) -> I420Buffer; + } + + impl I420BufferExt for I420Buffer { + fn new(width: u32, height: u32) -> I420Buffer { + vf_imp::I420Buffer::new(width, height) + } + } + + pub trait VideoFrameBufferExt: VideoFrameBuffer { + fn to_i420(&self) -> I420Buffer; + fn to_argb( + &self, + format: VideoFormatType, + dst: &mut [u8], + dst_stride: i32, + dst_width: i32, + dst_height: i32, + ) -> Result<(), ConvertError>; + } + + impl VideoFrameBufferExt for T { + fn to_i420(&self) -> I420Buffer { + self.to_i420() + } + + fn to_argb( + &self, + format: VideoFormatType, + dst: &mut [u8], + dst_stride: i32, + dst_width: i32, + dst_height: i32, + ) -> Result<(), ConvertError> { + self.to_argb(format, dst, dst_stride, dst_width, dst_height) + } + } +} + +impl internal::BufferInternal for Box { + fn sys_handle(&self) -> &webrtc_sys::video_frame_buffer::ffi::VideoFrameBuffer { + self.as_ref().sys_handle() + } + + fn to_i420(&self) -> I420Buffer { + self.as_ref().to_i420() + } + + fn to_argb( + &self, + format: VideoFormatType, + dst: &mut [u8], + dst_stride: i32, + dst_width: i32, + dst_height: i32, + ) -> Result<(), self::native::ConvertError> { + self.as_ref() + .to_argb(format, dst, dst_stride, dst_width, dst_height) + } +} + +impl VideoFrameBuffer for Box { + fn width(&self) -> i32 { + self.as_ref().width() + } + + fn height(&self) -> i32 { + self.as_ref().height() + } + + fn buffer_type(&self) -> VideoFrameBufferType { + self.as_ref().buffer_type() + } +} + +#[cfg(target_arch = "wasm32")] +pub mod web { + use super::VideoFrameBuffer; + + #[derive(Debug)] + pub struct WebGlBuffer {} + + impl VideoFrameBuffer for WebGlBuffer {} +} diff --git a/livekit-webrtc/src/video_frame_buffer.rs b/livekit-webrtc/src/video_frame_buffer.rs deleted file mode 100644 index d3d1659..0000000 --- a/livekit-webrtc/src/video_frame_buffer.rs +++ /dev/null @@ -1,578 +0,0 @@ -use cxx::UniquePtr; -use livekit_utils::enum_dispatch; -use std::pin::Pin; -use std::slice; -use webrtc_sys::video_frame_buffer as vfb_sys; - -use crate::yuv_helper::{self, ConvertError}; - -macro_rules! recursive_cast { - ($ptr:expr $(, $fnc:ident)*) => { - { - let ptr = $ptr; - $( - let ptr = unsafe { vfb_sys::ffi::$fnc(ptr) }; - )* - ptr - } - }; -} - -#[derive(Debug)] -pub enum VideoFrameBufferType { - Native, - I420, - I420A, - I422, - I444, - I010, - NV12, -} - -// types to convert to -#[derive(Debug)] -pub enum VideoFormatType { - ARGB, - BGRA, - ABGR, - RGBA, -} - -impl From for VideoFrameBufferType { - fn from(buffer_type: vfb_sys::ffi::VideoFrameBufferType) -> Self { - match buffer_type { - vfb_sys::ffi::VideoFrameBufferType::Native => Self::Native, - vfb_sys::ffi::VideoFrameBufferType::I420 => Self::I420, - vfb_sys::ffi::VideoFrameBufferType::I420A => Self::I420A, - vfb_sys::ffi::VideoFrameBufferType::I422 => Self::I422, - vfb_sys::ffi::VideoFrameBufferType::I444 => Self::I444, - vfb_sys::ffi::VideoFrameBufferType::I010 => Self::I010, - vfb_sys::ffi::VideoFrameBufferType::NV12 => Self::NV12, - _ => unreachable!(), - } - } -} - -pub trait VideoFrameBufferTrait { - fn buffer_type(&self) -> VideoFrameBufferType; // Useful for the FFI - fn width(&self) -> i32; - fn height(&self) -> i32; - fn to_i420(self) -> I420Buffer; -} - -pub trait PlanarYuvBuffer: VideoFrameBufferTrait { - fn chroma_width(&self) -> i32; - fn chroma_height(&self) -> i32; - fn stride_y(&self) -> i32; - fn stride_u(&self) -> i32; - fn stride_v(&self) -> i32; -} - -pub trait PlanarYuv8Buffer: PlanarYuvBuffer { - fn data_y(&self) -> &[u8]; - fn data_u(&self) -> &[u8]; - fn data_v(&self) -> &[u8]; -} - -pub trait PlanarYuv16BBuffer: PlanarYuvBuffer { - fn data_y(&self) -> &[u16]; - fn data_u(&self) -> &[u16]; - fn data_v(&self) -> &[u16]; -} - -pub trait BiplanarYuvBuffer: VideoFrameBufferTrait { - fn chroma_width(&self) -> i32; - fn chroma_height(&self) -> i32; - fn stride_y(&self) -> i32; - fn stride_uv(&self) -> i32; -} - -pub trait BiplanarYuv8Buffer: BiplanarYuvBuffer { - fn data_y(&self) -> &[u8]; - fn data_uv(&self) -> &[u8]; -} - -pub enum VideoFrameBuffer { - Native(NativeBuffer), - I420(I420Buffer), - I420A(I420ABuffer), - I422(I422Buffer), - I444(I444Buffer), - I010(I010Buffer), - NV12(NV12Buffer), -} - -impl VideoFrameBuffer { - pub(crate) fn new(mut cxx_handle: UniquePtr) -> Self { - unsafe { - match cxx_handle.buffer_type().into() { - VideoFrameBufferType::Native => Self::Native(NativeBuffer::from(cxx_handle)), - VideoFrameBufferType::I420 => { - Self::I420(I420Buffer::from(cxx_handle.pin_mut().get_i420())) - } - VideoFrameBufferType::I420A => { - Self::I420A(I420ABuffer::from(cxx_handle.pin_mut().get_i420a())) - } - VideoFrameBufferType::I422 => { - Self::I422(I422Buffer::from(cxx_handle.pin_mut().get_i422())) - } - VideoFrameBufferType::I444 => { - Self::I444(I444Buffer::from(cxx_handle.pin_mut().get_i444())) - } - VideoFrameBufferType::I010 => { - Self::I010(I010Buffer::from(cxx_handle.pin_mut().get_i010())) - } - VideoFrameBufferType::NV12 => { - Self::NV12(NV12Buffer::from(cxx_handle.pin_mut().get_nv12())) - } - } - } - } - - #[allow(unused_unsafe)] - pub(crate) fn release(self) -> UniquePtr { - unsafe { - match self { - VideoFrameBuffer::Native(native) => native.release(), - VideoFrameBuffer::I420(i420) => UniquePtr::from_raw(recursive_cast!( - i420.release().into_raw(), - i420_to_yuv8, - yuv8_to_yuv, - yuv_to_vfb - ) as *mut _), - VideoFrameBuffer::I420A(i420a) => UniquePtr::from_raw(recursive_cast!( - i420a.release().into_raw(), - i420a_to_yuv8, - yuv8_to_yuv, - yuv_to_vfb - ) as *mut _), - VideoFrameBuffer::I422(i422) => UniquePtr::from_raw(recursive_cast!( - i422.release().into_raw(), - i422_to_yuv8, - yuv8_to_yuv, - yuv_to_vfb - ) as *mut _), - VideoFrameBuffer::I444(i444) => UniquePtr::from_raw(recursive_cast!( - i444.release().into_raw(), - i444_to_yuv8, - yuv8_to_yuv, - yuv_to_vfb - ) as *mut _), - VideoFrameBuffer::I010(i010) => UniquePtr::from_raw(recursive_cast!( - i010.release().into_raw(), - i010_to_yuv16b, - yuv16b_to_yuv, - yuv_to_vfb - ) as *mut _), - VideoFrameBuffer::NV12(nv12) => UniquePtr::from_raw(recursive_cast!( - nv12.release().into_raw(), - nv12_to_biyuv8, - biyuv8_to_biyuv, - biyuv_to_vfb - ) as *mut _), - } - } - } - - pub fn to_argb( - &self, - format: VideoFormatType, - dst: &mut [u8], - dst_stride: i32, - dst_width: i32, - dst_height: i32, - ) -> Result<(), ConvertError> { - match self { - Self::I420(i420) => match format { - VideoFormatType::ARGB => yuv_helper::i420_to_argb( - i420.data_y(), - i420.stride_y(), - i420.data_u(), - i420.stride_u(), - i420.data_v(), - i420.stride_v(), - dst, - dst_stride, - dst_width, - dst_height, - )?, - VideoFormatType::BGRA => yuv_helper::i420_to_bgra( - i420.data_y(), - i420.stride_y(), - i420.data_u(), - i420.stride_u(), - i420.data_v(), - i420.stride_v(), - dst, - dst_stride, - dst_width, - dst_height, - )?, - VideoFormatType::ABGR => yuv_helper::i420_to_abgr( - i420.data_y(), - i420.stride_y(), - i420.data_u(), - i420.stride_u(), - i420.data_v(), - i420.stride_v(), - dst, - dst_stride, - dst_width, - dst_height, - )?, - VideoFormatType::RGBA => yuv_helper::i420_to_rgba( - i420.data_y(), - i420.stride_y(), - i420.data_u(), - i420.stride_u(), - i420.data_v(), - i420.stride_v(), - dst, - dst_stride, - dst_width, - dst_height, - )?, - }, - _ => { - // TODO(theomonnom): Support other buffer types - } - }; - - Ok(()) - } -} - -impl VideoFrameBufferTrait for VideoFrameBuffer { - enum_dispatch!( - [Native, I420, I420A, I422, I444, I010, NV12] - fnc!(buffer_type, &Self, [], VideoFrameBufferType); - fnc!(width, &Self, [], i32); - fnc!(height, &Self, [], i32); - fnc!(to_i420, Self, [], I420Buffer); - ); -} - -macro_rules! impl_video_frame_buffer { - ($x:ty $(, $cast:ident)*) => { - - // Allow unused_unsafe when we don't do any cast ( e.g. NativeBuffer ) - #[allow(unused_unsafe)] - impl VideoFrameBufferTrait for $x { - fn buffer_type(&self) -> VideoFrameBufferType { - let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); - unsafe { - (*ptr).buffer_type().into() - } - } - - fn width(&self) -> i32 { - let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); - unsafe { - (*ptr).width() - } - } - - fn height(&self) -> i32 { - let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); - unsafe { - (*ptr).height() - } - } - - // Require ownership because libwebrtc uses the same pointers - fn to_i420(self) -> I420Buffer { - let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*) - as *const vfb_sys::ffi::VideoFrameBuffer - as *mut vfb_sys::ffi::VideoFrameBuffer; - - unsafe { - I420Buffer::from(Pin::new_unchecked(&mut *ptr).to_i420()) - } - } - } - }; -} - -macro_rules! impl_yuv_buffer { - ($x:ty $(, $cast:ident)*) => { - impl PlanarYuvBuffer for $x { - fn chroma_width(&self) -> i32 { - let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); - unsafe { - (*ptr).chroma_width() - } - } - - fn chroma_height(&self) -> i32 { - let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); - unsafe { - (*ptr).chroma_height() - } - } - - fn stride_y(&self) -> i32 { - let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); - unsafe { - (*ptr).stride_y() - } - } - - fn stride_u(&self) -> i32 { - let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); - unsafe { - (*ptr).stride_u() - } - } - - fn stride_v(&self) -> i32 { - let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); - unsafe { - (*ptr).stride_v() - } - } - } - }; -} - -macro_rules! impl_yuv8_buffer { - ($x:ty $(, $cast:ident)*) => { - impl PlanarYuv8Buffer for $x { - fn data_y(&self) -> &[u8] { - let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); - unsafe { - slice::from_raw_parts((*ptr).data_y(), (self.width() * self.height()) as usize) - } - } - - fn data_u(&self) -> &[u8] { - let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); - unsafe { - let chroma_height = (self.height() + 1) / 2; - slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * chroma_height) as usize) - } - } - - fn data_v(&self) -> &[u8] { - let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); - unsafe { - let chroma_height = (self.height() + 1) / 2; - slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * chroma_height) as usize) - } - } - } - }; -} - -macro_rules! impl_yuv16_buffer { - ($x:ty $(, $cast:ident)*) => { - impl PlanarYuv16BBuffer for $x { - fn data_y(&self) -> &[u16] { - let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); - unsafe { - slice::from_raw_parts((*ptr).data_y(), (self.width() * self.height()) as usize) - } - } - - fn data_u(&self) -> &[u16] { - let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); - unsafe { - let chroma_height = (self.height() + 1) / 2; - slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * chroma_height) as usize) - } - } - - fn data_v(&self) -> &[u16] { - let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); - unsafe { - let chroma_height = (self.height() + 1) / 2; - slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * chroma_height) as usize) - } - } - } - }; -} - -macro_rules! impl_biyuv_buffer { - ($x:ty $(, $cast:ident)*) => { - impl BiplanarYuvBuffer for $x { - fn chroma_width(&self) -> i32 { - let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); - unsafe { - (*ptr).chroma_width() - } - } - - fn chroma_height(&self) -> i32 { - let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); - unsafe { - (*ptr).chroma_height() - } - } - - fn stride_y(&self) -> i32 { - let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); - unsafe { - (*ptr).stride_y() - } - } - - fn stride_uv(&self) -> i32 { - let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); - unsafe { - (*ptr).stride_uv() - } - } - } - }; -} - -macro_rules! impl_biyuv8_buffer { - ($x:ty $(, $cast:ident)*) => { - impl BiplanarYuv8Buffer for $x { - fn data_y(&self) -> &[u8] { - let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); - unsafe { - slice::from_raw_parts((*ptr).data_y(), (self.width() * self.height()) as usize) - } - } - - fn data_uv(&self) -> &[u8] { - let ptr = recursive_cast!(&*self.cxx_handle $(, $cast)*); - unsafe { - let chroma_height = (self.height() + 1) / 2; - slice::from_raw_parts((*ptr).data_uv(), (self.stride_uv() * chroma_height) as usize) - } - } - } - }; -} - -pub struct NativeBuffer { - cxx_handle: UniquePtr, -} - -pub struct I420Buffer { - cxx_handle: UniquePtr, -} - -pub struct I420ABuffer { - cxx_handle: UniquePtr, -} - -pub struct I422Buffer { - cxx_handle: UniquePtr, -} - -pub struct I444Buffer { - cxx_handle: UniquePtr, -} - -pub struct I010Buffer { - cxx_handle: UniquePtr, -} - -pub struct NV12Buffer { - cxx_handle: UniquePtr, -} - -impl_video_frame_buffer!(NativeBuffer); -impl_video_frame_buffer!(I420Buffer, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb); -impl_video_frame_buffer!(I420ABuffer, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb); -impl_video_frame_buffer!(I422Buffer, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb); -impl_video_frame_buffer!(I444Buffer, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb); -impl_video_frame_buffer!(I010Buffer, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb); -impl_video_frame_buffer!(NV12Buffer, nv12_to_biyuv8, biyuv8_to_biyuv, biyuv_to_vfb); - -impl_yuv_buffer!(I420Buffer, i420_to_yuv8, yuv8_to_yuv); -impl_yuv_buffer!(I420ABuffer, i420a_to_yuv8, yuv8_to_yuv); -impl_yuv_buffer!(I422Buffer, i422_to_yuv8, yuv8_to_yuv); -impl_yuv_buffer!(I444Buffer, i444_to_yuv8, yuv8_to_yuv); -impl_yuv_buffer!(I010Buffer, i010_to_yuv16b, yuv16b_to_yuv); - -impl_yuv8_buffer!(I420Buffer, i420_to_yuv8); -impl_yuv8_buffer!(I420ABuffer, i420a_to_yuv8); -impl_yuv8_buffer!(I422Buffer, i422_to_yuv8); -impl_yuv8_buffer!(I444Buffer, i444_to_yuv8); - -impl_yuv16_buffer!(I010Buffer, i010_to_yuv16b); - -impl_biyuv_buffer!(NV12Buffer, nv12_to_biyuv8, biyuv8_to_biyuv); - -impl_biyuv8_buffer!(NV12Buffer, nv12_to_biyuv8); - -impl NativeBuffer { - fn from(cxx_handle: UniquePtr) -> Self { - Self { cxx_handle } - } - - pub(crate) fn release(self) -> UniquePtr { - self.cxx_handle - } -} - -impl I420Buffer { - pub fn new(width: u32, height: u32) -> Self { - Self::from(vfb_sys::ffi::create_i420_buffer( - width as i32, - height as i32, - )) - } - - fn from(cxx_handle: UniquePtr) -> Self { - Self { cxx_handle } - } - - pub(crate) fn release(self) -> UniquePtr { - self.cxx_handle - } -} - -impl I420ABuffer { - fn from(cxx_handle: UniquePtr) -> Self { - Self { cxx_handle } - } - - pub(crate) fn release(self) -> UniquePtr { - self.cxx_handle - } -} - -impl I422Buffer { - fn from(cxx_handle: UniquePtr) -> Self { - Self { cxx_handle } - } - - pub(crate) fn release(self) -> UniquePtr { - self.cxx_handle - } -} - -impl I444Buffer { - fn from(cxx_handle: UniquePtr) -> Self { - Self { cxx_handle } - } - - pub(crate) fn release(self) -> UniquePtr { - self.cxx_handle - } -} - -impl I010Buffer { - fn from(cxx_handle: UniquePtr) -> Self { - Self { cxx_handle } - } - - pub(crate) fn release(self) -> UniquePtr { - self.cxx_handle - } -} - -impl NV12Buffer { - fn from(cxx_handle: UniquePtr) -> Self { - Self { cxx_handle } - } - - pub(crate) fn release(self) -> UniquePtr { - self.cxx_handle - } -} diff --git a/livekit-webrtc/src/video_source.rs b/livekit-webrtc/src/video_source.rs new file mode 100644 index 0000000..d6b3b6f --- /dev/null +++ b/livekit-webrtc/src/video_source.rs @@ -0,0 +1,28 @@ +use crate::imp::video_source as vs_imp; + +#[cfg(not(target_arch = "wasm32"))] +pub mod native { + use super::vs_imp; + use crate::video_frame::{VideoFrame, VideoFrameBuffer}; + use std::fmt::{Debug, Formatter}; + + #[derive(Default, Clone)] + pub struct NativeVideoSource { + pub(crate) handle: vs_imp::NativeVideoSource, + } + + impl Debug for NativeVideoSource { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + f.debug_struct("NativeVideoSource").finish() + } + } + + impl NativeVideoSource { + pub fn capture_frame(&self, frame: &VideoFrame) { + self.handle.capture_frame(frame) + } + } +} + +#[cfg(target_arch = "wasm32")] +pub mod web {} diff --git a/livekit-webrtc/src/video_stream.rs b/livekit-webrtc/src/video_stream.rs new file mode 100644 index 0000000..f80b9ff --- /dev/null +++ b/livekit-webrtc/src/video_stream.rs @@ -0,0 +1,54 @@ +use crate::imp::video_stream as stream_imp; + +// There is no shared sink between native and web platforms. +// Each platform requires different configuration (e.g: WebGlContext, ..) + +#[cfg(not(target_arch = "wasm32"))] +pub mod native { + use super::stream_imp; + use crate::media_stream::RtcVideoTrack; + use crate::video_frame::BoxVideoFrame; + use futures::stream::Stream; + use std::fmt::Debug; + use std::pin::Pin; + use std::task::{Context, Poll}; + + pub struct NativeVideoStream { + pub(crate) handle: stream_imp::NativeVideoStream, + } + + impl Debug for NativeVideoStream { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.debug_struct("NativeVideoStream") + .field("track", &self.track()) + .finish() + } + } + + impl NativeVideoStream { + pub fn new(video_track: RtcVideoTrack) -> Self { + Self { + handle: stream_imp::NativeVideoStream::new(video_track), + } + } + + pub fn track(&self) -> RtcVideoTrack { + self.handle.track() + } + + pub fn close(&mut self) { + self.handle.close(); + } + } + + impl Stream for NativeVideoStream { + type Item = BoxVideoFrame; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + Pin::new(&mut self.get_mut().handle).poll_next(cx) + } + } +} + +#[cfg(target_arch = "wasm32")] +pub mod web {} diff --git a/livekit-webrtc/src/web/data_channel.rs b/livekit-webrtc/src/web/data_channel.rs new file mode 100644 index 0000000..dcd330e --- /dev/null +++ b/livekit-webrtc/src/web/data_channel.rs @@ -0,0 +1,103 @@ +use core::str; +use std::cell::RefCell; +use std::rc::Rc; +use wasm_bindgen::prelude::*; +use web_sys::{MessageEvent, RtcDataChannelEvent, RtcDataChannelState}; + +use crate::data_channel::{ + DataChannelError, DataChannelTrait, DataState, OnBufferedAmountChange, OnMessage, OnStateChange, +}; + +impl From for DataState { + fn from(value: RtcDataChannelState) -> Self { + match value { + RtcDataChannelState::Connecting => Self::Connecting, + RtcDataChannelState::Open => Self::Open, + RtcDataChannelState::Closing => Self::Closing, + RtcDataChannelState::Closed => Self::Closed, + _ => panic!("unknown data channel state"), + } + } +} + +#[derive(Clone)] +pub struct DataChannel { + sys_handle: web_sys::RtcDataChannel, + on_closing: Rc>>, +} + +impl DataChannelTrait for DataChannel { + fn send(&self, data: &[u8], binary: bool) -> Result<(), DataChannelError> { + if binary { + self.sys_handle + .send_with_u8_array(data) + .map_err(|_| DataChannelError::Send) + } else { + let utf8 = str::from_utf8(data)?; + self.sys_handle + .send_with_str(utf8) + .map_err(|_| DataChannelError::Send) + } + } + + fn label(&self) -> String { + self.sys_handle.label() + } + + fn state(&self) -> DataState { + self.sys_handle.ready_state().into() + } + + fn close(&self) { + self.sys_handle.close(); + } + + fn on_state_change(&self, callback: Option) { + if let Some(mut callback) = callback { + let dc = self.clone(); + let js_callback = Closure::new(move |_: RtcDataChannelEvent| { + callback(dc.state()); + }); + let js_callback = js_callback.into_js_value(); + self.sys_handle + .set_onopen(Some(js_callback.unchecked_ref())); + self.sys_handle + .set_onclose(Some(js_callback.unchecked_ref())); + self.sys_handle + .add_event_listener_with_callback("closing", js_callback.unchecked_ref()) + .unwrap(); + + self.on_closing.replace(Some(js_callback)); + } else { + self.sys_handle.set_onopen(None); + self.sys_handle.set_onclose(None); + if let Some(on_closing) = self.on_closing.take() { + self.sys_handle + .remove_event_listener_with_callback("closing", on_closing.unchecked_ref()) + .unwrap(); + } + self.on_closing.replace(None); + } + } + + fn on_message(&self, callback: Option) { + let js_callback = callback.map(|mut callback| { + Closure::new(move |event: MessageEvent| { + if let Some(str) = event.as_string() { + callback(str.as_bytes(), false); + } + }) + .into_js_value() + }); + + self.sys_handle.set_onmessage( + js_callback + .as_ref() + .map(|callback| callback.unchecked_ref()), + ); + } + + fn on_buffered_amount_change(&self, _callback: Option) { + todo!("onbufferedamountlow instead?") + } +} diff --git a/livekit-webrtc/src/web/media_stream.rs b/livekit-webrtc/src/web/media_stream.rs new file mode 100644 index 0000000..4d8aa3d --- /dev/null +++ b/livekit-webrtc/src/web/media_stream.rs @@ -0,0 +1,358 @@ +use super::VideoTrack; +use crate::{ + media_stream::{ + BiplanarYuv8Buffer, BiplanarYuvBuffer, PlanarYuv16BBuffer, PlanarYuv8Buffer, + PlanarYuvBuffer, VideoFrameBuffer, + }, + video_frame::{BiplanarYuv8Buffer, I420Buffer, SinkError, VideoFrame, VideoFrameBuffer}, + I010Buffer, I420ABuffer, I422Buffer, I444Buffer, NV12Buffer, +}; +use std::sync::mpsc; +use web_sys::{WebGlRenderingContext, WebGlTexture}; + +#[derive(Debug)] +pub struct WebGlVideoSink { + track: Arc, + gl_ctx: WebGlRenderingContext, + tex: WebGlTexture, +} + +/// Create a new WebGL texture and update it inside requestAnimationFrame +impl WebGlVideoSink { + pub fn new( + track: Arc, + gl_ctx: WebGlRenderingContext, + ) -> Result<(Self, mpsc::Receiver>), SinkError> { + let (sender, receiver) = mpsc::channel(); + let tex = gl_ctx.create_texture()?; + Ok((Self { track, gl_ctx, tex }, receiver)) + } +} + +#[derive(Debug, Clone)] +pub struct WebGlBuffer { + width: i32, + height: i32, + tex: WebGlTexture, +} + +impl VideoFrameBuffer for WebGlBuffer { + fn width(&self) -> i32 { + self.width + } + + fn height(&self) -> i32 { + self.height + } +} + +/// The following types could be implemented if we want +/// to support VideoFrame with WebCodecs + +#[derive(Debug)] +pub struct I420Buffer {} + +#[derive(Debug)] +pub struct I420ABuffer {} + +#[derive(Debug)] +pub struct I422Buffer {} + +#[derive(Debug)] +pub struct I444Buffer {} + +#[derive(Debug)] +pub struct I010Buffer {} + +#[derive(Debug)] +pub struct NV12Buffer {} + +impl VideoFrameBuffer for I420Buffer { + fn width(&self) -> i32 { + unimplemented!() + } + + fn height(&self) -> i32 { + unimplemented!() + } +} + +impl VideoFrameBuffer for I420ABuffer { + fn width(&self) -> i32 { + unimplemented!() + } + + fn height(&self) -> i32 { + unimplemented!() + } +} + +impl VideoFrameBuffer for I422Buffer { + fn width(&self) -> i32 { + unimplemented!() + } + + fn height(&self) -> i32 { + unimplemented!() + } +} + +impl VideoFrameBuffer for I444Buffer { + fn width(&self) -> i32 { + unimplemented!() + } + + fn height(&self) -> i32 { + unimplemented!() + } +} + +impl VideoFrameBuffer for I010Buffer { + fn width(&self) -> i32 { + unimplemented!() + } + + fn height(&self) -> i32 { + unimplemented!() + } +} + +impl VideoFrameBuffer for NV12Buffer { + fn width(&self) -> i32 { + unimplemented!() + } + + fn height(&self) -> i32 { + unimplemented!() + } +} + +impl PlanarYuvBuffer for I420Buffer { + fn chroma_width(&self) -> i32 { + unimplemented!() + } + + fn chroma_height(&self) -> i32 { + unimplemented!() + } + + fn stride_y(&self) -> i32 { + unimplemented!() + } + + fn stride_u(&self) -> i32 { + unimplemented!() + } + + fn stride_v(&self) -> i32 { + unimplemented!() + } +} + +impl PlanarYuvBuffer for I420ABuffer { + fn chroma_width(&self) -> i32 { + unimplemented!() + } + + fn chroma_height(&self) -> i32 { + unimplemented!() + } + + fn stride_y(&self) -> i32 { + unimplemented!() + } + + fn stride_u(&self) -> i32 { + unimplemented!() + } + + fn stride_v(&self) -> i32 { + unimplemented!() + } +} + +impl PlanarYuvBuffer for I422Buffer { + fn chroma_width(&self) -> i32 { + unimplemented!() + } + + fn chroma_height(&self) -> i32 { + unimplemented!() + } + + fn stride_y(&self) -> i32 { + unimplemented!() + } + + fn stride_u(&self) -> i32 { + unimplemented!() + } + + fn stride_v(&self) -> i32 { + unimplemented!() + } +} + +impl PlanarYuvBuffer for I444Buffer { + fn chroma_width(&self) -> i32 { + unimplemented!() + } + + fn chroma_height(&self) -> i32 { + unimplemented!() + } + + fn stride_y(&self) -> i32 { + unimplemented!() + } + + fn stride_u(&self) -> i32 { + unimplemented!() + } + + fn stride_v(&self) -> i32 { + unimplemented!() + } +} + +impl PlanarYuvBuffer for I010Buffer { + fn chroma_width(&self) -> i32 { + unimplemented!() + } + + fn chroma_height(&self) -> i32 { + unimplemented!() + } + + fn stride_y(&self) -> i32 { + unimplemented!() + } + + fn stride_u(&self) -> i32 { + unimplemented!() + } + + fn stride_v(&self) -> i32 { + unimplemented!() + } +} + +impl PlanarYuvBuffer for NV12Buffer { + fn chroma_width(&self) -> i32 { + unimplemented!() + } + + fn chroma_height(&self) -> i32 { + unimplemented!() + } + + fn stride_y(&self) -> i32 { + unimplemented!() + } + + fn stride_u(&self) -> i32 { + unimplemented!() + } + + fn stride_v(&self) -> i32 { + unimplemented!() + } +} + +impl PlanarYuv8Buffer for I420Buffer { + fn data_y(&self) -> &[u8] { + unimplemented!() + } + + fn data_u(&self) -> &[u8] { + unimplemented!() + } + + fn data_v(&self) -> &[u8] { + unimplemented!() + } +} + +impl PlanarYuv8Buffer for I420ABuffer { + fn data_y(&self) -> &[u8] { + unimplemented!() + } + + fn data_u(&self) -> &[u8] { + unimplemented!() + } + + fn data_v(&self) -> &[u8] { + unimplemented!() + } +} + +impl PlanarYuv8Buffer for I422Buffer { + fn data_y(&self) -> &[u8] { + unimplemented!() + } + + fn data_u(&self) -> &[u8] { + unimplemented!() + } + + fn data_v(&self) -> &[u8] { + unimplemented!() + } +} + +impl PlanarYuv8Buffer for I444Buffer { + fn data_y(&self) -> &[u8] { + unimplemented!() + } + + fn data_u(&self) -> &[u8] { + unimplemented!() + } + + fn data_v(&self) -> &[u8] { + unimplemented!() + } +} + +impl PlanarYuv16BBuffer for I010Buffer { + fn data_y(&self) -> &[u16] { + unimplemented!() + } + + fn data_u(&self) -> &[u16] { + unimplemented!() + } + + fn data_v(&self) -> &[u16] { + unimplemented!() + } +} + +impl BiplanarYuvBuffer for NV12Buffer { + fn chroma_width(&self) -> i32 { + unimplemented!() + } + + fn chroma_height(&self) -> i32 { + unimplemented!() + } + + fn stride_y(&self) -> i32 { + unimplemented!() + } + + fn stride_uv(&self) -> i32 { + unimplemented!() + } +} + +impl BiplanarYuv8Buffer for NV12Buffer { + fn data_y(&self) -> &[u8] { + unimplemented!() + } + + fn data_uv(&self) -> &[u8] { + unimplemented!() + } +} diff --git a/livekit-webrtc/src/web/mod.rs b/livekit-webrtc/src/web/mod.rs new file mode 100644 index 0000000..77d30de --- /dev/null +++ b/livekit-webrtc/src/web/mod.rs @@ -0,0 +1 @@ +unimplemented!() diff --git a/livekit-webrtc/src/webrtc.rs b/livekit-webrtc/src/webrtc.rs deleted file mode 100644 index 483f5c7..0000000 --- a/livekit-webrtc/src/webrtc.rs +++ /dev/null @@ -1,24 +0,0 @@ -use cxx::SharedPtr; - -use webrtc_sys::webrtc as sys_rtc; - -pub use sys_rtc::ffi::MediaType; -pub use sys_rtc::ffi::Priority; -pub use sys_rtc::ffi::RtpTransceiverDirection; - -#[derive(Clone)] -pub struct RTCRuntime { - cxx_handle: SharedPtr, -} - -impl RTCRuntime { - pub fn new() -> Self { - Self { - cxx_handle: sys_rtc::ffi::create_rtc_runtime(), - } - } - - pub(crate) fn release(self) -> SharedPtr { - self.cxx_handle - } -} diff --git a/livekit-webrtc/src/yuv_helper.rs b/livekit-webrtc/src/yuv_helper.rs deleted file mode 100644 index 87d5543..0000000 --- a/livekit-webrtc/src/yuv_helper.rs +++ /dev/null @@ -1,98 +0,0 @@ -use thiserror::Error; - -use webrtc_sys::yuv_helper as yuv_sys; - -#[derive(Error, Debug)] -pub enum ConvertError { - #[error("conversion failed: {0}")] - Convert(&'static str), -} - -fn i420_safety( - src_y: &[u8], - src_stride_y: i32, - src_u: &[u8], - src_stride_u: i32, - src_v: &[u8], - src_stride_v: i32, - dst: &mut [u8], - dst_stride: i32, - _width: i32, - height: i32, -) -> Result<(), ConvertError> { - let chroma_height = (height + 1) / 2; - let min_y = (src_stride_y * height) as usize; - let min_u = (src_stride_u * chroma_height) as usize; - let min_v = (src_stride_v * chroma_height) as usize; - let min_dst = (dst_stride * height) as usize; - - if src_y.len() < min_y { - return Err(ConvertError::Convert("src_y isn't large enough")); - } - - if src_u.len() < min_u { - return Err(ConvertError::Convert("src_u isn't large enough")); - } - - if src_v.len() < min_v { - return Err(ConvertError::Convert("src_v isn't large enough")); - } - - if dst.len() < min_dst { - return Err(ConvertError::Convert("dst isn't large enough")); - } - - Ok(()) -} - -macro_rules! i420_to_x { - ($x:ident) => { - pub fn $x( - src_y: &[u8], - src_stride_y: i32, - src_u: &[u8], - src_stride_u: i32, - src_v: &[u8], - src_stride_v: i32, - dst: &mut [u8], - dst_stride: i32, - width: i32, - height: i32, - ) -> Result<(), ConvertError> { - i420_safety( - src_y, - src_stride_y, - src_u, - src_stride_u, - src_v, - src_stride_v, - dst, - dst_stride, - width, - height, - )?; - - unsafe { - yuv_sys::ffi::$x( - src_y.as_ptr(), - src_stride_y, - src_u.as_ptr(), - src_stride_u, - src_v.as_ptr(), - src_stride_v, - dst.as_mut_ptr(), - dst_stride, - width, - height, - ); - } - - Ok(()) - } - }; -} - -i420_to_x!(i420_to_argb); -i420_to_x!(i420_to_bgra); -i420_to_x!(i420_to_abgr); -i420_to_x!(i420_to_rgba); diff --git a/livekit/Cargo.toml b/livekit/Cargo.toml index 76425ab..4ee9f9c 100644 --- a/livekit/Cargo.toml +++ b/livekit/Cargo.toml @@ -11,9 +11,9 @@ livekit-webrtc = { path = "../livekit-webrtc", version = "0.1.1" } livekit-utils = { path = "../livekit-utils", version = "0.1.1" } serde = { version = "1", features = ["derive"] } serde_json = "1.0" -tokio-tungstenite = { version = "0.17.2", features = ["native-tls"] } +tokio-tungstenite = { version = "0.18", features = ["native-tls"] } tokio = { version = "1", features = ["full"] } -futures = "0.3" +tokio-stream = "0.1" parking_lot = { version = "0.12.1", features = ["send_guard"] } url = "2.2.2" futures-util = "0.3.23" diff --git a/livekit/src/lib.rs b/livekit/src/lib.rs index d6f131a..1b1e748 100644 --- a/livekit/src/lib.rs +++ b/livekit/src/lib.rs @@ -1,12 +1,9 @@ extern crate core; -pub mod proto { - include!(concat!(env!("OUT_DIR"), "/livekit.rs")); -} - +pub mod proto; +mod room; mod rtc_engine; mod signal_client; -mod room; pub mod webrtc { pub use livekit_webrtc::*; diff --git a/livekit/src/prelude.rs b/livekit/src/prelude.rs index 9958e97..f8c204b 100644 --- a/livekit/src/prelude.rs +++ b/livekit/src/prelude.rs @@ -1,31 +1,12 @@ -pub use crate::participant::{ - LocalParticipant, Participant, ParticipantEvent, ParticipantTrait, RemoteParticipant, -}; +pub use crate::participant::{LocalParticipant, Participant, ParticipantEvent, RemoteParticipant}; -pub use crate::{ConnectionState, Room, RoomError, RoomEvent, RoomSession}; +pub use crate::{ConnectionState, Room, RoomError, RoomEvent, RoomResult, RoomSession}; -pub use crate::publication::{ - LocalTrackPublication, RemoteTrackPublication, TrackPublication, TrackPublicationTrait, -}; +pub use crate::publication::{LocalTrackPublication, RemoteTrackPublication, TrackPublication}; pub use crate::track::{ - AudioTrackHandle, LocalAudioTrack, LocalTrackHandle, LocalVideoTrack, RemoteAudioTrack, - RemoteTrackHandle, RemoteVideoTrack, StreamState, TrackEvent, TrackHandle, TrackKind, - TrackSource, TrackTrait, VideoTrackHandle, + AudioTrack, LocalAudioTrack, LocalTrack, LocalVideoTrack, RemoteAudioTrack, RemoteTrack, + RemoteVideoTrack, StreamState, Track, TrackEvent, TrackKind, TrackSource, VideoTrack, }; pub use crate::id::*; - -pub use crate::webrtc::{ - data_channel::DataChannel, - media_stream::{ - AudioTrack, MediaStream, MediaStreamTrackHandle, MediaStreamTrackTrait, - OnConstraintsChangedHandler, OnDiscardedFrameHandler, OnFrameHandler, VideoTrack, - }, - rtp_receiver::RtpReceiver, - rtp_transceiver::RtpTransceiver, - video_frame::{VideoFrame, VideoRotation}, - video_frame_buffer::{ - VideoFormatType, VideoFrameBuffer, VideoFrameBufferTrait, VideoFrameBufferType, - }, -}; diff --git a/livekit/src/proto.rs b/livekit/src/proto.rs new file mode 100644 index 0000000..52f8504 --- /dev/null +++ b/livekit/src/proto.rs @@ -0,0 +1,49 @@ +use crate::track; + +include!(concat!(env!("OUT_DIR"), "/livekit.rs")); + +// Conversions +impl TryFrom for track::TrackKind { + type Error = &'static str; + + fn try_from(r#type: TrackType) -> Result { + match r#type { + TrackType::Audio => Ok(Self::Audio), + TrackType::Video => Ok(Self::Video), + TrackType::Data => Err("data tracks are not implemented yet"), + } + } +} + +impl From for TrackType { + fn from(kind: track::TrackKind) -> Self { + match kind { + track::TrackKind::Audio => Self::Audio, + track::TrackKind::Video => Self::Video, + } + } +} + +impl From for track::TrackSource { + fn from(source: TrackSource) -> Self { + match source { + TrackSource::Camera => Self::Camera, + TrackSource::Microphone => Self::Microphone, + TrackSource::ScreenShare => Self::Screenshare, + TrackSource::ScreenShareAudio => Self::ScreenshareAudio, + TrackSource::Unknown => Self::Unknown, + } + } +} + +impl From for TrackSource { + fn from(source: track::TrackSource) -> Self { + match source { + track::TrackSource::Camera => Self::Camera, + track::TrackSource::Microphone => Self::Microphone, + track::TrackSource::Screenshare => Self::ScreenShare, + track::TrackSource::ScreenshareAudio => Self::ScreenShareAudio, + track::TrackSource::Unknown => Self::Unknown, + } + } +} diff --git a/livekit/src/room/mod.rs b/livekit/src/room/mod.rs index 56f05f7..1af1dcb 100644 --- a/livekit/src/room/mod.rs +++ b/livekit/src/room/mod.rs @@ -1,3 +1,4 @@ +use self::track::RemoteTrack; use crate::participant::ConnectionQuality; use crate::prelude::*; use crate::proto; @@ -10,6 +11,7 @@ use tokio::sync::mpsc; pub use crate::rtc_engine::SimulateScenario; pub mod id; +pub mod options; pub mod participant; pub mod publication; pub mod room_session; @@ -25,34 +27,36 @@ pub enum RoomError { Engine(#[from] EngineError), #[error("room failure: {0}")] Internal(String), + #[error("this track or a track of the same source is already published")] + TrackAlreadyPublished, } #[derive(Clone, Debug)] pub enum RoomEvent { - ParticipantConnected(Arc), - ParticipantDisconnected(Arc), + ParticipantConnected(RemoteParticipant), + ParticipantDisconnected(RemoteParticipant), TrackSubscribed { - track: RemoteTrackHandle, + track: RemoteTrack, publication: RemoteTrackPublication, - participant: Arc, + participant: RemoteParticipant, }, TrackPublished { publication: RemoteTrackPublication, - participant: Arc, + participant: RemoteParticipant, }, TrackUnpublished { publication: RemoteTrackPublication, - participant: Arc, + participant: RemoteParticipant, }, TrackUnsubscribed { - track: RemoteTrackHandle, + track: RemoteTrack, publication: RemoteTrackPublication, - participant: Arc, + participant: RemoteParticipant, }, TrackSubscriptionFailed { error: track::TrackError, sid: TrackSid, - participant: Arc, + participant: RemoteParticipant, }, TrackMuted { participant: Participant, @@ -72,7 +76,7 @@ pub enum RoomEvent { DataReceived { payload: Arc>, kind: proto::data_packet::Kind, - participant: Arc, + participant: RemoteParticipant, }, ConnectionStateChanged(ConnectionState), Connected, diff --git a/livekit/src/room/observer.rs b/livekit/src/room/observer.rs new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/livekit/src/room/observer.rs @@ -0,0 +1 @@ + diff --git a/livekit/src/room/options.rs b/livekit/src/room/options.rs new file mode 100644 index 0000000..118810a --- /dev/null +++ b/livekit/src/room/options.rs @@ -0,0 +1,372 @@ +use crate::prelude::*; +use crate::proto; +use livekit_webrtc::prelude::*; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum VideoCodec { + VP8, + H264, + AV1, +} + +impl VideoCodec { + pub fn as_str(&self) -> &'static str { + match self { + VideoCodec::VP8 => "vp8", + VideoCodec::H264 => "h264", + VideoCodec::AV1 => "av1", + } + } +} + +#[derive(Debug, Clone)] +pub struct VideoResolution { + pub width: u32, + pub height: u32, + pub frame_rate: f64, + pub aspect_ratio: f32, +} + +#[derive(Debug, Clone)] +pub struct VideoEncoding { + pub max_bitrate: u64, + pub max_framerate: f64, +} + +#[derive(Debug, Clone)] +pub struct VideoPreset { + pub encoding: VideoEncoding, + pub width: u32, + pub height: u32, +} + +#[derive(Debug, Clone)] +pub struct AudioPreset { + pub max_bitrate: u32, +} + +impl AudioPreset { + pub const fn new(max_bitrate: u32) -> Self { + Self { max_bitrate } + } +} + +#[derive(Clone, Debug)] +pub struct VideoCaptureOptions { + pub preset: VideoPreset, +} + +impl Default for VideoCaptureOptions { + fn default() -> Self { + Self { + preset: video::H720, + } + } +} + +#[derive(Clone, Debug)] +pub struct TrackPublishOptions { + pub dynacast: bool, + pub video_codec: VideoCodec, + pub dtx: bool, + pub red: bool, + pub simulcast: bool, + pub screenshare: bool, + pub name: String, + pub source: TrackSource, +} + +impl Default for TrackPublishOptions { + fn default() -> Self { + Self { + dynacast: false, + video_codec: VideoCodec::VP8, + dtx: true, + red: true, + simulcast: true, + screenshare: false, + name: "unnamed track".to_owned(), + source: TrackSource::Unknown, + } + } +} + +impl VideoPreset { + pub const fn new(width: u32, height: u32, max_bitrate: u64, max_framerate: f64) -> Self { + Self { + width, + height, + encoding: VideoEncoding { + max_bitrate, + max_framerate, + }, + } + } + + pub fn resolution(&self) -> VideoResolution { + VideoResolution { + width: self.width, + height: self.height, + frame_rate: self.encoding.max_framerate, + aspect_ratio: self.width as f32 / self.height as f32, + } + } +} + +/// Compute appropriate RtpEncodingParameters from the video resolution. +/// TrackPublishOptions helps to find the most appropriate encodings +pub fn compute_video_encodings( + width: u32, + height: u32, + options: &TrackPublishOptions, +) -> Vec { + let encoding = compute_appropriate_encoding(options.screenshare, width, height); + + let initial_preset = VideoPreset { + width, + height, + encoding: VideoEncoding { + max_bitrate: encoding.max_bitrate, + max_framerate: encoding.max_framerate, + }, + }; + + if !options.simulcast { + return into_rtp_encodings(width, height, &[initial_preset]); + } + + let mut simulcast_presets = + compute_default_simulcast_presets(options.screenshare, &initial_preset); + + let mid_preset = simulcast_presets.pop(); + let low_preset = simulcast_presets.pop(); + + let size = u32::max(width, height); + if size >= 960 && low_preset.is_some() { + return into_rtp_encodings( + width, + height, + &[low_preset.unwrap(), mid_preset.unwrap(), initial_preset], + ); + } else if size >= 480 { + return into_rtp_encodings(width, height, &[mid_preset.unwrap(), initial_preset]); + } + + // Other layers not needed + into_rtp_encodings(width, height, &[initial_preset]) +} + +/// Return an appropriate VideoEncdoding for the specified resolution based on our presets +pub fn compute_appropriate_encoding( + is_screenshare: bool, + width: u32, + height: u32, +) -> VideoEncoding { + let presets = compute_presets_for_resolution(is_screenshare, width, height); + let size = u32::max(width, height); + + for preset in presets { + if preset.width >= size { + return preset.encoding.clone(); + } + } + + unreachable!() +} + +pub fn compute_presets_for_resolution( + is_screenshare: bool, + width: u32, + height: u32, +) -> &'static [VideoPreset] { + if is_screenshare { + return screenshare::PRESETS; + } + + // Check how close width & height are from 16/9 or 4/3 + let ar = landscape_aspect_ratio(width, height); + if f32::abs(ar - 16.0 / 9.0) < f32::abs(ar - 4.0 / 3.0) { + return video::PRESETS; + } + + video43::PRESETS +} + +/// Returns our most appropriate default presets +pub fn compute_default_simulcast_presets( + is_screenshare: bool, + initial: &VideoPreset, +) -> Vec { + if is_screenshare { + return vec![screenshare::compute_default_simulcast_preset(initial)]; + } + + let ar = landscape_aspect_ratio(initial.width, initial.height); + if f32::abs(ar - 16.0 / 9.0) < f32::abs(ar - 4.0 / 3.0) { + return video::DEFAULT_SIMULCAST_PRESETS.to_owned(); + } + + video43::DEFAULT_SIMULCAST_PRESETS.to_owned() +} + +pub fn landscape_aspect_ratio(width: u32, height: u32) -> f32 { + if width > height { + width as f32 / height as f32 + } else { + height as f32 / width as f32 + } +} + +/// Presets must be ordered +pub fn into_rtp_encodings( + initial_width: u32, + initial_height: u32, + presets: &[VideoPreset], +) -> Vec { + let mut encodings = Vec::with_capacity(presets.len()); + let size = u32::min(initial_width, initial_height); + for (i, preset) in presets.iter().enumerate() { + encodings.push(RtpEncodingParameters { + rid: VIDEO_RIDS[i].to_string(), + scale_resolution_down_by: Some(f64::max( + 1.0, + size as f64 / u32::min(preset.width, preset.height) as f64, + )), + max_bitrate: Some(preset.encoding.max_bitrate), + max_framerate: Some(preset.encoding.max_framerate), + ..Default::default() + }) + } + + encodings +} + +pub fn video_quality_for_rid(rid: &str) -> Option { + match rid { + "f" => Some(proto::VideoQuality::High), + "h" => Some(proto::VideoQuality::Medium), + "q" => Some(proto::VideoQuality::Low), + _ => None, + } +} + +pub fn video_layers_from_encodings( + width: u32, + height: u32, + encodings: &[RtpEncodingParameters], +) -> Vec { + if encodings.is_empty() { + return vec![proto::VideoLayer { + quality: proto::VideoQuality::High as i32, + width, + height, + bitrate: 0, + ssrc: 0, + }]; + } + + let mut layers = Vec::with_capacity(encodings.len()); + for encoding in encodings { + let scale = encoding.scale_resolution_down_by.unwrap_or(1.0); + let quality = video_quality_for_rid(&encoding.rid).unwrap_or(proto::VideoQuality::High); + + layers.push(proto::VideoLayer { + quality: quality as i32, + width: (width as f64 / scale) as u32, + height: (height as f64 / scale) as u32, + bitrate: encoding.max_bitrate.unwrap_or(0) as u32, + ssrc: 0, + }); + } + + layers +} + +const VIDEO_RIDS: &[char] = &['q', 'h', 'f']; + +pub mod audio { + use super::AudioPreset; + + pub const TELEPHONE: AudioPreset = AudioPreset::new(12_000); + pub const SPEECH: AudioPreset = AudioPreset::new(20_000); + pub const MUSIC: AudioPreset = AudioPreset::new(32_000); + pub const MUSIC_STEREO: AudioPreset = AudioPreset::new(48_000); + pub const MUSIC_HIGH_QUALITY: AudioPreset = AudioPreset::new(64_000); + pub const MUSIC_HIGH_QUALITY_STEREO: AudioPreset = AudioPreset::new(96_000); + + pub const PRESETS: &[AudioPreset] = &[ + TELEPHONE, + SPEECH, + MUSIC, + MUSIC_STEREO, + MUSIC_HIGH_QUALITY, + MUSIC_HIGH_QUALITY_STEREO, + ]; +} + +pub mod video { + use super::VideoPreset; + + pub const H90: VideoPreset = VideoPreset::new(160, 90, 60_000, 15.0); + pub const H180: VideoPreset = VideoPreset::new(320, 180, 120_000, 15.0); + pub const H216: VideoPreset = VideoPreset::new(384, 216, 180_000, 15.0); + pub const H360: VideoPreset = VideoPreset::new(640, 360, 300_000, 20.0); + pub const H540: VideoPreset = VideoPreset::new(960, 540, 600_000, 25.0); + pub const H720: VideoPreset = VideoPreset::new(1280, 720, 1_700_000, 30.0); + pub const H1080: VideoPreset = VideoPreset::new(1920, 1080, 3_000_000, 30.0); + pub const H1440: VideoPreset = VideoPreset::new(2560, 1440, 5_000_000, 30.0); + pub const H2160: VideoPreset = VideoPreset::new(3840, 2160, 8_000_000, 30.0); + + pub const PRESETS: &[VideoPreset] = &[H90, H180, H216, H360, H540, H720, H1080, H1440, H2160]; + pub const DEFAULT_SIMULCAST_PRESETS: &[VideoPreset] = &[H180, H360]; +} + +pub mod video43 { + use super::VideoPreset; + + pub const H120: VideoPreset = VideoPreset::new(160, 120, 80_000, 15.0); + pub const H180: VideoPreset = VideoPreset::new(240, 180, 100_000, 15.0); + pub const H240: VideoPreset = VideoPreset::new(320, 240, 150_000, 15.0); + pub const H360: VideoPreset = VideoPreset::new(480, 360, 225_000, 20.0); + pub const H480: VideoPreset = VideoPreset::new(640, 480, 300_000, 20.0); + pub const H540: VideoPreset = VideoPreset::new(720, 540, 450_000, 25.0); + pub const H720: VideoPreset = VideoPreset::new(960, 720, 1_500_000, 30.0); + pub const H1080: VideoPreset = VideoPreset::new(1440, 1080, 2_500_000, 30.0); + pub const H1440: VideoPreset = VideoPreset::new(1920, 1440, 3_500_000, 30.0); + + pub const PRESETS: &[VideoPreset] = &[H120, H180, H240, H360, H480, H540, H720, H1080, H1440]; + pub const DEFAULT_SIMULCAST_PRESETS: &[VideoPreset] = &[H180, H360]; +} + +pub mod screenshare { + /// The screenshare presets are optimized for quality. + /// When simulcasting, we prefer to reduce the FPS. + use super::VideoPreset; + + pub const H360_FPS3: VideoPreset = VideoPreset::new(640, 360, 200_000, 3.0); + pub const H720_FPS5: VideoPreset = VideoPreset::new(1280, 720, 400_000, 5.0); + pub const H720_FPS15: VideoPreset = VideoPreset::new(1280, 720, 1_000_000, 15.0); + pub const H1080_FPS15: VideoPreset = VideoPreset::new(1920, 1080, 1_500_000, 15.0); + pub const H1080_FPS30: VideoPreset = VideoPreset::new(1920, 1080, 3_000_000, 30.0); + + pub const PRESETS: &[VideoPreset] = + &[H360_FPS3, H720_FPS5, H720_FPS15, H1080_FPS15, H1080_FPS30]; + + /// Only one additional layer for screenshares. (Prioritize quality) + pub fn compute_default_simulcast_preset(initial: &VideoPreset) -> VideoPreset { + const SCALE_DOWN_FACTOR: u32 = 2; + const FPS: f64 = 3.0; + + VideoPreset::new( + initial.width / SCALE_DOWN_FACTOR, + initial.height / SCALE_DOWN_FACTOR, + u64::max( + 150_000, + initial.encoding.max_bitrate as u64 + / (SCALE_DOWN_FACTOR.pow(2) as u64 + * (initial.encoding.max_framerate / FPS) as u64), + ), + FPS, + ) + } +} diff --git a/livekit/src/room/participant/local_participant.rs b/livekit/src/room/participant/local_participant.rs index bde5c00..fceb405 100644 --- a/livekit/src/room/participant/local_participant.rs +++ b/livekit/src/room/participant/local_participant.rs @@ -1,37 +1,130 @@ -use super::{ - impl_participant_trait, ConnectionQuality, ParticipantInternalTrait, ParticipantShared, -}; +use super::{ConnectionQuality, ParticipantInner}; +use crate::options::compute_video_encodings; +use crate::options::video_layers_from_encodings; +use crate::options::TrackPublishOptions; use crate::prelude::*; use crate::proto; -use crate::publication::TrackPublication; -use crate::rtc_engine::RTCEngine; +use crate::rtc_engine::RtcEngine; use parking_lot::RwLockReadGuard; use std::collections::HashMap; -use std::sync::atomic::Ordering; use std::sync::Arc; use tokio::sync::mpsc; +use tracing::{debug, trace}; -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct LocalParticipant { - shared: ParticipantShared, - rtc_engine: Arc, + inner: Arc, + rtc_engine: Arc, } impl LocalParticipant { pub(crate) fn new( - rtc_engine: Arc, + rtc_engine: Arc, sid: ParticipantSid, identity: ParticipantIdentity, name: String, metadata: String, ) -> Self { Self { - shared: ParticipantShared::new(sid, identity, name, metadata), + inner: Arc::new(ParticipantInner::new(sid, identity, name, metadata)), rtc_engine, } } - pub async fn publish_track(&self) {} + pub async fn publish_track( + &self, + track: LocalTrack, + options: TrackPublishOptions, + ) -> RoomResult { + let mut req = proto::AddTrackRequest { + cid: track.rtc_track().id(), + name: options.name.clone(), + r#type: proto::TrackType::from(track.kind()) as i32, + muted: track.muted(), + source: proto::TrackSource::from(options.source) as i32, + disable_dtx: !options.dtx, + disable_red: !options.red, + ..Default::default() + }; + + let mut encodings = Vec::default(); + match &track { + LocalTrack::Video(video_track) => { + // Get the video dimension + // TODO(theomonnom): Use MediaStreamTrack::getSettings() on web + let capture_options = video_track.capture_options(); + req.width = capture_options.preset.width; + req.height = capture_options.preset.height; + + encodings = compute_video_encodings(req.width, req.height, &options); + req.layers = video_layers_from_encodings(req.width, req.height, &encodings); + } + LocalTrack::Audio(_audio_track) => {} + } + + let track_info = self.rtc_engine.add_track(req).await?; + let publication = + LocalTrackPublication::new(track_info.clone(), track.clone(), options.clone()); + track.update_info(track_info); // Update SID + Source + debug!("publishing track with cid {:?}", track.rtc_track().id()); + let transceiver = self + .rtc_engine + .create_sender(track.clone(), options, encodings) + .await?; + + track.update_transceiver(Some(transceiver)); + track.start(); + + tokio::spawn({ + let rtc_engine = self.rtc_engine.clone(); + async move { + let _ = rtc_engine.negotiate_publisher().await; + } + }); + + self.inner + .add_track_publication(TrackPublication::Local(publication.clone())); + + self.inner + .dispatcher + .dispatch(&ParticipantEvent::LocalTrackPublished { + publication: publication.clone(), + }); + + Ok(publication) + } + + pub async fn unpublish_track( + &self, + track: TrackSid, + stop_on_unpublish: bool, + ) -> RoomResult { + let mut tracks = self.inner.tracks.write(); + if let Some(TrackPublication::Local(publication)) = tracks.remove(&track) { + let track = publication.track().unwrap(); + let sender = track.transceiver().unwrap().sender(); + self.rtc_engine.remove_track(sender).await?; + track.update_transceiver(None); + + self.inner + .dispatcher + .dispatch(&ParticipantEvent::LocalTrackUnpublished { + publication: publication.clone(), + }); + publication.update_track(None); + + tokio::spawn({ + let rtc_engine = self.rtc_engine.clone(); + async move { + let _ = rtc_engine.negotiate_publisher().await; + } + }); + + Ok(publication) + } else { + Err(RoomError::Internal("track not found".to_string())) + } + } pub async fn publish_data( &self, @@ -52,24 +145,80 @@ impl LocalParticipant { .await .map_err(Into::into) } -} -impl ParticipantInternalTrait for LocalParticipant { - fn update_info(self: &Arc, info: proto::ParticipantInfo, _emit_events: bool) { - self.shared.update_info(info); + #[inline] + pub fn get_track_publication(&self, sid: &TrackSid) -> Option { + self.inner.tracks.read().get(sid).map(|track| { + if let TrackPublication::Local(local) = track { + return local.clone(); + } + + unreachable!() + }) } - fn set_speaking(&self, speaking: bool) { - self.shared.set_speaking(speaking); + #[inline] + pub fn sid(&self) -> ParticipantSid { + self.inner.sid() } - fn set_audio_level(&self, level: f32) { - self.shared.set_audio_level(level); + #[inline] + pub fn identity(&self) -> ParticipantIdentity { + self.inner.identity() } - fn set_connection_quality(&self, quality: ConnectionQuality) { - self.shared.set_connection_quality(quality); + #[inline] + pub fn name(&self) -> String { + self.inner.name() + } + + #[inline] + pub fn metadata(&self) -> String { + self.inner.metadata() + } + + #[inline] + pub fn is_speaking(&self) -> bool { + self.inner.is_speaking() + } + + #[inline] + pub fn tracks(&self) -> RwLockReadGuard> { + self.inner.tracks() + } + + #[inline] + pub fn audio_level(&self) -> f32 { + self.inner.audio_level() + } + + #[inline] + pub fn connection_quality(&self) -> ConnectionQuality { + self.inner.connection_quality() + } + + #[inline] + pub fn register_observer(&self) -> mpsc::UnboundedReceiver { + self.inner.register_observer() + } + + #[inline] + pub(crate) fn update_info(self: &Self, info: proto::ParticipantInfo) { + self.inner.update_info(info); + } + + #[inline] + pub(crate) fn set_speaking(&self, speaking: bool) { + self.inner.set_speaking(speaking); + } + + #[inline] + pub(crate) fn set_audio_level(&self, level: f32) { + self.inner.set_audio_level(level); + } + + #[inline] + pub(crate) fn set_connection_quality(&self, quality: ConnectionQuality) { + self.inner.set_connection_quality(quality); } } - -impl_participant_trait!(LocalParticipant); diff --git a/livekit/src/room/participant/mod.rs b/livekit/src/room/participant/mod.rs index 318a9aa..57eb535 100644 --- a/livekit/src/room/participant/mod.rs +++ b/livekit/src/room/participant/mod.rs @@ -24,11 +24,11 @@ pub enum ParticipantEvent { publication: RemoteTrackPublication, }, TrackSubscribed { - track: RemoteTrackHandle, + track: RemoteTrack, publication: RemoteTrackPublication, }, TrackUnsubscribed { - track: RemoteTrackHandle, + track: RemoteTrack, publication: RemoteTrackPublication, }, TrackSubscriptionFailed { @@ -51,6 +51,12 @@ pub enum ParticipantEvent { ConnectionQualityChanged { quality: ConnectionQuality, }, + LocalTrackPublished { + publication: LocalTrackPublication, + }, + LocalTrackUnpublished { + publication: LocalTrackPublication, + }, } #[derive(Debug, Clone, Copy, Eq, PartialEq)] @@ -83,21 +89,48 @@ impl From for ConnectionQuality { } } -#[derive(Debug)] -pub(super) struct ParticipantShared { - pub(super) sid: Mutex, - pub(super) identity: Mutex, - pub(super) name: Mutex, - pub(super) metadata: Mutex, - pub(super) tracks: RwLock>, - pub(super) speaking: AtomicBool, - pub(super) audio_level: AtomicU32, - pub(super) connection_quality: AtomicU8, - pub(super) dispatcher: Mutex>, +#[derive(Debug, Clone)] +pub enum Participant { + Local(LocalParticipant), + Remote(RemoteParticipant), } -impl ParticipantShared { - pub(super) fn new( +impl Participant { + enum_dispatch!( + [Local, Remote]; + pub fn sid(self: &Self) -> ParticipantSid; + pub fn identity(self: &Self) -> ParticipantIdentity; + pub fn name(self: &Self) -> String; + pub fn metadata(self: &Self) -> String; + pub fn is_speaking(self: &Self) -> bool; + pub fn audio_level(self: &Self) -> f32; + pub fn connection_quality(self: &Self) -> ConnectionQuality; + pub fn tracks(self: &Self) -> RwLockReadGuard>; + pub fn register_observer(self: &Self) -> mpsc::UnboundedReceiver; + + // Internal functions + pub(crate) fn set_speaking(self: &Self, speaking: bool) -> (); + pub(crate) fn set_audio_level(self: &Self, level: f32) -> (); + pub(crate) fn set_connection_quality(self: &Self, quality: ConnectionQuality) -> (); + pub(crate) fn update_info(self: &Self, info: proto::ParticipantInfo) -> (); + ); +} + +#[derive(Debug)] +pub(crate) struct ParticipantInner { + pub sid: Mutex, + pub identity: Mutex, + pub name: Mutex, + pub metadata: Mutex, + pub speaking: AtomicBool, + pub tracks: RwLock>, + pub audio_level: AtomicU32, + pub connection_quality: AtomicU8, + pub dispatcher: Dispatcher, +} + +impl ParticipantInner { + pub fn new( sid: ParticipantSid, identity: ParticipantIdentity, name: String, @@ -116,128 +149,64 @@ impl ParticipantShared { } } - pub(crate) fn update_info(&self, info: proto::ParticipantInfo) { + pub fn sid(&self) -> ParticipantSid { + self.sid.lock().clone() + } + + pub fn identity(&self) -> ParticipantIdentity { + self.identity.lock().clone() + } + + pub fn name(&self) -> String { + self.name.lock().clone() + } + + pub fn metadata(&self) -> String { + self.metadata.lock().clone() + } + + pub fn is_speaking(&self) -> bool { + self.speaking.load(Ordering::SeqCst) + } + + pub fn tracks(&self) -> RwLockReadGuard> { + self.tracks.read() + } + + pub fn audio_level(&self) -> f32 { + f32::from_bits(self.audio_level.load(Ordering::SeqCst)) + } + + pub fn connection_quality(&self) -> ConnectionQuality { + self.connection_quality.load(Ordering::SeqCst).into() + } + + pub fn register_observer(&self) -> mpsc::UnboundedReceiver { + self.dispatcher.register() + } + + pub fn update_info(&self, info: proto::ParticipantInfo) { *self.sid.lock() = info.sid.into(); *self.identity.lock() = info.identity.into(); *self.name.lock() = info.name; *self.metadata.lock() = info.metadata; // TODO(theomonnom): callback MetadataChanged } - pub(crate) fn set_speaking(&self, speaking: bool) { + pub fn set_speaking(&self, speaking: bool) { self.speaking.store(speaking, Ordering::SeqCst); } - pub(crate) fn set_audio_level(&self, audio_level: f32) { + pub fn set_audio_level(&self, audio_level: f32) { self.audio_level .store(audio_level.to_bits(), Ordering::SeqCst) } - pub(crate) fn register_observer(&self) -> mpsc::UnboundedReceiver { - self.dispatcher.lock().register() - } - - pub(crate) fn set_connection_quality(&self, quality: ConnectionQuality) { + pub fn set_connection_quality(&self, quality: ConnectionQuality) { self.connection_quality .store(quality as u8, Ordering::SeqCst); } - pub(crate) fn add_track_publication(&self, publication: TrackPublication) { + pub fn add_track_publication(&self, publication: TrackPublication) { self.tracks.write().insert(publication.sid(), publication); } } - -pub(crate) trait ParticipantInternalTrait { - fn set_speaking(&self, speaking: bool); - fn set_audio_level(&self, level: f32); - fn set_connection_quality(&self, quality: ConnectionQuality); - fn update_info(self: &Arc, info: proto::ParticipantInfo, emit_events: bool); -} - -pub trait ParticipantTrait { - fn sid(&self) -> ParticipantSid; - fn identity(&self) -> ParticipantIdentity; - fn name(&self) -> String; - fn metadata(&self) -> String; - fn is_speaking(&self) -> bool; - fn audio_level(&self) -> f32; - fn connection_quality(&self) -> ConnectionQuality; - fn tracks(&self) -> RwLockReadGuard>; - fn register_observer(&self) -> mpsc::UnboundedReceiver; -} - -#[derive(Debug, Clone)] -pub enum Participant { - Local(Arc), - Remote(Arc), -} - -// TODO(theomonnom): Should I provide a WeakParticipant here ? - -impl Participant { - enum_dispatch!( - [Local, Remote] - fnc!(pub(crate), update_info, &Self, [info: proto::ParticipantInfo, emit_events: bool], ()); - fnc!(pub(crate), set_speaking, &Self, [speaking: bool], ()); - fnc!(pub(crate), set_audio_level, &Self, [audio_level: f32], ()); - fnc!(pub(crate), set_connection_quality, &Self, [quality: ConnectionQuality], ()); - ); -} - -impl ParticipantTrait for Participant { - enum_dispatch!( - [Local, Remote] - fnc!(sid, &Self, [], ParticipantSid); - fnc!(identity, &Self, [], ParticipantIdentity); - fnc!(name, &Self, [], String); - fnc!(metadata, &Self, [], String); - fnc!(is_speaking, &Self, [], bool); - fnc!(audio_level, &Self, [], f32); - fnc!(connection_quality, &Self, [], ConnectionQuality); - fnc!(tracks, &Self, [], RwLockReadGuard>); - fnc!(register_observer, &Self, [], mpsc::UnboundedReceiver); - ); -} - -macro_rules! impl_participant_trait { - ($x:ty) => { - impl crate::room::participant::ParticipantTrait for $x { - fn sid(&self) -> ParticipantSid { - self.shared.sid.lock().clone() - } - - fn identity(&self) -> ParticipantIdentity { - self.shared.identity.lock().clone() - } - - fn name(&self) -> String { - self.shared.name.lock().clone() - } - - fn metadata(&self) -> String { - self.shared.metadata.lock().clone() - } - - fn is_speaking(&self) -> bool { - self.shared.speaking.load(Ordering::SeqCst) - } - - fn audio_level(&self) -> f32 { - f32::from_bits(self.shared.audio_level.load(Ordering::SeqCst)) - } - - fn connection_quality(&self) -> ConnectionQuality { - self.shared.connection_quality.load(Ordering::SeqCst).into() - } - - fn tracks(&self) -> RwLockReadGuard> { - self.shared.tracks.read() - } - - fn register_observer(&self) -> mpsc::UnboundedReceiver { - self.shared.register_observer() - } - } - }; -} - -pub(super) use impl_participant_trait; diff --git a/livekit/src/room/participant/remote_participant.rs b/livekit/src/room/participant/remote_participant.rs index 3206338..4ae18e4 100644 --- a/livekit/src/room/participant/remote_participant.rs +++ b/livekit/src/room/participant/remote_participant.rs @@ -1,15 +1,11 @@ -use super::{ - impl_participant_trait, ConnectionQuality, ParticipantInternalTrait, ParticipantShared, -}; +use super::{ConnectionQuality, ParticipantInner}; use crate::prelude::*; use crate::proto; -use crate::publication::TrackPublicationInternalTrait; use crate::track::TrackError; -use livekit_webrtc::prelude::*; +use livekit_webrtc as rtc; use parking_lot::RwLockReadGuard; -use std::collections::HashMap; -use std::collections::HashSet; -use std::sync::atomic::Ordering; +use rtc::prelude::MediaStreamTrack; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc; @@ -18,9 +14,9 @@ use tracing::{debug, error, instrument, Level}; const ADD_TRACK_TIMEOUT: Duration = Duration::from_secs(5); -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct RemoteParticipant { - shared: ParticipantShared, + inner: Arc, } impl RemoteParticipant { @@ -31,26 +27,25 @@ impl RemoteParticipant { metadata: String, ) -> Self { Self { - shared: ParticipantShared::new(sid, identity, name, metadata), + inner: Arc::new(ParticipantInner::new(sid, identity, name, metadata)), } } - fn get_track_publication(&self, sid: &TrackSid) -> Option { - self.shared.tracks.read().get(sid).map(|track| { + #[inline] + pub fn get_track_publication(&self, sid: &TrackSid) -> Option { + self.inner.tracks.read().get(sid).map(|track| { if let TrackPublication::Remote(remote) = track { - remote.clone() - } else { - unreachable!() + return remote.clone(); } + unreachable!() }) } - /// Called by the RoomSession when receiving data by the RTCSession + /// Called by the RoomSession when receiving data from the RrcSession /// It is just used to emit the Data event on the participant dispatcher. pub(crate) fn on_data_received(&self, data: Arc>, kind: proto::data_packet::Kind) { - self.shared + self.inner .dispatcher - .lock() .dispatch(&ParticipantEvent::DataReceived { payload: data, kind, @@ -59,9 +54,9 @@ impl RemoteParticipant { #[instrument(level = Level::DEBUG)] pub(crate) async fn add_subscribed_media_track( - self: Arc, + &self, sid: TrackSid, - media_track: MediaStreamTrackHandle, + media_track: rtc::media_stream::MediaStreamTrack, ) { let wait_publication = { let participant = self.clone(); @@ -73,7 +68,7 @@ impl RemoteParticipant { return publication; } - tokio::task::yield_now().await; + tokio::task::yield_now().await; // Remove yield } } }; @@ -81,25 +76,25 @@ impl RemoteParticipant { if let Ok(remote_publication) = timeout(ADD_TRACK_TIMEOUT, wait_publication).await { let track = match remote_publication.kind() { TrackKind::Audio => { - if let MediaStreamTrackHandle::Audio(rtc_track) = media_track { + if let MediaStreamTrack::Audio(rtc_track) = media_track { let audio_track = RemoteAudioTrack::new( remote_publication.sid().into(), remote_publication.name(), rtc_track, ); - RemoteTrackHandle::Audio(Arc::new(audio_track)) + RemoteTrack::Audio(audio_track) } else { unreachable!(); } } TrackKind::Video => { - if let MediaStreamTrackHandle::Video(rtc_track) = media_track { + if let MediaStreamTrack::Video(rtc_track) = media_track { let video_track = RemoteVideoTrack::new( remote_publication.sid().into(), remote_publication.name(), rtc_track, ); - RemoteTrackHandle::Video(Arc::new(video_track)) + RemoteTrack::Video(video_track) } else { unreachable!() } @@ -110,23 +105,30 @@ impl RemoteParticipant { debug!("starting track: {:?}", sid); remote_publication.update_track(Some(track.clone().into())); - self.shared + track.set_muted(remote_publication.muted()); + track.update_info(proto::TrackInfo { + sid: remote_publication.sid().to_string(), + name: remote_publication.name().to_string(), + r#type: proto::TrackType::from(remote_publication.kind()) as i32, + source: proto::TrackSource::from(remote_publication.source()) as i32, + ..Default::default() + }); + + self.inner .add_track_publication(TrackPublication::Remote(remote_publication.clone())); track.start(); - self.shared + self.inner .dispatcher - .lock() .dispatch(&ParticipantEvent::TrackSubscribed { - track: track, + track, publication: remote_publication, }); } else { error!("could not find published track with sid: {:?}", sid); - self.shared + self.inner .dispatcher - .lock() .dispatch(&ParticipantEvent::TrackSubscriptionFailed { sid: sid.clone(), error: TrackError::TrackNotFound(sid.clone().to_string()), @@ -134,81 +136,118 @@ impl RemoteParticipant { } } - pub(crate) fn unpublish_track(self: &Arc, sid: &TrackSid, emit_events: bool) { + pub(crate) fn unpublish_track(&self, sid: &TrackSid) { if let Some(publication) = self.get_track_publication(sid) { // Unsubscribe to the track if needed if let Some(track) = publication.track() { track.stop(); - self.shared + self.inner .dispatcher - .lock() .dispatch(&ParticipantEvent::TrackUnsubscribed { track: track.clone(), publication: publication.clone(), }); } - if emit_events { - self.shared - .dispatcher - .lock() - .dispatch(&ParticipantEvent::TrackUnpublished { - publication: publication.clone(), - }); - } + self.inner + .dispatcher + .dispatch(&ParticipantEvent::TrackUnpublished { + publication: publication.clone(), + }); publication.update_track(None); } } -} -impl ParticipantInternalTrait for RemoteParticipant { - fn update_info(self: &Arc, info: proto::ParticipantInfo, emit_events: bool) { - self.shared.update_info(info.clone()); + pub(crate) fn update_info(&self, info: proto::ParticipantInfo) { + self.inner.update_info(info.clone()); let mut valid_tracks = HashSet::::new(); for track in info.tracks { if let Some(publication) = self.get_track_publication(&track.sid.clone().into()) { publication.update_info(track.clone()); } else { - let publication = RemoteTrackPublication::new(track.clone(), self.sid(), None); - self.shared + let publication = RemoteTrackPublication::new(track.clone(), None); + self.inner .add_track_publication(TrackPublication::Remote(publication.clone())); // This is a new track, dispatch publish event - if emit_events { - self.shared - .dispatcher - .lock() - .dispatch(&ParticipantEvent::TrackPublished { publication }); - } + self.inner + .dispatcher + .dispatch(&ParticipantEvent::TrackPublished { publication }); } valid_tracks.insert(track.sid.into()); } // remove tracks that are no longer valid - for (sid, _) in self.shared.tracks.read().iter() { + for (sid, _) in self.inner.tracks.read().iter() { if valid_tracks.contains(sid) { continue; } - self.unpublish_track(sid, emit_events); + self.unpublish_track(sid); } } - fn set_speaking(&self, speaking: bool) { - self.shared.set_speaking(speaking); + #[inline] + pub fn sid(&self) -> ParticipantSid { + self.inner.sid() } - fn set_audio_level(&self, level: f32) { - self.shared.set_audio_level(level); + #[inline] + pub fn identity(&self) -> ParticipantIdentity { + self.inner.identity() } - fn set_connection_quality(&self, quality: ConnectionQuality) { - self.shared.set_connection_quality(quality); + #[inline] + pub fn name(&self) -> String { + self.inner.name() + } + + #[inline] + pub fn metadata(&self) -> String { + self.inner.metadata() + } + + #[inline] + pub fn is_speaking(&self) -> bool { + self.inner.is_speaking() + } + + #[inline] + pub fn tracks(&self) -> RwLockReadGuard> { + self.inner.tracks() + } + + #[inline] + pub fn audio_level(&self) -> f32 { + self.inner.audio_level() + } + + #[inline] + pub fn connection_quality(&self) -> ConnectionQuality { + self.inner.connection_quality() + } + + #[inline] + pub fn register_observer(&self) -> mpsc::UnboundedReceiver { + self.inner.register_observer() + } + + #[inline] + pub(crate) fn set_speaking(&self, speaking: bool) { + self.inner.set_speaking(speaking); + } + + #[inline] + pub(crate) fn set_audio_level(&self, level: f32) { + self.inner.set_audio_level(level); + } + + #[inline] + pub(crate) fn set_connection_quality(&self, quality: ConnectionQuality) { + self.inner.set_connection_quality(quality); } } - -impl_participant_trait!(RemoteParticipant); diff --git a/livekit/src/room/publication/local.rs b/livekit/src/room/publication/local.rs new file mode 100644 index 0000000..7bad0af --- /dev/null +++ b/livekit/src/room/publication/local.rs @@ -0,0 +1,91 @@ +use super::TrackPublicationInner; +use crate::id::TrackSid; +use crate::options::TrackPublishOptions; +use crate::proto; +use crate::track::{LocalTrack, Track, TrackDimension, TrackKind, TrackSource}; +use parking_lot::Mutex; +use std::sync::Arc; + +#[derive(Debug)] +struct LocalTrackPublicationInner { + publication_inner: TrackPublicationInner, + options: Mutex, +} + +#[derive(Clone, Debug)] +pub struct LocalTrackPublication { + inner: Arc, +} + +impl LocalTrackPublication { + pub(crate) fn new( + info: proto::TrackInfo, + track: LocalTrack, + options: TrackPublishOptions, + ) -> Self { + Self { + inner: Arc::new(LocalTrackPublicationInner { + publication_inner: TrackPublicationInner::new(info, Some(track.into())), + options: Mutex::new(options), + }), + } + } + + #[inline] + pub fn sid(&self) -> TrackSid { + self.inner.publication_inner.sid() + } + + #[inline] + pub fn name(&self) -> String { + self.inner.publication_inner.name() + } + + #[inline] + pub fn kind(&self) -> TrackKind { + self.inner.publication_inner.kind() + } + + #[inline] + pub fn source(&self) -> TrackSource { + self.inner.publication_inner.source() + } + + #[inline] + pub fn simulcasted(&self) -> bool { + self.inner.publication_inner.simulcasted() + } + + #[inline] + pub fn dimension(&self) -> TrackDimension { + self.inner.publication_inner.dimension() + } + + #[inline] + pub fn track(&self) -> Option { + self.inner + .publication_inner + .track() + .map(|track| track.try_into().unwrap()) + } + + #[inline] + pub fn mime_type(&self) -> String { + self.inner.publication_inner.mime_type() + } + + #[inline] + pub fn muted(&self) -> bool { + self.inner.publication_inner.muted() + } + + #[inline] + pub(crate) fn update_track(&self, track: Option) { + self.inner.publication_inner.update_track(track); + } + + #[inline] + pub(crate) fn update_info(&self, info: proto::TrackInfo) { + self.inner.publication_inner.update_info(info); + } +} diff --git a/livekit/src/room/publication/mod.rs b/livekit/src/room/publication/mod.rs index dc383e7..7595bae 100644 --- a/livekit/src/room/publication/mod.rs +++ b/livekit/src/room/publication/mod.rs @@ -1,58 +1,46 @@ +use super::track::{TrackDimension, TrackEvent}; use crate::prelude::*; use crate::proto; +use crate::track::Track; +use futures_util::stream::StreamExt; use livekit_utils::enum_dispatch; use livekit_utils::observer::Dispatcher; use parking_lot::Mutex; use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; use std::sync::Arc; -use tokio::sync::{mpsc, oneshot}; +use tokio::sync::Notify; +use tokio_stream::wrappers::UnboundedReceiverStream; -use super::track::{TrackDimension, TrackEvent}; +mod local; +pub use local::*; -pub(crate) trait TrackPublicationInternalTrait { - fn update_track(&self, track: Option); - fn update_info(&self, info: proto::TrackInfo); -} - -pub trait TrackPublicationTrait { - fn name(&self) -> String; - fn sid(&self) -> TrackSid; - fn kind(&self) -> TrackKind; - fn source(&self) -> TrackSource; - fn simulcasted(&self) -> bool; - fn dimension(&self) -> TrackDimension; - fn mime_type(&self) -> String; - fn muted(&self) -> bool; -} +mod remote; +pub use remote::*; #[derive(Debug)] -pub(super) struct TrackPublicationShared { - pub(super) track: Mutex>, - pub(super) name: Mutex, - pub(super) sid: Mutex, - pub(super) kind: AtomicU8, // Casted to TrackKind - pub(super) source: AtomicU8, // Casted to TrackSource - pub(super) simulcasted: AtomicBool, - pub(super) dimension: Mutex, - pub(super) mime_type: Mutex, - pub(super) muted: AtomicBool, - pub(super) participant: ParticipantSid, - pub(super) dispatcher: Mutex>, - pub(super) close_sender: Mutex>>, +pub(crate) struct TrackPublicationInner { + track: Mutex>, + name: Mutex, + sid: Mutex, + kind: AtomicU8, // Casted to TrackKind + source: AtomicU8, // Casted to TrackSource + simulcasted: AtomicBool, + dimension: Mutex, + mime_type: Mutex, + muted: AtomicBool, + dispatcher: Dispatcher, + close_notifier: Arc, } -impl TrackPublicationShared { - pub fn new( - info: proto::TrackInfo, - participant: ParticipantSid, - track: Option, - ) -> Arc { - Arc::new(Self { +impl TrackPublicationInner { + pub fn new(info: proto::TrackInfo, track: Option) -> Self { + Self { track: Mutex::new(track), name: Mutex::new(info.name), sid: Mutex::new(info.sid.into()), kind: AtomicU8::new( - TrackKind::from(proto::TrackType::from_i32(info.r#type).unwrap()) as u8, + TrackKind::try_from(proto::TrackType::from_i32(info.r#type).unwrap()).unwrap() + as u8, ), source: AtomicU8::new(TrackSource::from( proto::TrackSource::from_i32(info.source).unwrap(), @@ -62,46 +50,32 @@ impl TrackPublicationShared { mime_type: Mutex::new(info.mime_type), muted: AtomicBool::new(info.muted), dispatcher: Default::default(), - close_sender: Default::default(), - participant, - }) + close_notifier: Default::default(), + } } - pub fn update_track(self: &Arc, track: Option) { + pub fn update_track(&self, track: Option) { let mut old_track = self.track.lock(); - - if let Some(close_sender) = self.close_sender.lock().take() { - let _ = close_sender.send(()); - } - *old_track = track.clone(); - if let Some(track) = track { - let (close_sender, close_receiver) = oneshot::channel(); - self.close_sender.lock().replace(close_sender); - let track_receiver = track.register_observer(); - tokio::spawn( - self.clone() - .publication_task(close_receiver, track_receiver), - ); - } - } + self.close_notifier.notify_waiters(); - /// Task used to forward TrackHandle's events to the TrackPublications's dispatcher - async fn publication_task( - self: Arc, - mut close_receiver: oneshot::Receiver<()>, - mut track_receiver: mpsc::UnboundedReceiver, - ) { - loop { - tokio::select! { - Some(event) = track_receiver.recv() => { - self.dispatcher.lock().dispatch(&event); + if let Some(track) = track.as_ref() { + let track_stream = UnboundedReceiverStream::new(track.register_observer()); + tokio::spawn({ + let dispatcher = self.dispatcher.clone(); + let notifier = self.close_notifier.clone(); + + async move { + let notified = notifier.notified(); + futures_util::pin_mut!(notified); + futures_util::future::select( + track_stream.map(Ok).forward(dispatcher), + notified, + ) + .await; } - _ = &mut close_receiver => { - break; - } - } + }); } } @@ -111,7 +85,7 @@ impl TrackPublicationShared { *self.dimension.lock() = TrackDimension(info.width, info.height); *self.mime_type.lock() = info.mime_type; self.kind.store( - TrackKind::from(proto::TrackType::from_i32(info.r#type).unwrap()) as u8, + TrackKind::try_from(proto::TrackType::from_i32(info.r#type).unwrap()).unwrap() as u8, Ordering::SeqCst, ); self.source.store( @@ -125,13 +99,41 @@ impl TrackPublicationShared { track.set_muted(info.muted); } } -} -impl Drop for TrackPublicationShared { - fn drop(&mut self) { - if let Some(close_sender) = self.close_sender.lock().take() { - let _ = close_sender.send(()); - } + pub fn sid(&self) -> TrackSid { + self.sid.lock().clone() + } + + pub fn name(&self) -> String { + self.name.lock().clone() + } + + pub fn kind(&self) -> TrackKind { + self.kind.load(Ordering::SeqCst).try_into().unwrap() + } + + pub fn source(&self) -> TrackSource { + self.source.load(Ordering::SeqCst).into() + } + + pub fn simulcasted(&self) -> bool { + self.simulcasted.load(Ordering::Relaxed) + } + + pub fn dimension(&self) -> TrackDimension { + self.dimension.lock().clone() + } + + pub fn mime_type(&self) -> String { + self.mime_type.lock().clone() + } + + pub fn track(&self) -> Option { + self.track.lock().clone() + } + + pub fn muted(&self) -> bool { + self.muted.load(Ordering::Relaxed) } } @@ -142,134 +144,22 @@ pub enum TrackPublication { } impl TrackPublication { - pub fn track(&self) -> Option { - // Not calling Local/Remote function here, we don't need "cast" + enum_dispatch!( + [Local, Remote]; + pub fn sid(self: &Self) -> TrackSid; + pub fn name(self: &Self) -> String; + pub fn kind(self: &Self) -> TrackKind; + pub fn source(self: &Self) -> TrackSource; + pub fn simulcasted(self: &Self) -> bool; + pub fn dimension(self: &Self) -> TrackDimension; + pub fn mime_type(self: &Self) -> String; + pub fn muted(self: &Self) -> bool; + ); + + pub fn track(&self) -> Option { match self { - TrackPublication::Local(p) => p.shared.track.lock().clone(), - TrackPublication::Remote(p) => p.shared.track.lock().clone(), + TrackPublication::Local(p) => p.track().map(Into::into), + TrackPublication::Remote(p) => p.track().map(Into::into), } } } - -impl TrackPublicationInternalTrait for TrackPublication { - enum_dispatch!( - [Local, Remote] - fnc!(update_track, &Self, [track: Option], ()); - fnc!(update_info, &Self, [info: proto::TrackInfo], ()); - ); -} - -impl TrackPublicationTrait for TrackPublication { - enum_dispatch!( - [Local, Remote] - fnc!(sid, &Self, [], TrackSid); - fnc!(name, &Self, [], String); - fnc!(kind, &Self, [], TrackKind); - fnc!(source, &Self, [], TrackSource); - fnc!(simulcasted, &Self, [], bool); - fnc!(dimension, &Self, [], TrackDimension); - fnc!(mime_type, &Self, [], String); - fnc!(muted, &Self, [], bool); - ); -} - -macro_rules! impl_publication_trait { - ($x:ident) => { - impl TrackPublicationTrait for $x { - fn name(&self) -> String { - self.shared.name.lock().clone() - } - - fn sid(&self) -> TrackSid { - self.shared.sid.lock().clone() - } - - fn kind(&self) -> TrackKind { - self.shared.kind.load(Ordering::SeqCst).into() - } - - fn source(&self) -> TrackSource { - self.shared.source.load(Ordering::SeqCst).into() - } - - fn simulcasted(&self) -> bool { - self.shared.simulcasted.load(Ordering::SeqCst) - } - - fn dimension(&self) -> TrackDimension { - self.shared.dimension.lock().clone() - } - - fn mime_type(&self) -> String { - self.shared.mime_type.lock().clone() - } - - fn muted(&self) -> bool { - self.shared.muted.load(Ordering::SeqCst) - } - } - }; -} - -#[derive(Clone, Debug)] -pub struct LocalTrackPublication { - shared: Arc, -} - -impl LocalTrackPublication { - pub fn track(&self) -> Option { - self.shared - .track - .lock() - .clone() - .map(|local_track| local_track.try_into().unwrap()) - } -} - -impl TrackPublicationInternalTrait for LocalTrackPublication { - fn update_track(&self, track: Option) { - self.shared.update_track(track); - } - - fn update_info(&self, info: proto::TrackInfo) { - self.shared.update_info(info); - } -} - -#[derive(Clone, Debug)] -pub struct RemoteTrackPublication { - shared: Arc, -} - -impl RemoteTrackPublication { - pub fn new( - info: proto::TrackInfo, - participant: ParticipantSid, - track: Option, - ) -> Self { - Self { - shared: TrackPublicationShared::new(info, participant, track), - } - } - - pub fn track(&self) -> Option { - self.shared - .track - .lock() - .clone() - .map(|track| track.try_into().unwrap()) - } -} - -impl TrackPublicationInternalTrait for RemoteTrackPublication { - fn update_track(&self, track: Option) { - self.shared.update_track(track); - } - - fn update_info(&self, info: proto::TrackInfo) { - self.shared.update_info(info); - } -} - -impl_publication_trait!(LocalTrackPublication); -impl_publication_trait!(RemoteTrackPublication); diff --git a/livekit/src/room/publication/remote.rs b/livekit/src/room/publication/remote.rs new file mode 100644 index 0000000..a55af07 --- /dev/null +++ b/livekit/src/room/publication/remote.rs @@ -0,0 +1,73 @@ +use super::TrackPublicationInner; +use crate::id::TrackSid; +use crate::proto; +use crate::track::{RemoteTrack, Track, TrackDimension, TrackKind, TrackSource}; +use std::sync::Arc; + +#[derive(Clone, Debug)] +pub struct RemoteTrackPublication { + inner: Arc, +} + +impl RemoteTrackPublication { + pub(crate) fn new(info: proto::TrackInfo, track: Option) -> Self { + Self { + inner: Arc::new(TrackPublicationInner::new(info, track.map(Into::into))), + } + } + + #[inline] + pub fn sid(&self) -> TrackSid { + self.inner.sid() + } + + #[inline] + pub fn name(&self) -> String { + self.inner.name() + } + + #[inline] + pub fn kind(&self) -> TrackKind { + self.inner.kind() + } + + #[inline] + pub fn source(&self) -> TrackSource { + self.inner.source() + } + + #[inline] + pub fn simulcasted(&self) -> bool { + self.inner.simulcasted() + } + + #[inline] + pub fn dimension(&self) -> TrackDimension { + self.inner.dimension() + } + + #[inline] + pub fn track(&self) -> Option { + self.inner.track().map(|track| track.try_into().unwrap()) + } + + #[inline] + pub fn mime_type(&self) -> String { + self.inner.mime_type() + } + + #[inline] + pub fn muted(&self) -> bool { + self.inner.muted() + } + + #[inline] + pub(crate) fn update_track(&self, track: Option) { + self.inner.update_track(track); + } + + #[inline] + pub(crate) fn update_info(&self, info: proto::TrackInfo) { + self.inner.update_info(info); + } +} diff --git a/livekit/src/room/room_session.rs b/livekit/src/room/room_session.rs index 8d4f87d..ebf5d3f 100644 --- a/livekit/src/room/room_session.rs +++ b/livekit/src/room/room_session.rs @@ -1,7 +1,7 @@ -use crate::participant::{ConnectionQuality, ParticipantInternalTrait}; +use crate::participant::ConnectionQuality; use crate::prelude::*; use crate::proto; -use crate::rtc_engine::{EngineEvent, EngineEvents, EngineResult, RTCEngine}; +use crate::rtc_engine::{EngineEvent, EngineEvents, EngineResult, RtcEngine}; use crate::signal_client::SignalOptions; use crate::{RoomError, RoomEvent, RoomResult, SimulateScenario}; use livekit_utils::observer::Dispatcher; @@ -9,8 +9,7 @@ use parking_lot::{Mutex, RwLock, RwLockReadGuard}; use std::collections::HashMap; use std::sync::atomic::{AtomicU8, Ordering}; use std::sync::Arc; -use tokio::sync::mpsc; -use tokio::sync::oneshot; +use tokio::sync::{mpsc, oneshot}; use tokio::task::JoinHandle; use tracing::{error, info, instrument, Level}; @@ -22,17 +21,6 @@ pub enum ConnectionState { Unknown, } -impl From for ConnectionState { - fn from(value: u8) -> Self { - match value { - 0 => ConnectionState::Disconnected, - 1 => ConnectionState::Connected, - 2 => ConnectionState::Reconnecting, - _ => ConnectionState::Unknown, - } - } -} - /// Internal representation of a RoomSession #[derive(Debug)] struct SessionInner { @@ -40,12 +28,12 @@ struct SessionInner { sid: Mutex, name: Mutex, metadata: Mutex, - participants: RwLock>>, + participants: RwLock>, participants_tasks: RwLock, oneshot::Sender<()>)>>, active_speakers: RwLock>, - rtc_engine: Arc, - local_participant: Arc, - dispatcher: Mutex>, + rtc_engine: Arc, + local_participant: LocalParticipant, + dispatcher: Dispatcher, } #[derive(Debug)] @@ -64,7 +52,7 @@ pub struct RoomSession { impl SessionHandle { pub async fn connect(url: &str, token: &str) -> RoomResult { - let (rtc_engine, engine_events) = RTCEngine::new(); + let (rtc_engine, engine_events) = RtcEngine::new(); let rtc_engine = Arc::new(rtc_engine); rtc_engine .connect(url, token, SignalOptions::default()) @@ -72,13 +60,13 @@ impl SessionHandle { let join_response = rtc_engine.join_response().unwrap(); let pi = join_response.participant.unwrap().clone(); - let local_participant = Arc::new(LocalParticipant::new( + let local_participant = LocalParticipant::new( rtc_engine.clone(), pi.sid.into(), pi.identity.into(), pi.name, pi.metadata, - )); + ); let room_info = join_response.room.unwrap(); let inner = Arc::new(SessionInner { @@ -99,7 +87,7 @@ impl SessionHandle { let pi = pi.clone(); inner.create_participant(pi.sid.into(), pi.identity.into(), pi.name, pi.metadata) }; - participant.update_info(pi.clone(), false); + participant.update_info(pi.clone()); } let (close_emitter, close_receiver) = oneshot::channel(); @@ -108,7 +96,7 @@ impl SessionHandle { inner.update_connection_state(ConnectionState::Connected); let session = Self { - session: RoomSession::from(inner), + session: RoomSession { inner }, session_task, close_emitter, }; @@ -122,7 +110,7 @@ impl SessionHandle { } pub fn subscribe(&self) -> mpsc::UnboundedReceiver { - self.session.inner.dispatcher.lock().register() + self.session.inner.dispatcher.register() } pub fn session(&self) -> RoomSession { @@ -131,10 +119,6 @@ impl SessionHandle { } impl RoomSession { - fn from(inner: Arc) -> Self { - Self { inner } - } - pub fn sid(&self) -> RoomSid { self.inner.sid.lock().clone() } @@ -147,7 +131,7 @@ impl RoomSession { self.inner.metadata.lock().clone() } - pub fn local_participant(&self) -> Arc { + pub fn local_participant(&self) -> LocalParticipant { self.inner.local_participant.clone() } @@ -155,7 +139,7 @@ impl RoomSession { self.inner.state.load(Ordering::Acquire).try_into().unwrap() } - pub fn participants(&self) -> RwLockReadGuard>> { + pub fn participants(&self) -> RwLockReadGuard> { self.inner.participants.read() } @@ -226,36 +210,30 @@ impl SessionInner { if let Participant::Remote(remote_participant) = participant { match event { ParticipantEvent::TrackPublished { publication } => { - self.dispatcher.lock().dispatch(&RoomEvent::TrackPublished { + self.dispatcher.dispatch(&RoomEvent::TrackPublished { participant: remote_participant.clone(), publication, }); } ParticipantEvent::TrackUnpublished { publication } => { - self.dispatcher - .lock() - .dispatch(&RoomEvent::TrackUnpublished { - participant: remote_participant.clone(), - publication, - }); + self.dispatcher.dispatch(&RoomEvent::TrackUnpublished { + participant: remote_participant.clone(), + publication, + }); } ParticipantEvent::TrackSubscribed { track, publication } => { - self.dispatcher - .lock() - .dispatch(&RoomEvent::TrackSubscribed { - participant: remote_participant.clone(), - track, - publication, - }); + self.dispatcher.dispatch(&RoomEvent::TrackSubscribed { + participant: remote_participant.clone(), + track, + publication, + }); } ParticipantEvent::TrackUnsubscribed { track, publication } => { - self.dispatcher - .lock() - .dispatch(&RoomEvent::TrackUnsubscribed { - participant: remote_participant.clone(), - track, - publication, - }); + self.dispatcher.dispatch(&RoomEvent::TrackUnsubscribed { + participant: remote_participant.clone(), + track, + publication, + }); } _ => {} }; @@ -303,12 +281,12 @@ impl SessionInner { } EngineEvent::Resuming => { if self.update_connection_state(ConnectionState::Reconnecting) { - self.dispatcher.lock().dispatch(&RoomEvent::Reconnecting); + self.dispatcher.dispatch(&RoomEvent::Reconnecting); } } EngineEvent::Resumed => { self.update_connection_state(ConnectionState::Connected); - self.dispatcher.lock().dispatch(&RoomEvent::Reconnected); + self.dispatcher.dispatch(&RoomEvent::Reconnected); // TODO(theomonnom): Update subscriptions settings // TODO(theomonnom): Send sync state @@ -323,7 +301,7 @@ impl SessionInner { } => { let payload = Arc::new(payload); if let Some(participant) = self.get_participant(&participant_sid.into()) { - self.dispatcher.lock().dispatch(&RoomEvent::DataReceived { + self.dispatcher.dispatch(&RoomEvent::DataReceived { payload: payload.clone(), kind, participant: participant.clone(), @@ -357,7 +335,6 @@ impl SessionInner { self.state.store(state as u8, Ordering::Release); self.dispatcher - .lock() .dispatch(&RoomEvent::ConnectionStateChanged(state)); return true; } @@ -368,11 +345,10 @@ impl SessionInner { #[instrument(level = Level::DEBUG)] fn handle_participant_update(self: &Arc, updates: Vec) { for pi in updates { - info!("test"); if pi.sid == self.local_participant.sid() || pi.identity == self.local_participant.identity() { - self.local_participant.clone().update_info(pi, true); + self.local_participant.clone().update_info(pi); continue; } @@ -386,7 +362,7 @@ impl SessionInner { .handle_participant_disconnect(remote_participant) } else { // Participant is already connected, update the it - remote_participant.update_info(pi.clone(), true); + remote_participant.update_info(pi.clone()); } } else { // Create a new participant @@ -398,10 +374,9 @@ impl SessionInner { let _ = self .dispatcher - .lock() .dispatch(&RoomEvent::ParticipantConnected(remote_participant.clone())); - remote_participant.update_info(pi.clone(), true); + remote_participant.update_info(pi.clone()); } } } @@ -438,7 +413,6 @@ impl SessionInner { let _ = self .dispatcher - .lock() .dispatch(&RoomEvent::ActiveSpeakersChanged { speakers }); } @@ -466,7 +440,6 @@ impl SessionInner { participant.set_connection_quality(quality); self.dispatcher - .lock() .dispatch(&RoomEvent::ConnectionQualityChanged { participant, quality, @@ -483,7 +456,7 @@ impl SessionInner { } if self.update_connection_state(ConnectionState::Reconnecting) { - self.dispatcher.lock().dispatch(&RoomEvent::Reconnecting); + self.dispatcher.dispatch(&RoomEvent::Reconnecting); } } @@ -493,10 +466,10 @@ impl SessionInner { let join_response = self.rtc_engine.join_response().unwrap(); self.update_connection_state(ConnectionState::Connected); - self.dispatcher.lock().dispatch(&RoomEvent::Reconnected); + self.dispatcher.dispatch(&RoomEvent::Reconnected); if let Some(pi) = join_response.participant { - self.local_participant.update_info(pi, true); // The sid may have changed + self.local_participant.update_info(pi); // The sid may have changed } self.handle_participant_update(join_response.other_participants); @@ -511,7 +484,7 @@ impl SessionInner { } self.update_connection_state(ConnectionState::Disconnected); - self.dispatcher.lock().dispatch(&RoomEvent::Disconnected); + self.dispatcher.dispatch(&RoomEvent::Disconnected); } /// Create a new participant @@ -523,13 +496,8 @@ impl SessionInner { identity: ParticipantIdentity, name: String, metadata: String, - ) -> Arc { - let participant = Arc::new(RemoteParticipant::new( - sid.clone(), - identity, - name, - metadata, - )); + ) -> RemoteParticipant { + let participant = RemoteParticipant::new(sid.clone(), identity, name, metadata); // Create the participant task let (close_tx, close_rx) = oneshot::channel(); @@ -549,10 +517,10 @@ impl SessionInner { /// A participant has disconnected /// Cleanup the participant and emit an event #[instrument(level = Level::DEBUG)] - fn handle_participant_disconnect(self: Arc, remote_participant: Arc) { + fn handle_participant_disconnect(self: Arc, remote_participant: RemoteParticipant) { tokio::spawn(async move { for (sid, _) in &*remote_participant.tracks() { - remote_participant.unpublish_track(&sid, true); + remote_participant.unpublish_track(&sid); } // Close the participant task @@ -566,16 +534,12 @@ impl SessionInner { } self.participants.write().remove(&remote_participant.sid()); - self.dispatcher - .lock() - .dispatch(&RoomEvent::ParticipantDisconnected( - remote_participant.clone(), - )); + .dispatch(&RoomEvent::ParticipantDisconnected(remote_participant)); }); } - fn get_participant(&self, sid: &ParticipantSid) -> Option> { + fn get_participant(&self, sid: &ParticipantSid) -> Option { self.participants.read().get(sid).cloned() } } @@ -590,3 +554,14 @@ fn unpack_stream_id(stream_id: &str) -> Option<(&str, &str)> { None } } + +impl From for ConnectionState { + fn from(value: u8) -> Self { + match value { + 0 => ConnectionState::Disconnected, + 1 => ConnectionState::Connected, + 2 => ConnectionState::Reconnecting, + _ => ConnectionState::Unknown, + } + } +} diff --git a/livekit/src/room/track/audio_track.rs b/livekit/src/room/track/audio_track.rs deleted file mode 100644 index 12960ff..0000000 --- a/livekit/src/room/track/audio_track.rs +++ /dev/null @@ -1,32 +0,0 @@ -use super::impl_track_trait; -use crate::prelude::*; -use std::sync::Arc; - -#[derive(Clone)] -pub enum AudioTrackHandle { - Local(Arc), - Remote(Arc), -} - -impl From for TrackHandle { - fn from(audio_track: AudioTrackHandle) -> Self { - match audio_track { - AudioTrackHandle::Local(local_audio) => Self::LocalAudio(local_audio), - AudioTrackHandle::Remote(remote_audio) => Self::RemoteAudio(remote_audio), - } - } -} - -impl TryFrom for AudioTrackHandle { - type Error = &'static str; - - fn try_from(track: TrackHandle) -> Result { - match track { - TrackHandle::LocalAudio(local_audio) => Ok(Self::Local(local_audio)), - TrackHandle::RemoteAudio(remote_audio) => Ok(Self::Remote(remote_audio)), - _ => Err("not a audio track"), - } - } -} - -impl_track_trait!(AudioTrackHandle, enum_dispatch, [Local, Remote]); diff --git a/livekit/src/room/track/local_audio_track.rs b/livekit/src/room/track/local_audio_track.rs index 860068c..09f08d2 100644 --- a/livekit/src/room/track/local_audio_track.rs +++ b/livekit/src/room/track/local_audio_track.rs @@ -1,8 +1,105 @@ -use super::{impl_track_trait, TrackShared}; +use super::TrackInner; +use crate::prelude::*; +use crate::proto; +use livekit_webrtc as rtc; +use std::sync::Arc; +use tokio::sync::mpsc; -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct LocalAudioTrack { - shared: TrackShared, + pub(crate) inner: Arc, } -impl_track_trait!(LocalAudioTrack); +impl LocalAudioTrack { + pub(crate) fn new( + sid: TrackSid, + name: String, + rtc_track: rtc::media_stream::RtcAudioTrack, + ) -> Self { + Self { + inner: Arc::new(TrackInner::new( + sid, + name, + TrackKind::Audio, + rtc::media_stream::MediaStreamTrack::Audio(rtc_track), + )), + } + } + + #[inline] + pub fn sid(&self) -> TrackSid { + self.inner.sid() + } + + #[inline] + pub fn name(&self) -> String { + self.inner.name() + } + + #[inline] + pub fn kind(&self) -> TrackKind { + self.inner.kind() + } + + #[inline] + pub fn source(&self) -> TrackSource { + self.inner.source() + } + + #[inline] + pub fn stream_state(&self) -> StreamState { + self.inner.stream_state() + } + + #[inline] + pub fn start(&self) { + self.inner.start() + } + + #[inline] + pub fn stop(&self) { + self.inner.stop() + } + + #[inline] + pub fn muted(&self) -> bool { + self.inner.muted() + } + + #[inline] + pub fn set_muted(&self, muted: bool) { + self.inner.set_muted(muted) + } + + #[inline] + pub fn rtc_track(&self) -> rtc::media_stream::RtcAudioTrack { + if let rtc::media_stream::MediaStreamTrack::Audio(audio) = self.inner.rtc_track() { + audio + } else { + unreachable!() + } + } + + #[inline] + pub fn register_observer(&self) -> mpsc::UnboundedReceiver { + self.inner.register_observer() + } + + #[inline] + pub(crate) fn transceiver(&self) -> Option { + self.inner.transceiver() + } + + #[inline] + pub(crate) fn update_transceiver( + &self, + transceiver: Option, + ) { + self.inner.update_transceiver(transceiver) + } + + #[inline] + pub(crate) fn update_info(&self, info: proto::TrackInfo) { + self.inner.update_info(info) + } +} diff --git a/livekit/src/room/track/local_track.rs b/livekit/src/room/track/local_track.rs deleted file mode 100644 index 82ac845..0000000 --- a/livekit/src/room/track/local_track.rs +++ /dev/null @@ -1,32 +0,0 @@ -use super::impl_track_trait; -use crate::prelude::*; -use std::sync::Arc; - -#[derive(Clone)] -pub enum LocalTrackHandle { - Audio(Arc), - Video(Arc), -} - -impl From for TrackHandle { - fn from(local_track: LocalTrackHandle) -> Self { - match local_track { - LocalTrackHandle::Audio(local_audio) => Self::LocalAudio(local_audio), - LocalTrackHandle::Video(local_video) => Self::LocalVideo(local_video), - } - } -} - -impl TryFrom for LocalTrackHandle { - type Error = &'static str; - - fn try_from(track: TrackHandle) -> Result { - match track { - TrackHandle::LocalAudio(local_audio) => Ok(Self::Audio(local_audio)), - TrackHandle::LocalVideo(local_video) => Ok(Self::Video(local_video)), - _ => Err("not a local track"), - } - } -} - -impl_track_trait!(LocalTrackHandle, enum_dispatch, [Audio, Video]); diff --git a/livekit/src/room/track/local_video_track.rs b/livekit/src/room/track/local_video_track.rs index 6fb5d52..51064b3 100644 --- a/livekit/src/room/track/local_video_track.rs +++ b/livekit/src/room/track/local_video_track.rs @@ -1,8 +1,138 @@ -use super::{impl_track_trait, TrackShared}; +use super::TrackInner; +use crate::proto; +use crate::rtc_engine::lk_runtime::LkRuntime; +use crate::{options::VideoCaptureOptions, prelude::*}; +use livekit_webrtc as rtc; +use livekit_webrtc::peer_connection_factory::native::PeerConnectionFactoryExt; +use parking_lot::Mutex; +use rtc::video_source::native::NativeVideoSource; +use std::sync::Arc; +use tokio::sync::mpsc; #[derive(Debug)] -pub struct LocalVideoTrack { - shared: TrackShared, +struct LocalVideoTrackInner { + track_inner: TrackInner, + capture_options: Mutex, } -impl_track_trait!(LocalVideoTrack); +#[derive(Clone, Debug)] +pub struct LocalVideoTrack { + inner: Arc, +} + +impl LocalVideoTrack { + pub fn new( + name: String, + rtc_track: rtc::media_stream::RtcVideoTrack, + capture_options: VideoCaptureOptions, + ) -> Self { + Self { + inner: Arc::new(LocalVideoTrackInner { + track_inner: TrackInner::new( + "unknown".to_string().into(), // sid + name, + TrackKind::Video, + rtc::media_stream::MediaStreamTrack::Video(rtc_track), + ), + capture_options: Mutex::new(capture_options), + }), + } + } + + pub fn capture_options(&self) -> VideoCaptureOptions { + self.inner.capture_options.lock().clone() + } + + #[inline] + pub fn sid(&self) -> TrackSid { + self.inner.track_inner.sid() + } + + #[inline] + pub fn name(&self) -> String { + self.inner.track_inner.name() + } + + #[inline] + pub fn kind(&self) -> TrackKind { + self.inner.track_inner.kind() + } + + #[inline] + pub fn source(&self) -> TrackSource { + self.inner.track_inner.source() + } + + #[inline] + pub fn stream_state(&self) -> StreamState { + self.inner.track_inner.stream_state() + } + + #[inline] + pub fn start(&self) { + self.inner.track_inner.start() + } + + #[inline] + pub fn stop(&self) { + self.inner.track_inner.stop() + } + + #[inline] + pub fn muted(&self) -> bool { + self.inner.track_inner.muted() + } + + #[inline] + pub fn set_muted(&self, muted: bool) { + self.inner.track_inner.set_muted(muted) + } + + #[inline] + pub fn rtc_track(&self) -> rtc::media_stream::RtcVideoTrack { + if let rtc::media_stream::MediaStreamTrack::Video(video) = + self.inner.track_inner.rtc_track() + { + video + } else { + unreachable!() + } + } + + #[inline] + pub fn register_observer(&self) -> mpsc::UnboundedReceiver { + self.inner.track_inner.register_observer() + } + + #[inline] + pub fn transceiver(&self) -> Option { + self.inner.track_inner.transceiver() + } + + #[inline] + pub(crate) fn update_transceiver( + &self, + transceiver: Option, + ) { + self.inner.track_inner.update_transceiver(transceiver) + } + + #[inline] + pub(crate) fn update_info(&self, info: proto::TrackInfo) { + self.inner.track_inner.update_info(info) + } +} + +impl LocalVideoTrack { + pub fn create_video_track( + name: &str, + options: VideoCaptureOptions, + source: NativeVideoSource, + ) -> LocalVideoTrack { + let rtc_track = LkRuntime::instance() + .pc_factory + .create_video_track(&rtc::native::create_random_uuid(), source); + + Self::new(name.to_string(), rtc_track, options) + } +} diff --git a/livekit/src/room/track/mod.rs b/livekit/src/room/track/mod.rs index f224ff0..2df12ff 100644 --- a/livekit/src/room/track/mod.rs +++ b/livekit/src/room/track/mod.rs @@ -2,29 +2,22 @@ use crate::prelude::*; use crate::proto; use livekit_utils::enum_dispatch; use livekit_utils::observer::Dispatcher; +use livekit_webrtc as rtc; use parking_lot::Mutex; +use rtc::MediaType; use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; -use std::sync::Arc; use thiserror::Error; use tokio::sync::mpsc; -pub mod audio_track; pub mod local_audio_track; -pub mod local_track; pub mod local_video_track; pub mod remote_audio_track; -pub mod remote_track; pub mod remote_video_track; -pub mod video_track; -pub use audio_track::*; pub use local_audio_track::*; -pub use local_track::*; pub use local_video_track::*; pub use remote_audio_track::*; -pub use remote_track::*; pub use remote_video_track::*; -pub use video_track::*; #[derive(Error, Debug, Clone)] pub enum TrackError { @@ -32,51 +25,19 @@ pub enum TrackError { TrackNotFound(String), } -#[derive(Debug)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum TrackKind { - Unknown, Audio, Video, } -impl From for TrackKind { - fn from(val: u8) -> Self { - match val { - 1 => Self::Audio, - 2 => Self::Video, - _ => Self::Unknown, - } - } -} - -impl From for TrackKind { - fn from(r#type: proto::TrackType) -> Self { - match r#type { - proto::TrackType::Audio => Self::Audio, - proto::TrackType::Video => Self::Video, - _ => Self::Unknown, - } - } -} - -#[derive(Debug)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum StreamState { - Unknown, Active, Paused, } -impl From for StreamState { - fn from(val: u8) -> Self { - match val { - 1 => Self::Active, - 2 => Self::Paused, - _ => Self::Unknown, - } - } -} - -#[derive(Debug)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum TrackSource { Unknown, Camera, @@ -85,9 +46,364 @@ pub enum TrackSource { ScreenshareAudio, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TrackEvent { + Mute, + Unmute, +} + +#[derive(Clone, Copy, Debug)] +pub struct TrackDimension(pub u32, pub u32); + +#[derive(Clone, Debug)] +pub enum Track { + LocalAudio(LocalAudioTrack), + LocalVideo(LocalVideoTrack), + RemoteAudio(RemoteAudioTrack), + RemoteVideo(RemoteVideoTrack), +} + +#[derive(Clone, Debug)] +pub enum LocalTrack { + Audio(LocalAudioTrack), + Video(LocalVideoTrack), +} + +#[derive(Clone, Debug)] +pub enum RemoteTrack { + Audio(RemoteAudioTrack), + Video(RemoteVideoTrack), +} + +#[derive(Clone, Debug)] +pub enum VideoTrack { + Local(LocalVideoTrack), + Remote(RemoteVideoTrack), +} + +#[derive(Clone, Debug)] +pub enum AudioTrack { + Local(LocalAudioTrack), + Remote(RemoteAudioTrack), +} + +macro_rules! track_dispatch { + ([$($variant:ident),+]) => { + enum_dispatch!( + [$($variant),+]; + pub fn sid(self: &Self) -> TrackSid; + pub fn name(self: &Self) -> String; + pub fn kind(self: &Self) -> TrackKind; + pub fn source(self: &Self) -> TrackSource; + pub fn stream_state(self: &Self) -> StreamState; + pub fn start(self: &Self) -> (); + pub fn stop(self: &Self) -> (); + pub fn muted(self: &Self) -> bool; + pub fn set_muted(self: &Self, muted: bool) -> (); + pub fn register_observer(self: &Self) -> mpsc::UnboundedReceiver; + + pub(crate) fn transceiver(self: &Self) -> Option; + pub(crate) fn update_transceiver(self: &Self, transceiver: Option) -> (); + pub(crate) fn update_info(self: &Self, info: proto::TrackInfo) -> (); + ); + }; +} + +impl Track { + track_dispatch!([LocalAudio, LocalVideo, RemoteAudio, RemoteVideo]); + + #[inline] + pub fn rtc_track(&self) -> rtc::media_stream::MediaStreamTrack { + match self { + Self::LocalAudio(track) => track.rtc_track().into(), + Self::LocalVideo(track) => track.rtc_track().into(), + Self::RemoteAudio(track) => track.rtc_track().into(), + Self::RemoteVideo(track) => track.rtc_track().into(), + } + } +} + +impl LocalTrack { + track_dispatch!([Audio, Video]); + + #[inline] + pub fn rtc_track(&self) -> rtc::media_stream::MediaStreamTrack { + match self { + Self::Audio(track) => track.rtc_track().into(), + Self::Video(track) => track.rtc_track().into(), + } + } +} + +impl RemoteTrack { + track_dispatch!([Audio, Video]); + + #[inline] + pub fn rtc_track(&self) -> rtc::media_stream::MediaStreamTrack { + match self { + Self::Audio(track) => track.rtc_track().into(), + Self::Video(track) => track.rtc_track().into(), + } + } +} + +impl VideoTrack { + track_dispatch!([Local, Remote]); + + #[inline] + pub fn rtc_track(&self) -> rtc::media_stream::RtcVideoTrack { + match self { + Self::Local(track) => track.rtc_track(), + Self::Remote(track) => track.rtc_track(), + } + } +} + +impl AudioTrack { + track_dispatch!([Local, Remote]); + + #[inline] + pub fn rtc_track(&self) -> rtc::media_stream::RtcAudioTrack { + match self { + Self::Local(track) => track.rtc_track().into(), + Self::Remote(track) => track.rtc_track().into(), + } + } +} + +#[derive(Debug)] +pub(crate) struct TrackInner { + pub sid: Mutex, + pub name: Mutex, + pub kind: AtomicU8, // TrackKind + pub source: AtomicU8, // TrackSource + pub stream_state: AtomicU8, // StreamState + pub muted: AtomicBool, + pub rtc_track: rtc::media_stream::MediaStreamTrack, + pub transceiver: Mutex>, + pub dispatcher: Dispatcher, +} + +impl TrackInner { + pub fn new( + sid: TrackSid, + name: String, + kind: TrackKind, + rtc_track: rtc::media_stream::MediaStreamTrack, + ) -> Self { + Self { + sid: Mutex::new(sid), + name: Mutex::new(name), + kind: AtomicU8::new(kind as u8), + source: AtomicU8::new(TrackSource::Unknown as u8), + stream_state: AtomicU8::new(StreamState::Active as u8), + muted: AtomicBool::new(false), + rtc_track, + transceiver: Default::default(), + dispatcher: Default::default(), + } + } + + pub fn sid(&self) -> TrackSid { + self.sid.lock().clone() + } + + pub fn name(&self) -> String { + self.name.lock().clone() + } + + pub fn kind(&self) -> TrackKind { + self.kind.load(Ordering::SeqCst).try_into().unwrap() + } + + pub fn source(&self) -> TrackSource { + self.source.load(Ordering::SeqCst).into() + } + + pub fn stream_state(&self) -> StreamState { + self.stream_state.load(Ordering::SeqCst).try_into().unwrap() + } + + pub fn muted(&self) -> bool { + self.muted.load(Ordering::SeqCst) + } + + pub fn start(&self) { + self.rtc_track.set_enabled(true); + } + + pub fn stop(&self) { + self.rtc_track.set_enabled(false); + } + + pub fn set_muted(&self, muted: bool) { + if self + .muted + .compare_exchange(!muted, muted, Ordering::SeqCst, Ordering::SeqCst) + .is_err() + { + return; + } + + if !muted { + self.start(); + } else { + self.stop(); + } + + let event = if muted { + TrackEvent::Mute + } else { + TrackEvent::Unmute + }; + + self.dispatcher.dispatch(&event); + } + + pub fn rtc_track(&self) -> rtc::media_stream::MediaStreamTrack { + self.rtc_track.clone() + } + + pub fn register_observer(&self) -> mpsc::UnboundedReceiver { + self.dispatcher.register() + } + + pub fn transceiver(&self) -> Option { + self.transceiver.lock().clone() + } + + pub fn update_transceiver(&self, transceiver: Option) { + *self.transceiver.lock() = transceiver; + } + + pub fn update_info(&self, info: proto::TrackInfo) { + *self.name.lock() = info.name; + *self.sid.lock() = info.sid.into(); + self.kind.store( + TrackKind::try_from(proto::TrackType::from_i32(info.r#type).unwrap()).unwrap() as u8, + Ordering::SeqCst, + ); + self.source.store( + TrackSource::from(proto::TrackSource::from_i32(info.source).unwrap()) as u8, + Ordering::SeqCst, + ); + // Muted and StreamState are not handled separately (events) + } +} + +impl From for Track { + fn from(track: RemoteTrack) -> Self { + match track { + RemoteTrack::Audio(track) => Self::RemoteAudio(track), + RemoteTrack::Video(track) => Self::RemoteVideo(track), + } + } +} + +impl From for Track { + fn from(track: LocalTrack) -> Self { + match track { + LocalTrack::Audio(track) => Self::LocalAudio(track), + LocalTrack::Video(track) => Self::LocalVideo(track), + } + } +} + +impl From for Track { + fn from(track: VideoTrack) -> Self { + match track { + VideoTrack::Local(track) => Self::LocalVideo(track), + VideoTrack::Remote(track) => Self::RemoteVideo(track), + } + } +} + +impl From for Track { + fn from(track: AudioTrack) -> Self { + match track { + AudioTrack::Local(track) => Self::LocalAudio(track), + AudioTrack::Remote(track) => Self::RemoteAudio(track), + } + } +} + +impl TryFrom for RemoteTrack { + type Error = &'static str; + + fn try_from(track: Track) -> Result { + match track { + Track::RemoteAudio(track) => Ok(Self::Audio(track)), + Track::RemoteVideo(track) => Ok(Self::Video(track)), + _ => Err("not a remote track"), + } + } +} + +impl TryFrom for LocalTrack { + type Error = &'static str; + + fn try_from(track: Track) -> Result { + match track { + Track::LocalAudio(track) => Ok(Self::Audio(track)), + Track::LocalVideo(track) => Ok(Self::Video(track)), + _ => Err("not a local track"), + } + } +} + +impl TryFrom for VideoTrack { + type Error = &'static str; + + fn try_from(track: Track) -> Result { + match track { + Track::LocalVideo(track) => Ok(Self::Local(track)), + Track::RemoteVideo(track) => Ok(Self::Remote(track)), + _ => Err("not a video track"), + } + } +} + +impl TryFrom for AudioTrack { + type Error = &'static str; + + fn try_from(track: Track) -> Result { + match track { + Track::LocalAudio(track) => Ok(Self::Local(track)), + Track::RemoteAudio(track) => Ok(Self::Remote(track)), + _ => Err("not an audio track"), + } + } +} + +// Conversions from integers (Useful since we're using atomic values to represent our enums) + +impl TryFrom for TrackKind { + type Error = &'static str; + + fn try_from(kind: u8) -> Result { + match kind { + 0 => Ok(Self::Audio), + 1 => Ok(Self::Video), + _ => Err("invalid track kind"), + } + } +} + +impl TryFrom for StreamState { + type Error = &'static str; + + fn try_from(state: u8) -> Result { + match state { + 0 => Ok(Self::Active), + 1 => Ok(Self::Paused), + _ => Err("invalid stream state"), + } + } +} + impl From for TrackSource { - fn from(val: u8) -> Self { - match val { + fn from(source: u8) -> Self { + match source { 1 => Self::Camera, 2 => Self::Microphone, 3 => Self::Screenshare, @@ -97,197 +413,11 @@ impl From for TrackSource { } } -impl From for TrackSource { - fn from(source: proto::TrackSource) -> Self { - match source { - proto::TrackSource::Camera => Self::Camera, - proto::TrackSource::Microphone => Self::Microphone, - proto::TrackSource::ScreenShare => Self::Screenshare, - proto::TrackSource::ScreenShareAudio => Self::ScreenshareAudio, - proto::TrackSource::Unknown => Self::Unknown, +impl From for MediaType { + fn from(kind: TrackKind) -> Self { + match kind { + TrackKind::Audio => Self::Audio, + TrackKind::Video => Self::Video, } } } - -#[derive(Clone, Copy, Debug)] -pub struct TrackDimension(pub u32, pub u32); - -pub trait TrackTrait { - fn sid(&self) -> TrackSid; - fn name(&self) -> String; - fn kind(&self) -> TrackKind; - fn stream_state(&self) -> StreamState; - fn muted(&self) -> bool; - fn start(&self); - fn stop(&self); - fn register_observer(&self) -> mpsc::UnboundedReceiver; - fn set_muted(&self, muted: bool); -} - -#[derive(Debug, Clone)] -pub enum TrackEvent { - Mute, - Unmute, -} - -#[derive(Debug)] -pub(super) struct TrackShared { - pub(super) sid: Mutex, - pub(super) name: Mutex, - pub(super) kind: AtomicU8, // TrackKind - pub(super) stream_state: AtomicU8, // StreamState - pub(super) muted: AtomicBool, - pub(super) rtc_track: MediaStreamTrackHandle, - pub(super) dispatcher: Mutex>, -} - -impl TrackShared { - pub(crate) fn new( - sid: TrackSid, - name: String, - kind: TrackKind, - rtc_track: MediaStreamTrackHandle, - ) -> Self { - Self { - sid: Mutex::new(sid), - name: Mutex::new(name), - kind: AtomicU8::new(kind as u8), - stream_state: AtomicU8::new(StreamState::Active as u8), - muted: AtomicBool::new(false), - rtc_track, - dispatcher: Default::default(), - } - } - - pub(crate) fn start(&self) { - self.rtc_track.set_enabled(true); - } - - pub(crate) fn stop(&self) { - self.rtc_track.set_enabled(false); - } - - pub(crate) fn set_muted(&self, muted: bool) { - if self.muted.load(Ordering::SeqCst) == muted { - return; - } - - self.muted.store(muted, Ordering::SeqCst); - self.rtc_track.set_enabled(!muted); - - self.dispatcher.lock().dispatch(if muted { - &TrackEvent::Mute - } else { - &TrackEvent::Unmute - }); - } - - pub(crate) fn register_observer(&self) -> mpsc::UnboundedReceiver { - self.dispatcher.lock().register() - } -} - -#[derive(Clone, Debug)] -pub enum TrackHandle { - LocalVideo(Arc), - LocalAudio(Arc), - RemoteVideo(Arc), - RemoteAudio(Arc), -} - -impl TrackTrait for TrackHandle { - enum_dispatch!( - [LocalVideo, LocalAudio, RemoteVideo, RemoteAudio] - fnc!(sid, &Self, [], TrackSid); - fnc!(name, &Self, [], String); - fnc!(kind, &Self, [], TrackKind); - fnc!(stream_state, &Self, [], StreamState); - fnc!(muted, &Self, [], bool); - fnc!(start, &Self, [], ()); - fnc!(stop, &Self, [], ()); - fnc!(register_observer, &Self, [], mpsc::UnboundedReceiver); - fnc!(set_muted, &Self, [muted: bool], ()); - ); -} - -impl TrackHandle { - pub fn rtc_track(&self) -> MediaStreamTrackHandle { - match self { - Self::RemoteVideo(remote_video) => { - MediaStreamTrackHandle::Video(remote_video.rtc_track()) - } - Self::RemoteAudio(remote_audio) => { - MediaStreamTrackHandle::Audio(remote_audio.rtc_track()) - } - _ => todo!(), - } - } -} - -macro_rules! impl_track_trait { - ($x:ident) => { - use std::sync::atomic::Ordering; - use tokio::sync::mpsc; - use $crate::room::id::TrackSid; - use $crate::room::track::{StreamState, TrackEvent, TrackKind, TrackTrait}; - - impl TrackTrait for $x { - fn sid(&self) -> TrackSid { - self.shared.sid.lock().clone() - } - - fn name(&self) -> String { - self.shared.name.lock().clone() - } - - fn kind(&self) -> TrackKind { - self.shared.kind.load(Ordering::SeqCst).into() - } - - fn stream_state(&self) -> StreamState { - self.shared.stream_state.load(Ordering::SeqCst).into() - } - - fn muted(&self) -> bool { - self.shared.muted.load(Ordering::SeqCst) - } - - fn start(&self) { - self.shared.start(); - } - - fn stop(&self) { - self.shared.stop(); - } - - fn register_observer(&self) -> mpsc::UnboundedReceiver { - self.shared.register_observer() - } - - fn set_muted(&self, muted: bool) { - self.shared.set_muted(muted); - } - } - }; - ($x:ident, enum_dispatch, [$($variant:ident),+]) => { - use livekit_utils::enum_dispatch; - use tokio::sync::mpsc; - - impl TrackTrait for $x { - enum_dispatch!( - [$($variant),+] - fnc!(sid, &Self, [], TrackSid); - fnc!(name, &Self, [], String); - fnc!(kind, &Self, [], TrackKind); - fnc!(stream_state, &Self, [], StreamState); - fnc!(muted, &Self, [], bool); - fnc!(start, &Self, [], ()); - fnc!(stop, &Self, [], ()); - fnc!(register_observer, &Self, [], mpsc::UnboundedReceiver); - fnc!(set_muted, &Self, [muted: bool], ()); - ); - } - }; -} - -pub(super) use impl_track_trait; diff --git a/livekit/src/room/track/remote_audio_track.rs b/livekit/src/room/track/remote_audio_track.rs index d691a78..5b76030 100644 --- a/livekit/src/room/track/remote_audio_track.rs +++ b/livekit/src/room/track/remote_audio_track.rs @@ -1,31 +1,105 @@ -use super::{impl_track_trait, TrackShared}; +use super::TrackInner; use crate::prelude::*; +use crate::proto; +use livekit_webrtc as rtc; use std::sync::Arc; +use tokio::sync::mpsc; -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct RemoteAudioTrack { - shared: TrackShared, + pub(crate) inner: Arc, } impl RemoteAudioTrack { - pub(crate) fn new(sid: TrackSid, name: String, track: Arc) -> Self { + pub(crate) fn new( + sid: TrackSid, + name: String, + rtc_track: rtc::media_stream::RtcAudioTrack, + ) -> Self { Self { - shared: TrackShared::new( + inner: Arc::new(TrackInner::new( sid, name, TrackKind::Audio, - MediaStreamTrackHandle::Audio(track), - ), + rtc::media_stream::MediaStreamTrack::Audio(rtc_track), + )), } } - pub fn rtc_track(&self) -> Arc { - if let MediaStreamTrackHandle::Audio(audio) = &self.shared.rtc_track { - audio.clone() + #[inline] + pub fn sid(&self) -> TrackSid { + self.inner.sid() + } + + #[inline] + pub fn name(&self) -> String { + self.inner.name() + } + + #[inline] + pub fn kind(&self) -> TrackKind { + self.inner.kind() + } + + #[inline] + pub fn source(&self) -> TrackSource { + self.inner.source() + } + + #[inline] + pub fn stream_state(&self) -> StreamState { + self.inner.stream_state() + } + + #[inline] + pub fn start(&self) { + self.inner.start() + } + + #[inline] + pub fn stop(&self) { + self.inner.stop() + } + + #[inline] + pub fn muted(&self) -> bool { + self.inner.muted() + } + + #[inline] + pub fn set_muted(&self, muted: bool) { + self.inner.set_muted(muted) + } + + #[inline] + pub fn rtc_track(&self) -> rtc::media_stream::RtcAudioTrack { + if let rtc::media_stream::MediaStreamTrack::Audio(audio) = self.inner.rtc_track() { + audio } else { unreachable!() } } -} -impl_track_trait!(RemoteAudioTrack); + #[inline] + pub fn register_observer(&self) -> mpsc::UnboundedReceiver { + self.inner.register_observer() + } + + #[inline] + pub(crate) fn transceiver(&self) -> Option { + self.inner.transceiver() + } + + #[inline] + pub(crate) fn update_transceiver( + &self, + transceiver: Option, + ) { + self.inner.update_transceiver(transceiver) + } + + #[inline] + pub(crate) fn update_info(&self, info: proto::TrackInfo) { + self.inner.update_info(info) + } +} diff --git a/livekit/src/room/track/remote_track.rs b/livekit/src/room/track/remote_track.rs deleted file mode 100644 index a42a57e..0000000 --- a/livekit/src/room/track/remote_track.rs +++ /dev/null @@ -1,32 +0,0 @@ -use super::impl_track_trait; -use crate::prelude::*; -use std::sync::Arc; - -#[derive(Clone, Debug)] -pub enum RemoteTrackHandle { - Audio(Arc), - Video(Arc), -} - -impl From for TrackHandle { - fn from(remote_track: RemoteTrackHandle) -> Self { - match remote_track { - RemoteTrackHandle::Audio(remote_audio) => Self::RemoteAudio(remote_audio), - RemoteTrackHandle::Video(remote_video) => Self::RemoteVideo(remote_video), - } - } -} - -impl TryFrom for RemoteTrackHandle { - type Error = &'static str; - - fn try_from(track: TrackHandle) -> Result { - match track { - TrackHandle::RemoteAudio(remote_audio) => Ok(Self::Audio(remote_audio)), - TrackHandle::RemoteVideo(remote_video) => Ok(Self::Video(remote_video)), - _ => Err("not a remote track"), - } - } -} - -impl_track_trait!(RemoteTrackHandle, enum_dispatch, [Audio, Video]); diff --git a/livekit/src/room/track/remote_video_track.rs b/livekit/src/room/track/remote_video_track.rs index 428c403..cfaecf1 100644 --- a/livekit/src/room/track/remote_video_track.rs +++ b/livekit/src/room/track/remote_video_track.rs @@ -1,31 +1,105 @@ -use super::{impl_track_trait, TrackShared}; +use super::TrackInner; use crate::prelude::*; +use crate::proto; +use livekit_webrtc as rtc; use std::sync::Arc; +use tokio::sync::mpsc; -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct RemoteVideoTrack { - shared: TrackShared, + pub(crate) inner: Arc, } impl RemoteVideoTrack { - pub(crate) fn new(sid: TrackSid, name: String, track: Arc) -> Self { + pub(crate) fn new( + sid: TrackSid, + name: String, + rtc_track: rtc::media_stream::RtcVideoTrack, + ) -> Self { Self { - shared: TrackShared::new( + inner: Arc::new(TrackInner::new( sid, name, TrackKind::Video, - MediaStreamTrackHandle::Video(track), - ), + rtc::media_stream::MediaStreamTrack::Video(rtc_track), + )), } } - pub fn rtc_track(&self) -> Arc { - if let MediaStreamTrackHandle::Video(video) = &self.shared.rtc_track { - video.clone() + #[inline] + pub fn sid(&self) -> TrackSid { + self.inner.sid() + } + + #[inline] + pub fn name(&self) -> String { + self.inner.name() + } + + #[inline] + pub fn kind(&self) -> TrackKind { + self.inner.kind() + } + + #[inline] + pub fn source(&self) -> TrackSource { + self.inner.source() + } + + #[inline] + pub fn stream_state(&self) -> StreamState { + self.inner.stream_state() + } + + #[inline] + pub fn start(&self) { + self.inner.start() + } + + #[inline] + pub fn stop(&self) { + self.inner.stop() + } + + #[inline] + pub fn muted(&self) -> bool { + self.inner.muted() + } + + #[inline] + pub fn set_muted(&self, muted: bool) { + self.inner.set_muted(muted) + } + + #[inline] + pub fn rtc_track(&self) -> rtc::media_stream::RtcVideoTrack { + if let rtc::media_stream::MediaStreamTrack::Video(video) = self.inner.rtc_track() { + video } else { unreachable!() } } -} -impl_track_trait!(RemoteVideoTrack); + #[inline] + pub fn register_observer(&self) -> mpsc::UnboundedReceiver { + self.inner.register_observer() + } + + #[inline] + pub(crate) fn transceiver(&self) -> Option { + self.inner.transceiver() + } + + #[inline] + pub(crate) fn update_transceiver( + &self, + transceiver: Option, + ) { + self.inner.update_transceiver(transceiver) + } + + #[inline] + pub(crate) fn update_info(&self, info: proto::TrackInfo) { + self.inner.update_info(info); + } +} diff --git a/livekit/src/room/track/video_track.rs b/livekit/src/room/track/video_track.rs deleted file mode 100644 index beca465..0000000 --- a/livekit/src/room/track/video_track.rs +++ /dev/null @@ -1,32 +0,0 @@ -use super::impl_track_trait; -use crate::prelude::*; -use std::sync::Arc; - -#[derive(Clone)] -pub enum VideoTrackHandle { - Local(Arc), - Remote(Arc), -} - -impl From for TrackHandle { - fn from(video_track: VideoTrackHandle) -> Self { - match video_track { - VideoTrackHandle::Local(local_video) => Self::LocalVideo(local_video), - VideoTrackHandle::Remote(remote_video) => Self::RemoteVideo(remote_video), - } - } -} - -impl TryFrom for VideoTrackHandle { - type Error = &'static str; - - fn try_from(track: TrackHandle) -> Result { - match track { - TrackHandle::LocalVideo(local_video) => Ok(Self::Local(local_video)), - TrackHandle::RemoteVideo(remote_video) => Ok(Self::Remote(remote_video)), - _ => Err("not a video track"), - } - } -} - -impl_track_trait!(VideoTrackHandle, enum_dispatch, [Local, Remote]); diff --git a/livekit/src/rtc_engine/lk_runtime.rs b/livekit/src/rtc_engine/lk_runtime.rs index a8d3a4e..073b1b7 100644 --- a/livekit/src/rtc_engine/lk_runtime.rs +++ b/livekit/src/rtc_engine/lk_runtime.rs @@ -1,35 +1,48 @@ -use livekit_webrtc::peer_connection_factory::PeerConnectionFactory; -use livekit_webrtc::webrtc::RTCRuntime; +use lazy_static::lazy_static; +use livekit_webrtc::prelude::*; +use parking_lot::Mutex; use std::fmt::{Debug, Formatter}; +use std::sync::{Arc, Weak}; use tracing::trace; -/// SAFETY: The order of initialization and deletion is important for LKRuntime. -/// See the C++ constructors & destructors of these fields - -pub struct LKRuntime { - pub pc_factory: PeerConnectionFactory, - pub rtc_runtime: RTCRuntime, +lazy_static! { + static ref LK_RUNTIME: Mutex> = Mutex::new(Weak::new()); } -impl Debug for LKRuntime { +pub struct LkRuntime { + pub pc_factory: PeerConnectionFactory, +} + +impl Debug for LkRuntime { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - write!(f, "LKRuntime") + f.debug_struct("LkRuntime").finish() } } -impl Default for LKRuntime { - fn default() -> Self { - trace!("LKRuntime::default()"); - let rtc_runtime = RTCRuntime::new(); - Self { - pc_factory: PeerConnectionFactory::new(rtc_runtime.clone()), - rtc_runtime, +impl LkRuntime { + pub fn instance() -> Arc { + let mut lk_runtime_ref = LK_RUNTIME.lock(); + if let Some(lk_runtime) = lk_runtime_ref.upgrade() { + lk_runtime + } else { + let new_runtime = Arc::new(LkRuntime::default()); + *lk_runtime_ref = Arc::downgrade(&new_runtime); + new_runtime } } } -impl Drop for LKRuntime { - fn drop(&mut self) { - trace!("LKRuntime::drop()"); +impl Default for LkRuntime { + fn default() -> Self { + trace!("LkRuntime::default()"); + Self { + pc_factory: PeerConnectionFactory::default(), + } + } +} + +impl Drop for LkRuntime { + fn drop(&mut self) { + trace!("LkRuntime::drop()"); } } diff --git a/livekit/src/rtc_engine/mod.rs b/livekit/src/rtc_engine/mod.rs index ad2dd5b..b28e53b 100644 --- a/livekit/src/rtc_engine/mod.rs +++ b/livekit/src/rtc_engine/mod.rs @@ -1,14 +1,11 @@ -use crate::prelude::*; +use crate::options::TrackPublishOptions; +use crate::prelude::LocalTrack; use crate::proto; -use crate::rtc_engine::lk_runtime::LKRuntime; -use crate::rtc_engine::rtc_session::{RTCSession, SessionEvent, SessionEvents, SessionInfo}; +use crate::rtc_engine::lk_runtime::LkRuntime; +use crate::rtc_engine::rtc_session::{RtcSession, SessionEvent, SessionEvents, SessionInfo}; use crate::signal_client::{SignalError, SignalOptions}; -use futures::future::BoxFuture; -use futures::FutureExt; -use lazy_static::lazy_static; -use livekit_webrtc::data_channel::DataSendError; -use livekit_webrtc::jsep::SdpParseError; use livekit_webrtc::prelude::*; +use livekit_webrtc::session_description::SdpParseError; use parking_lot::Mutex; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Weak}; @@ -20,8 +17,8 @@ use tokio::task::JoinHandle; use tokio::time::{interval, Interval}; use tracing::{error, info, warn}; -mod lk_runtime; -mod pc_transport; +pub mod lk_runtime; +mod peer_transport; mod rtc_events; mod rtc_session; @@ -46,13 +43,13 @@ pub enum EngineError { #[error("signal failure: {0}")] Signal(#[from] SignalError), #[error("internal webrtc failure")] - Rtc(#[from] RTCError), + Rtc(#[from] RtcError), #[error("failed to parse sdp")] Parse(#[from] SdpParseError), #[error("serde error")] Serde(#[from] serde_json::Error), #[error("failed to send data to the datachannel")] - Data(#[from] DataSendError), + Data(#[from] DataChannelError), #[error("connection error: {0}")] Connection(String), #[error("decode error")] @@ -67,7 +64,7 @@ pub enum EngineEvent { updates: Vec, }, MediaTrack { - track: MediaStreamTrackHandle, + track: MediaStreamTrack, stream: MediaStream, receiver: RtpReceiver, }, @@ -92,23 +89,19 @@ pub enum EngineEvent { pub const RECONNECT_ATTEMPTS: u32 = 10; pub const RECONNECT_INTERVAL: Duration = Duration::from_secs(5); -lazy_static! { - // Share one LKRuntime across all RTCEngine instances - static ref LK_RUNTIME: Mutex> = Mutex::new(Weak::new()); -} /// /// Represents a running RTCSession with the ability to close the session /// and the engine_task #[derive(Debug)] struct EngineHandle { - session: RTCSession, + session: RtcSession, engine_task: JoinHandle<()>, close_sender: oneshot::Sender<()>, } #[derive(Debug)] struct EngineInner { - lk_runtime: Arc, + lk_runtime: Arc, session_info: Mutex>, // Last/Current Sessioninfo running_handle: AsyncRwLock>, opened: AtomicBool, @@ -121,26 +114,15 @@ struct EngineInner { } #[derive(Debug)] -pub struct RTCEngine { +pub struct RtcEngine { inner: Arc, } -impl RTCEngine { +impl RtcEngine { pub fn new() -> (Self, EngineEvents) { - let lk_runtime = { - let mut lk_runtime_ref = LK_RUNTIME.lock(); - if let Some(lk_runtime) = lk_runtime_ref.upgrade() { - lk_runtime - } else { - let new_runtime = Arc::new(LKRuntime::default()); - *lk_runtime_ref = Arc::downgrade(&new_runtime); - new_runtime - } - }; - let (engine_emitter, engine_events) = mpsc::channel(8); let inner = Arc::new(EngineInner { - lk_runtime, + lk_runtime: LkRuntime::instance(), session_info: Default::default(), running_handle: Default::default(), opened: Default::default(), @@ -153,6 +135,10 @@ impl RTCEngine { (Self { inner }, engine_events) } + pub(crate) fn lk_runtime(&self) -> Arc { + self.inner.lk_runtime.clone() + } + #[tracing::instrument] pub async fn connect( &self, @@ -200,6 +186,64 @@ impl RTCEngine { Ok(()) } + pub async fn add_track(&self, req: proto::AddTrackRequest) -> EngineResult { + self.inner.wait_reconnection().await?; + self.inner + .running_handle + .read() + .await + .as_ref() + .unwrap() + .session + .add_track(req) + .await + } + + pub async fn remove_track(&self, sender: RtpSender) -> EngineResult<()> { + self.inner.wait_reconnection().await?; + self.inner + .running_handle + .read() + .await + .as_ref() + .unwrap() + .session + .remove_track(sender) + .await + } + + pub async fn create_sender( + &self, + track: LocalTrack, + options: TrackPublishOptions, + encodings: Vec, + ) -> EngineResult { + self.inner.wait_reconnection().await?; + self.inner + .running_handle + .read() + .await + .as_ref() + .unwrap() + .session + .create_sender(track, options, encodings) + .await + } + + pub async fn negotiate_publisher(&self) -> EngineResult<()> { + // TODO(theomonnom): guard for reconnection + self.inner.wait_reconnection().await?; + self.inner + .running_handle + .read() + .await + .as_ref() + .unwrap() + .session + .negotiate_publisher() + .await + } + pub fn join_response(&self) -> Option { if let Some(info) = self.inner.session_info.lock().as_ref() { Some(info.join_response.clone()) @@ -307,37 +351,33 @@ impl EngineInner { Ok(()) } - fn connect<'a>( - self: &'a Arc, - url: &'a str, - token: &'a str, + async fn connect( + self: &Arc, + url: &str, + token: &str, options: SignalOptions, - ) -> BoxFuture<'a, EngineResult<()>> { - async { - let (session_emitter, session_events) = mpsc::unbounded_channel(); - let session = RTCSession::connect( - url, - token, - options, - self.lk_runtime.clone(), - session_emitter, - ) - .await?; + ) -> EngineResult<()> { + let (session_emitter, session_events) = mpsc::unbounded_channel(); + let session = RtcSession::connect( + url, + token, + options, + self.lk_runtime.clone(), + session_emitter, + ) + .await?; - let (close_sender, close_receiver) = oneshot::channel(); - let engine_task = - tokio::spawn(self.clone().engine_task(session_events, close_receiver)); - *self.session_info.lock() = Some(session.info().clone()); - *self.running_handle.write().await = Some(EngineHandle { - session, - engine_task, - close_sender, - }); + let (close_sender, close_receiver) = oneshot::channel(); + let engine_task = tokio::spawn(self.clone().engine_task(session_events, close_receiver)); + *self.session_info.lock() = Some(session.info().clone()); + *self.running_handle.write().await = Some(EngineHandle { + session, + engine_task, + close_sender, + }); - self.opened.store(true, Ordering::SeqCst); - Ok(()) - } - .boxed() + self.opened.store(true, Ordering::SeqCst); + Ok(()) } async fn terminate_session(&self) { @@ -362,7 +402,7 @@ impl EngineInner { } while self.reconnecting.load(Ordering::Acquire) { - tokio::task::yield_now().await; + tokio::task::yield_now().await; // TODO(theomonnom): Remove yield } if self.running_handle.read().await.is_none() { diff --git a/livekit/src/rtc_engine/pc_transport.rs b/livekit/src/rtc_engine/peer_transport.rs similarity index 69% rename from livekit/src/rtc_engine/pc_transport.rs rename to livekit/src/rtc_engine/peer_transport.rs index 9185f2d..930d799 100644 --- a/livekit/src/rtc_engine/pc_transport.rs +++ b/livekit/src/rtc_engine/peer_transport.rs @@ -1,35 +1,31 @@ use crate::proto; use livekit_webrtc::prelude::*; use std::fmt::{Debug, Formatter}; -use std::future::Future; -use std::pin::Pin; use std::time::Duration; -use tracing::{event, Level}; +use tracing::{debug, event, Level}; const NEGOTIATION_FREQUENCY: Duration = Duration::from_millis(150); -pub type OnOfferHandler = Box< - dyn (FnMut(SessionDescription) -> Pin + Send + 'static>>) - + Send - + Sync, ->; +pub type OnOfferCreated = Box; -pub struct PCTransport { +pub struct PeerTransport { signal_target: proto::SignalTarget, peer_connection: PeerConnection, pending_candidates: Vec, - on_offer_handler: Option, + on_offer_handler: Option, renegotiate: bool, restarting_ice: bool, } -impl Debug for PCTransport { +impl Debug for PeerTransport { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - f.write_str("PCTransport") + f.debug_struct("PeerTransport") + .field("target", &self.signal_target) + .finish() } } -impl PCTransport { +impl PeerTransport { pub fn new(peer_connection: PeerConnection, signal_target: proto::SignalTarget) -> Self { Self { signal_target, @@ -42,9 +38,10 @@ impl PCTransport { } pub fn is_connected(&self) -> bool { - self.peer_connection.ice_connection_state() == IceConnectionState::IceConnectionConnected - || self.peer_connection.ice_connection_state() - == IceConnectionState::IceConnectionCompleted + matches!( + self.peer_connection.ice_connection_state(), + IceConnectionState::Connected | IceConnectionState::Completed + ) } pub fn peer_connection(&mut self) -> &mut PeerConnection { @@ -55,8 +52,8 @@ impl PCTransport { self.signal_target.clone() } - pub fn on_offer(&mut self, handler: OnOfferHandler) { - self.on_offer_handler = Some(handler); + pub fn on_offer(&mut self, handler: Option) { + self.on_offer_handler = handler; } pub fn prepare_ice_restart(&mut self) { @@ -68,8 +65,8 @@ impl PCTransport { } #[tracing::instrument(level = Level::DEBUG)] - pub async fn add_ice_candidate(&mut self, ice_candidate: IceCandidate) -> Result<(), RTCError> { - if self.peer_connection.remote_description().is_some() && !self.restarting_ice { + pub async fn add_ice_candidate(&mut self, ice_candidate: IceCandidate) -> Result<(), RtcError> { + if self.peer_connection.current_remote_description().is_some() && !self.restarting_ice { self.peer_connection .add_ice_candidate(ice_candidate) .await?; @@ -85,7 +82,7 @@ impl PCTransport { pub async fn set_remote_description( &mut self, remote_description: SessionDescription, - ) -> Result<(), RTCError> { + ) -> Result<(), RtcError> { self.peer_connection .set_remote_description(remote_description) .await?; @@ -97,31 +94,26 @@ impl PCTransport { if self.renegotiate { self.renegotiate = false; - self.create_and_send_offer(RTCOfferAnswerOptions::default()) - .await?; + self.create_and_send_offer(OfferOptions::default()).await?; } Ok(()) } #[tracing::instrument(level = Level::DEBUG)] - pub async fn negotiate(&mut self) -> Result<(), RTCError> { + pub async fn negotiate(&mut self) -> Result<(), RtcError> { // TODO(theomonnom) Debounce here with NEGOTIATION_FREQUENCY - self.create_and_send_offer(RTCOfferAnswerOptions::default()) - .await + self.create_and_send_offer(OfferOptions::default()).await } #[tracing::instrument(level = Level::DEBUG)] pub async fn create_anwser( &mut self, offer: SessionDescription, - options: RTCOfferAnswerOptions, - ) -> Result { + options: AnswerOptions, + ) -> Result { self.set_remote_description(offer).await?; - let answer = self - .peer_connection() - .create_answer(RTCOfferAnswerOptions::default()) - .await?; + let answer = self.peer_connection().create_answer(options).await?; self.peer_connection() .set_local_description(answer.clone()) .await?; @@ -130,10 +122,7 @@ impl PCTransport { } #[tracing::instrument(level = Level::DEBUG)] - pub async fn create_and_send_offer( - &mut self, - options: RTCOfferAnswerOptions, - ) -> Result<(), RTCError> { + pub async fn create_and_send_offer(&mut self, options: OfferOptions) -> Result<(), RtcError> { if self.on_offer_handler.is_none() { return Ok(()); } @@ -145,7 +134,8 @@ impl PCTransport { if self.peer_connection.signaling_state() == SignalingState::HaveLocalOffer { if options.ice_restart { - if let Some(remote_description) = self.peer_connection.remote_description() { + if let Some(remote_description) = self.peer_connection.current_remote_description() + { self.peer_connection .set_remote_description(remote_description) .await?; @@ -165,7 +155,7 @@ impl PCTransport { self.peer_connection .set_local_description(offer.clone()) .await?; - self.on_offer_handler.as_mut().unwrap()(offer).await; + self.on_offer_handler.as_mut().unwrap()(offer); Ok(()) } } diff --git a/livekit/src/rtc_engine/rtc_events.rs b/livekit/src/rtc_engine/rtc_events.rs index e3a8c56..e0630f7 100644 --- a/livekit/src/rtc_engine/rtc_events.rs +++ b/livekit/src/rtc_engine/rtc_events.rs @@ -1,20 +1,15 @@ -use super::pc_transport::PCTransport; +use super::peer_transport::PeerTransport; use crate::proto; -use crate::rtc_engine::pc_transport::OnOfferHandler; -use livekit_webrtc::data_channel::OnMessageHandler; -use livekit_webrtc::peer_connection::{ - OnAddTrackHandler, OnConnectionChangeHandler, OnDataChannelHandler, OnIceCandidateErrorHandler, - OnIceCandidateHandler, PeerConnectionState, -}; -use livekit_webrtc::prelude::*; +use crate::rtc_engine::peer_transport::OnOfferCreated; +use livekit_webrtc::{self as rtc, prelude::*}; use tokio::sync::mpsc; -use tracing::error; +use tracing::{debug, error}; -pub type RTCEmitter = mpsc::UnboundedSender; -pub type RTCEvents = mpsc::UnboundedReceiver; +pub type RtcEmitter = mpsc::UnboundedSender; +pub type RtcEvents = mpsc::UnboundedReceiver; #[derive(Debug)] -pub enum RTCEvent { +pub enum RtcEvent { IceCandidate { ice_candidate: IceCandidate, target: proto::SignalTarget, @@ -27,14 +22,16 @@ pub enum RTCEvent { data_channel: DataChannel, target: proto::SignalTarget, }, - // TODO (theomonnom): Move Offer to PCTransport + // TODO (theomonnom): Move Offer to PeerTransport Offer { offer: SessionDescription, target: proto::SignalTarget, }, - AddTrack { - rtp_receiver: RtpReceiver, + Track { + receiver: RtpReceiver, streams: Vec, + track: MediaStreamTrack, + transceiver: RtpTransceiver, target: proto::SignalTarget, }, Data { @@ -46,99 +43,108 @@ pub enum RTCEvent { /// Handlers used to forward events to a channel /// Every callback here is called on the signaling thread -fn on_connection_change( +fn on_connection_state_change( target: proto::SignalTarget, - emitter: RTCEmitter, -) -> OnConnectionChangeHandler { + emitter: RtcEmitter, +) -> rtc::peer_connection::OnConnectionChange { Box::new(move |state| { - let _ = emitter.send(RTCEvent::ConnectionChange { state, target }); + let _ = emitter.send(RtcEvent::ConnectionChange { state, target }); }) } -fn on_ice_candidate(target: proto::SignalTarget, emitter: RTCEmitter) -> OnIceCandidateHandler { +fn on_ice_candidate( + target: proto::SignalTarget, + emitter: RtcEmitter, +) -> rtc::peer_connection::OnIceCandidate { Box::new(move |ice_candidate| { - let _ = emitter.send(RTCEvent::IceCandidate { + let _ = emitter.send(RtcEvent::IceCandidate { ice_candidate, target, }); }) } -fn on_offer(target: proto::SignalTarget, emitter: RTCEmitter) -> OnOfferHandler { +fn on_offer(target: proto::SignalTarget, emitter: RtcEmitter) -> OnOfferCreated { Box::new(move |offer| { - let _ = emitter.send(RTCEvent::Offer { offer, target }); - - Box::pin(async {}) + let _ = emitter.send(RtcEvent::Offer { offer, target }); }) } -fn on_data_channel(target: proto::SignalTarget, emitter: RTCEmitter) -> OnDataChannelHandler { - Box::new(move |mut data_channel| { - data_channel.on_message(on_message(emitter.clone())); +fn on_data_channel( + target: proto::SignalTarget, + emitter: RtcEmitter, +) -> rtc::peer_connection::OnDataChannel { + Box::new(move |data_channel| { + data_channel.on_message(Some(on_message(emitter.clone()))); - let _ = emitter.send(RTCEvent::DataChannel { + let _ = emitter.send(RtcEvent::DataChannel { data_channel, target, }); }) } -fn on_add_track(target: proto::SignalTarget, emitter: RTCEmitter) -> OnAddTrackHandler { - Box::new(move |rtp_receiver, streams| { - let _ = emitter.send(RTCEvent::AddTrack { - rtp_receiver, - streams, +fn on_track(target: proto::SignalTarget, emitter: RtcEmitter) -> rtc::peer_connection::OnTrack { + Box::new(move |event| { + let _ = emitter.send(RtcEvent::Track { + receiver: event.receiver, + streams: event.streams, + track: event.track, + transceiver: event.transceiver, target, }); }) } fn on_ice_candidate_error( - target: proto::SignalTarget, - _emitter: RTCEmitter, -) -> OnIceCandidateErrorHandler { - Box::new(move |address, port, url, error_code, error_text| { - error!( - "ICE candidate error ({:?}): address: {} - port: {} - url: {} - error_code: {} - error_text: {}", - target, address, port, url, error_code, error_text - ); + _target: proto::SignalTarget, + _emitter: RtcEmitter, +) -> rtc::peer_connection::OnIceCandidateError { + Box::new(move |ice_error| { + error!("{:?}", ice_error); }) } -pub fn forward_pc_events(transport: &mut PCTransport, rtc_emitter: RTCEmitter) { +pub fn forward_pc_events(transport: &mut PeerTransport, rtc_emitter: RtcEmitter) { let signal_target = transport.signal_target(); transport .peer_connection() - .on_ice_candidate(on_ice_candidate(signal_target, rtc_emitter.clone())); + .on_ice_candidate(Some(on_ice_candidate(signal_target, rtc_emitter.clone()))); transport .peer_connection() - .on_data_channel(on_data_channel(signal_target, rtc_emitter.clone())); + .on_data_channel(Some(on_data_channel(signal_target, rtc_emitter.clone()))); transport .peer_connection() - .on_add_track(on_add_track(signal_target, rtc_emitter.clone())); + .on_track(Some(on_track(signal_target, rtc_emitter.clone()))); transport .peer_connection() - .on_connection_change(on_connection_change(signal_target, rtc_emitter.clone())); + .on_connection_state_change(Some(on_connection_state_change( + signal_target, + rtc_emitter.clone(), + ))); transport .peer_connection() - .on_ice_candidate_error(on_ice_candidate_error(signal_target, rtc_emitter.clone())); + .on_ice_candidate_error(Some(on_ice_candidate_error( + signal_target, + rtc_emitter.clone(), + ))); - transport.on_offer(on_offer(transport.signal_target(), rtc_emitter.clone())); + transport.on_offer(Some(on_offer(signal_target, rtc_emitter.clone()))); } -fn on_message(emitter: RTCEmitter) -> OnMessageHandler { - Box::new(move |data, binary| { - let _ = emitter.send(RTCEvent::Data { - data: data.to_vec(), - binary, +fn on_message(emitter: RtcEmitter) -> rtc::data_channel::OnMessage { + Box::new(move |buffer| { + let _ = emitter.send(RtcEvent::Data { + data: buffer.data.to_vec(), + binary: buffer.binary, }); }) } -pub fn forward_dc_events(dc: &mut DataChannel, rtc_emitter: RTCEmitter) { - dc.on_message(on_message(rtc_emitter.clone())); +pub fn forward_dc_events(dc: &mut DataChannel, rtc_emitter: RtcEmitter) { + dc.on_message(Some(on_message(rtc_emitter.clone()))); } diff --git a/livekit/src/rtc_engine/rtc_session.rs b/livekit/src/rtc_engine/rtc_session.rs index b64ea13..86e40fc 100644 --- a/livekit/src/rtc_engine/rtc_session.rs +++ b/livekit/src/rtc_engine/rtc_session.rs @@ -1,23 +1,28 @@ use super::{rtc_events, EngineError, EngineResult, SimulateScenario}; -use crate::rtc_engine::lk_runtime::LKRuntime; -use crate::rtc_engine::pc_transport::PCTransport; -use crate::rtc_engine::rtc_events::{RTCEvent, RTCEvents}; +use crate::options::TrackPublishOptions; +use crate::rtc_engine::lk_runtime::LkRuntime; +use crate::rtc_engine::peer_transport::PeerTransport; +use crate::rtc_engine::rtc_events::{RtcEvent, RtcEvents}; use crate::signal_client::{SignalClient, SignalEvent, SignalEvents, SignalOptions}; +use crate::track::LocalTrack; use crate::{proto, signal_client}; use livekit_webrtc::prelude::*; use parking_lot::Mutex; use prost::Message; use serde::{Deserialize, Serialize}; +use std::collections::HashMap; use std::convert::TryInto; use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; use std::sync::Arc; use std::time::Duration; -use tokio::sync::{mpsc, watch, Mutex as AsyncMutex}; +use tokio::sync::Mutex as AsyncMutex; +use tokio::sync::{mpsc, oneshot, watch}; use tokio::task::JoinHandle; use tokio::time::sleep; use tracing::{debug, error, trace, warn}; -pub const MAX_ICE_CONNECT_TIMEOUT: Duration = Duration::from_secs(15); +pub const ICE_CONNECT_TIMEOUT: Duration = Duration::from_secs(15); +pub const TRACK_PUBLISH_TIMEOUT: Duration = Duration::from_secs(10); pub const LOSSY_DC_LABEL: &str = "_lossy"; pub const RELIABLE_DC_LABEL: &str = "_reliable"; @@ -35,7 +40,7 @@ pub enum SessionEvent { kind: proto::data_packet::Kind, }, MediaTrack { - track: MediaStreamTrackHandle, + track: MediaStreamTrack, stream: MediaStream, receiver: RtpReceiver, }, @@ -56,8 +61,8 @@ pub enum SessionEvent { Connected, } -#[repr(u8)] -pub enum PCState { +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PeerState { New, Connected, Disconnected, @@ -65,27 +70,27 @@ pub enum PCState { Closed, } -impl TryInto for u8 { +impl TryFrom for PeerState { type Error = &'static str; - fn try_into(self) -> Result { - match self { - 0 => Ok(PCState::New), - 1 => Ok(PCState::Connected), - 2 => Ok(PCState::Disconnected), - 3 => Ok(PCState::Reconnecting), - 4 => Ok(PCState::Closed), - _ => Err("invalid PCState"), + fn try_from(v: u8) -> Result { + match v { + 0 => Ok(Self::New), + 1 => Ok(Self::Connected), + 2 => Ok(Self::Disconnected), + 3 => Ok(Self::Reconnecting), + 4 => Ok(Self::Closed), + _ => Err("invalid PeerState"), } } } #[derive(Serialize, Deserialize)] -#[allow(non_snake_case)] -struct IceCandidateJSON { - sdpMid: String, - sdpMLineIndex: i32, - candidate: String, +#[serde(rename_all = "camelCase")] +struct IceCandidateJson { + pub sdp_mid: String, + pub sdp_m_line_index: i32, + pub candidate: String, } #[derive(Debug, Clone, Default)] @@ -104,8 +109,10 @@ struct SessionInner { pc_state: AtomicU8, // PCState has_published: AtomicBool, - publisher_pc: AsyncMutex, - subscriber_pc: AsyncMutex, + publisher_pc: AsyncMutex, + subscriber_pc: AsyncMutex, + + pending_tracks: Mutex>>, // Publisher data channels // used to send data to other participants ( The SFU forwards the messages ) @@ -124,20 +131,21 @@ struct SessionInner { /// /// RTCSession is also responsable for the signaling and the negotation #[derive(Debug)] -pub struct RTCSession { - lk_runtime: Arc, +pub struct RtcSession { + #[allow(dead_code)] + lk_runtime: Arc, inner: Arc, close_tx: watch::Sender, // false = is_running signal_task: JoinHandle<()>, rtc_task: JoinHandle<()>, } -impl RTCSession { +impl RtcSession { pub async fn connect( url: &str, token: &str, options: SignalOptions, - lk_runtime: Arc, + lk_runtime: Arc, session_emitter: SessionEmitter, ) -> EngineResult { // Connect to the SignalClient @@ -148,16 +156,16 @@ impl RTCSession { debug!("received JoinResponse: {:?}", join_response); let (rtc_emitter, rtc_events) = mpsc::unbounded_channel(); - let rtc_config = RTCConfiguration::from(join_response.clone()); + let rtc_config = RtcConfiguration::from(join_response.clone()); - let mut publisher_pc = PCTransport::new( + let mut publisher_pc = PeerTransport::new( lk_runtime .pc_factory .create_peer_connection(rtc_config.clone())?, proto::SignalTarget::Publisher, ); - let mut subscriber_pc = PCTransport::new( + let mut subscriber_pc = PeerTransport::new( lk_runtime .pc_factory .create_peer_connection(rtc_config.clone())?, @@ -181,7 +189,7 @@ impl RTCSession { }, )?; - // Forward events received in the Signaling Thread to our rtc channel + // Forward events received inside the signaling thread to our rtc channel rtc_events::forward_pc_events(&mut publisher_pc, rtc_emitter.clone()); rtc_events::forward_pc_events(&mut subscriber_pc, rtc_emitter.clone()); rtc_events::forward_dc_events(&mut lossy_dc, rtc_emitter.clone()); @@ -197,11 +205,12 @@ impl RTCSession { let (close_tx, close_rx) = watch::channel(false); let inner = Arc::new(SessionInner { info: session_info, - pc_state: AtomicU8::new(PCState::New as u8), + pc_state: AtomicU8::new(PeerState::New as u8), has_published: Default::default(), signal_client, publisher_pc: AsyncMutex::new(publisher_pc), subscriber_pc: AsyncMutex::new(subscriber_pc), + pending_tracks: Default::default(), lossy_dc, reliable_dc, subscriber_dc: Default::default(), @@ -228,6 +237,31 @@ impl RTCSession { Ok(session) } + #[inline] + pub async fn add_track(&self, req: proto::AddTrackRequest) -> EngineResult { + self.inner.add_track(req).await + } + + #[inline] + pub async fn remove_track(&self, sender: RtpSender) -> EngineResult<()> { + self.inner.remove_track(sender).await + } + + #[inline] + pub async fn create_sender( + &self, + track: LocalTrack, + options: TrackPublishOptions, + encodings: Vec, + ) -> EngineResult { + self.inner.create_sender(track, options, encodings).await + } + + #[inline] + pub async fn negotiate_publisher(&self) -> EngineResult<()> { + self.inner.negotiate_publisher().await + } + /// Close the PeerConnections and the SignalClient #[tracing::instrument] pub async fn close(self) { @@ -238,6 +272,7 @@ impl RTCSession { let _ = self.signal_task.await; } + #[inline] pub async fn publish_data( &self, data: &proto::DataPacket, @@ -246,25 +281,28 @@ impl RTCSession { self.inner.publish_data(data, kind).await } + #[inline] pub async fn restart(&self) -> EngineResult<()> { self.inner.restart_session().await } + #[inline] pub async fn wait_pc_connection(&self) -> EngineResult<()> { self.inner.wait_pc_connection().await } + #[inline] pub async fn simulate_scenario(&self, scenario: SimulateScenario) { self.inner.simulate_scenario(scenario).await } -} -impl RTCSession { + #[inline] pub fn info(&self) -> &SessionInfo { &self.inner.info } - pub fn state(&self) -> PCState { + #[inline] + pub fn state(&self) -> PeerState { self.inner .pc_state .load(Ordering::SeqCst) @@ -272,18 +310,22 @@ impl RTCSession { .unwrap() } - pub fn publisher(&self) -> &AsyncMutex { + #[inline] + pub fn publisher(&self) -> &AsyncMutex { &self.inner.publisher_pc } - pub fn subscriber(&self) -> &AsyncMutex { + #[inline] + pub fn subscriber(&self) -> &AsyncMutex { &self.inner.subscriber_pc } + #[inline] pub fn signal_client(&self) -> &Arc { &self.inner.signal_client } + #[inline] pub fn data_channel(&self, kind: proto::data_packet::Kind) -> &DataChannel { &self.inner.data_channel(kind) } @@ -292,7 +334,7 @@ impl RTCSession { impl SessionInner { async fn rtc_task( self: Arc, - mut rtc_events: RTCEvents, + mut rtc_events: RtcEvents, mut close_receiver: watch::Receiver, ) { loop { @@ -355,7 +397,8 @@ impl SessionInner { match event { proto::signal_response::Message::Answer(answer) => { trace!("received publisher answer: {:?}", answer); - let answer = SessionDescription::from(answer.r#type.parse().unwrap(), &answer.sdp)?; + let answer = + SessionDescription::parse(&answer.sdp, answer.r#type.parse().unwrap())?; self.publisher_pc .lock() .await @@ -364,12 +407,12 @@ impl SessionInner { } proto::signal_response::Message::Offer(offer) => { trace!("received subscriber offer: {:?}", offer); - let offer = SessionDescription::from(offer.r#type.parse().unwrap(), &offer.sdp)?; + let offer = SessionDescription::parse(&offer.sdp, offer.r#type.parse().unwrap())?; let answer = self .subscriber_pc .lock() .await - .create_anwser(offer, RTCOfferAnswerOptions::default()) + .create_anwser(offer, AnswerOptions::default()) .await?; self.signal_client @@ -384,11 +427,11 @@ impl SessionInner { proto::signal_response::Message::Trickle(trickle) => { let target = proto::SignalTarget::from_i32(trickle.target).unwrap(); let ice_candidate = { - let json = serde_json::from_str::(&trickle.candidate_init)?; - IceCandidate::from(&json.sdpMid, json.sdpMLineIndex, &json.candidate)? + let json = serde_json::from_str::(&trickle.candidate_init)?; + IceCandidate::parse(&json.sdp_mid, json.sdp_m_line_index, &json.candidate)? }; - trace!("received ice_candidate {:?} {:?}", target, ice_candidate); + debug!("received ice_candidate {:?} {:?}", target, ice_candidate); if target == proto::SignalTarget::Publisher { self.publisher_pc @@ -428,24 +471,31 @@ impl SessionInner { updates: quality.updates, }); } + proto::signal_response::Message::TrackPublished(publish_res) => { + let mut pending_tracks = self.pending_tracks.lock(); + if let Some(tx) = pending_tracks.remove(&publish_res.cid) { + let _ = tx.send(publish_res.track.unwrap()); + } + } + _ => {} } Ok(()) } - async fn on_rtc_event(&self, event: RTCEvent) -> EngineResult<()> { + async fn on_rtc_event(&self, event: RtcEvent) -> EngineResult<()> { match event { - RTCEvent::IceCandidate { + RtcEvent::IceCandidate { ice_candidate, target, } => { self.signal_client .send(proto::signal_request::Message::Trickle( proto::TrickleRequest { - candidate_init: serde_json::to_string(&IceCandidateJSON { - sdpMid: ice_candidate.sdp_mid(), - sdpMLineIndex: ice_candidate.sdp_mline_index(), + candidate_init: serde_json::to_string(&IceCandidateJson { + sdp_mid: ice_candidate.sdp_mid(), + sdp_m_line_index: ice_candidate.sdp_mline_index(), candidate: ice_candidate.candidate(), })?, target: target as i32, @@ -453,21 +503,21 @@ impl SessionInner { )) .await; } - RTCEvent::ConnectionChange { state, target } => { - trace!("connection change, {:?} {:?}", state, target); + RtcEvent::ConnectionChange { state, target } => { + debug!("connection change, {:?} {:?}", state, target); let is_primary = self.info.join_response.subscriber_primary && target == proto::SignalTarget::Subscriber; if is_primary && state == PeerConnectionState::Connected { let old_state = self .pc_state - .swap(PCState::Connected as u8, Ordering::SeqCst); - if old_state == PCState::New as u8 { + .swap(PeerState::Connected as u8, Ordering::SeqCst); + if old_state == PeerState::New as u8 { let _ = self.emitter.send(SessionEvent::Connected); } } else if state == PeerConnectionState::Failed { self.pc_state - .store(PCState::Disconnected as u8, Ordering::SeqCst); + .store(PeerState::Disconnected as u8, Ordering::SeqCst); self.on_session_disconnected( "pc_state failed", @@ -478,14 +528,15 @@ impl SessionInner { ); } } - RTCEvent::DataChannel { + RtcEvent::DataChannel { data_channel, target: _, } => { self.subscriber_dc.lock().push(data_channel); } - RTCEvent::Offer { offer, target: _ } => { + RtcEvent::Offer { offer, target: _ } => { // Send the publisher offer to the server + debug!("sending publisher offer: {:?}", offer); self.signal_client .send(proto::signal_request::Message::Offer( proto::SessionDescription { @@ -495,22 +546,24 @@ impl SessionInner { )) .await; } - RTCEvent::AddTrack { - rtp_receiver, + RtcEvent::Track { + receiver, mut streams, + track, + transceiver: _, target: _, } => { if !streams.is_empty() { let _ = self.emitter.send(SessionEvent::MediaTrack { - track: rtp_receiver.track(), stream: streams.remove(0), - receiver: rtp_receiver, + track, + receiver, }); } else { - warn!("AddTrack event with no streams"); + warn!("Track event with no streams"); } } - RTCEvent::Data { data, binary } => { + RtcEvent::Data { data, binary } => { if !binary { Err(EngineError::Internal( "text messages aren't supported".to_string(), @@ -534,6 +587,106 @@ impl SessionInner { Ok(()) } + async fn add_track(&self, req: proto::AddTrackRequest) -> EngineResult { + let (tx, rx) = oneshot::channel(); + let cid = req.cid.clone(); + { + let mut pendings_tracks = self.pending_tracks.lock(); + if pendings_tracks.contains_key(&req.cid) { + Err(EngineError::Internal("track already published".to_string()))?; + } + + pendings_tracks.insert(cid.clone(), tx); + } + + self.signal_client + .send(proto::signal_request::Message::AddTrack(req)) + .await; + + // Wait the result from the server (TrackInfo) + tokio::select! { + Ok(info) = rx => Ok(info), + _ = sleep(TRACK_PUBLISH_TIMEOUT) => { + self.pending_tracks.lock().remove(&cid); + Err(EngineError::Internal("track publication timed out, no response received from the server".to_string())) + }, + else => { + Err(EngineError::Internal( + "track publication cancelled".to_string(), + )) + } + } + } + + async fn remove_track(&self, sender: RtpSender) -> EngineResult<()> { + if let Some(track) = sender.track() { + let mut pending_tracks = self.pending_tracks.lock(); + pending_tracks.remove(&track.id()); + } + + self.publisher_pc + .lock() + .await + .peer_connection() + .remove_track(sender)?; + + Ok(()) + } + + async fn create_sender( + &self, + track: LocalTrack, + options: TrackPublishOptions, + encodings: Vec, + ) -> EngineResult { + let init = RtpTransceiverInit { + direction: RtpTransceiverDirection::SendOnly, + stream_ids: Default::default(), + send_encodings: encodings, + }; + + let transceiver = self + .publisher_pc + .lock() + .await + .peer_connection() + .add_transceiver(track.rtc_track(), init)?; + + let capabilities = LkRuntime::instance() + .pc_factory + .get_rtp_sender_capabilities(track.kind().into()); + + let mut matched = Vec::new(); + let mut partial_matched = Vec::new(); + let mut unmatched = Vec::new(); + + for codec in capabilities.codecs { + let mime_type = codec.mime_type.to_lowercase(); + if mime_type == "audio/opus" { + matched.push(codec); + } else if mime_type == format!("video/{}", options.video_codec.as_str()) { + if let Some(sdp_fmtp_line) = codec.sdp_fmtp_line.as_ref() { + // for h264 codecs that have sdpFmtpLine available, use only if the + // profile-level-id is 42e01f for cross-browser compatibility + if sdp_fmtp_line.contains("profile-level-id=42e01f") { + matched.push(codec); + continue; + } + } + partial_matched.push(codec); + } else { + unmatched.push(codec); + } + } + + matched.append(&mut partial_matched); + matched.append(&mut unmatched); + + transceiver.set_codec_preferences(matched)?; + + Ok(transceiver) + } + /// Called when the SignalClient or one of the PeerConnection has lost the connection /// The RTCEngine may try a reconnect. fn on_session_disconnected( @@ -663,7 +816,7 @@ impl SessionInner { self.publisher_pc .lock() .await - .create_and_send_offer(RTCOfferAnswerOptions { + .create_and_send_offer(OfferOptions { ice_restart: true, ..Default::default() }) @@ -680,7 +833,7 @@ impl SessionInner { // Timeout after ['MAX_ICE_CONNECT_TIMEOUT'] async fn wait_pc_connection(&self) -> EngineResult<()> { let wait_connected = async move { - while self.pc_state.load(Ordering::Acquire) != PCState::Connected as u8 { + while self.pc_state.load(Ordering::Acquire) != PeerState::Connected as u8 { if self.closed.load(Ordering::Acquire) { return Err(EngineError::Connection("closed".to_string())); } @@ -693,7 +846,7 @@ impl SessionInner { tokio::select! { res = wait_connected => res, - _ = sleep(MAX_ICE_CONNECT_TIMEOUT) => { + _ = sleep(ICE_CONNECT_TIMEOUT) => { let err = EngineError::Connection("wait_pc_connection timed out".to_string()); Err(err) } @@ -724,7 +877,7 @@ impl SessionInner { .await .peer_connection() .ice_connection_state() - != IceConnectionState::IceConnectionChecking + != IceConnectionState::Checking { let _ = self.negotiate_publisher().await; } @@ -749,7 +902,7 @@ impl SessionInner { tokio::select! { res = wait_connected => res, - _ = sleep(MAX_ICE_CONNECT_TIMEOUT) => { + _ = sleep(ICE_CONNECT_TIMEOUT) => { let err = EngineError::Connection("could not establish publisher connection: timeout".to_string()); error!(error = ?err); Err(err) diff --git a/livekit/src/signal_client/mod.rs b/livekit/src/signal_client/mod.rs index e8776f8..6e03b02 100644 --- a/livekit/src/signal_client/mod.rs +++ b/livekit/src/signal_client/mod.rs @@ -114,13 +114,13 @@ impl SignalClient { } } -impl From for RTCConfiguration { +impl From for RtcConfiguration { fn from(join_response: proto::JoinResponse) -> Self { Self { ice_servers: { let mut servers = vec![]; for ice_server in join_response.ice_servers.clone() { - servers.push(ICEServer { + servers.push(IceServer { urls: ice_server.urls, username: ice_server.username, password: ice_server.credential, diff --git a/livekit/src/signal_client/signal_stream.rs b/livekit/src/signal_client/signal_stream.rs index 76d6681..ccfef98 100644 --- a/livekit/src/signal_client/signal_stream.rs +++ b/livekit/src/signal_client/signal_stream.rs @@ -34,7 +34,7 @@ enum InternalMessage { /// /// It is replaced by [SignalClient] at each reconnection. #[derive(Debug)] -pub(crate) struct SignalStream { +pub(super) struct SignalStream { internal_tx: mpsc::Sender, read_handle: JoinHandle<()>, write_handle: JoinHandle<()>, @@ -46,7 +46,7 @@ impl SignalStream { /// /// SignalStream will never try to reconnect if the connection has been /// closed. - pub(super) async fn connect( + pub async fn connect( url: &str, token: &str, options: SignalOptions, diff --git a/webrtc-sys/build.rs b/webrtc-sys/build.rs index c4eab58..9291961 100644 --- a/webrtc-sys/build.rs +++ b/webrtc-sys/build.rs @@ -7,7 +7,7 @@ use std::path; use std::process::Command; use tar::Archive; -const WEBRTC_TAG: &str = "m104.5112.08"; +const WEBRTC_TAG: &str = "m104.5112.09"; fn download_prebuilt( target_os: &str, diff --git a/webrtc-sys/include/livekit/data_channel.h b/webrtc-sys/include/livekit/data_channel.h index 7ba33db..5c48e43 100644 --- a/webrtc-sys/include/livekit/data_channel.h +++ b/webrtc-sys/include/livekit/data_channel.h @@ -37,7 +37,7 @@ class DataChannel { std::shared_ptr rtc_runtime, rtc::scoped_refptr data_channel); - void register_observer(NativeDataChannelObserver& observer) const; + void register_observer(NativeDataChannelObserver* observer) const; void unregister_observer() const; bool send(const DataBuffer& buffer) const; rust::String label() const; @@ -52,14 +52,17 @@ class DataChannel { std::unique_ptr create_data_channel_init( DataChannelInit init); -static std::unique_ptr _unique_data_channel() { +static std::shared_ptr _shared_data_channel() { return nullptr; // Ignore } class NativeDataChannelObserver : public webrtc::DataChannelObserver { public: explicit NativeDataChannelObserver( - rust::Box observer); + rust::Box observer, + DataChannel* dc); + + ~NativeDataChannelObserver(); void OnStateChange() override; void OnMessage(const webrtc::DataBuffer& buffer) override; @@ -67,8 +70,10 @@ class NativeDataChannelObserver : public webrtc::DataChannelObserver { private: rust::Box observer_; + DataChannel* dc_; }; -std::unique_ptr create_native_data_channel_observer( - rust::Box observer); +std::shared_ptr create_native_data_channel_observer( + rust::Box observer, + DataChannel* dc); } // namespace livekit diff --git a/webrtc-sys/include/livekit/jsep.h b/webrtc-sys/include/livekit/jsep.h index 22a48f7..f720753 100644 --- a/webrtc-sys/include/livekit/jsep.h +++ b/webrtc-sys/include/livekit/jsep.h @@ -65,6 +65,7 @@ class SessionDescription { explicit SessionDescription( std::unique_ptr session_description); + SdpType sdp_type() const; rust::String stringify() const; std::unique_ptr clone() const; std::unique_ptr release(); diff --git a/webrtc-sys/include/livekit/media_stream.h b/webrtc-sys/include/livekit/media_stream.h index 9d96e1d..8960791 100644 --- a/webrtc-sys/include/livekit/media_stream.h +++ b/webrtc-sys/include/livekit/media_stream.h @@ -26,6 +26,7 @@ #include "rtc_base/synchronization/mutex.h" #include "rtc_base/timestamp_aligner.h" #include "rust/cxx.h" +#include "system_wrappers/include/clock.h" namespace livekit { class MediaStream; @@ -50,10 +51,8 @@ class MediaStream { std::shared_ptr find_audio_track(rust::String track_id) const; std::shared_ptr find_video_track(rust::String track_id) const; - bool add_audio_track(std::shared_ptr audio_track) const; - bool add_video_track(std::shared_ptr video_track) const; - bool remove_audio_track(std::shared_ptr audio_track) const; - bool remove_video_track(std::shared_ptr video_track) const; + bool add_track(std::shared_ptr track) const; + bool remove_track(std::shared_ptr track) const; private: rtc::scoped_refptr media_stream_; @@ -121,7 +120,7 @@ class NativeVideoFrameSink rust::Box observer_; }; -std::unique_ptr create_native_video_frame_sink( +std::unique_ptr new_native_video_frame_sink( rust::Box observer); // Native impl of the WebRTC interface @@ -146,7 +145,7 @@ class AdaptedVideoTrackSource { public: AdaptedVideoTrackSource(rtc::scoped_refptr source); - bool on_captured_frame(std::unique_ptr frame) + bool on_captured_frame(const std::unique_ptr& frame) const; // frames pushed from Rust (+interior mutability) rtc::scoped_refptr get() const; @@ -155,14 +154,26 @@ class AdaptedVideoTrackSource { rtc::scoped_refptr source_; }; -std::unique_ptr create_adapted_video_track_source(); +std::shared_ptr new_adapted_video_track_source(); -static const VideoTrack* media_to_video(const MediaStreamTrack* track) { - return static_cast(track); +static std::shared_ptr video_to_media( + std::shared_ptr track) { + return track; } -static const AudioTrack* media_to_audio(const MediaStreamTrack* track) { - return static_cast(track); +static std::shared_ptr audio_to_media( + std::shared_ptr track) { + return track; +} + +static std::shared_ptr media_to_video( + std::shared_ptr track) { + return std::static_pointer_cast(track); +} + +static std::shared_ptr media_to_audio( + std::shared_ptr track) { + return std::static_pointer_cast(track); } static std::shared_ptr _shared_media_stream_track() { diff --git a/webrtc-sys/include/livekit/peer_connection.h b/webrtc-sys/include/livekit/peer_connection.h index e288600..8d2801a 100644 --- a/webrtc-sys/include/livekit/peer_connection.h +++ b/webrtc-sys/include/livekit/peer_connection.h @@ -57,7 +57,7 @@ class PeerConnection { void set_remote_description(std::unique_ptr desc, NativeSetRemoteSdpObserverHandle& observer) const; - std::unique_ptr create_data_channel( + std::shared_ptr create_data_channel( rust::String label, std::unique_ptr init) const; @@ -84,24 +84,34 @@ class PeerConnection { rust::Vec get_transceivers() const; + std::unique_ptr current_local_description() const; + + std::unique_ptr current_remote_description() const; + + std::unique_ptr pending_local_description() const; + + std::unique_ptr pending_remote_description() const; + std::unique_ptr local_description() const; std::unique_ptr remote_description() const; + PeerConnectionState connection_state() const; + SignalingState signaling_state() const; IceGatheringState ice_gathering_state() const; IceConnectionState ice_connection_state() const; - void close(); + void close() const; private: std::shared_ptr rtc_runtime_; rtc::scoped_refptr peer_connection_; }; -static std::unique_ptr _unique_peer_connection() { +static std::shared_ptr _shared_peer_connection() { return nullptr; // Ignore } @@ -126,6 +136,8 @@ class NativePeerConnectionObserver : public webrtc::PeerConnectionObserver { std::shared_ptr rtc_runtime, rust::Box observer); + ~NativePeerConnectionObserver(); + void OnSignalingChange( webrtc::PeerConnectionInterface::SignalingState new_state) override; @@ -188,7 +200,7 @@ class NativePeerConnectionObserver : public webrtc::PeerConnectionObserver { rust::Box observer_; }; -std::unique_ptr +std::shared_ptr create_native_peer_connection_observer( std::shared_ptr rtc_runtime, rust::Box observer); diff --git a/webrtc-sys/include/livekit/peer_connection_factory.h b/webrtc-sys/include/livekit/peer_connection_factory.h index d4a3077..5e7fcba 100644 --- a/webrtc-sys/include/livekit/peer_connection_factory.h +++ b/webrtc-sys/include/livekit/peer_connection_factory.h @@ -17,7 +17,10 @@ #pragma once #include "api/peer_connection_interface.h" +#include "media_stream.h" #include "peer_connection.h" +#include "rtp_parameters.h" +#include "rust/cxx.h" #include "webrtc.h" namespace livekit { @@ -35,16 +38,24 @@ class PeerConnectionFactory { explicit PeerConnectionFactory(std::shared_ptr rtc_runtime); ~PeerConnectionFactory(); - std::unique_ptr create_peer_connection( + std::shared_ptr create_peer_connection( std::unique_ptr config, - NativePeerConnectionObserver& observer) const; + NativePeerConnectionObserver* observer) const; + + std::shared_ptr create_video_track( + rust::String label, + std::shared_ptr source) const; + + RtpCapabilities get_rtp_sender_capabilities(MediaType type) const; + + RtpCapabilities get_rtp_receiver_capabilities(MediaType type) const; private: std::shared_ptr rtc_runtime_; rtc::scoped_refptr peer_factory_; }; -std::unique_ptr create_peer_connection_factory( +std::shared_ptr create_peer_connection_factory( std::shared_ptr rtc_runtime); std::unique_ptr create_rtc_configuration( RTCConfiguration conf); diff --git a/webrtc-sys/include/livekit/video_frame.h b/webrtc-sys/include/livekit/video_frame.h index 712a39a..1acfb5b 100644 --- a/webrtc-sys/include/livekit/video_frame.h +++ b/webrtc-sys/include/livekit/video_frame.h @@ -57,7 +57,7 @@ class VideoFrameBuilder { VideoFrameBuilder() = default; // TODO(theomonnom): other setters? - void set_video_frame_buffer(std::unique_ptr buffer); + void set_video_frame_buffer(const VideoFrameBuffer& buffer); void set_timestamp_us(int64_t timestamp_us); void set_rotation(VideoRotation rotation); void set_id(uint16_t id); @@ -67,6 +67,6 @@ class VideoFrameBuilder { webrtc::VideoFrame::Builder builder_; }; -std::unique_ptr create_video_frame_builder(); +std::unique_ptr new_video_frame_builder(); } // namespace livekit diff --git a/webrtc-sys/include/livekit/video_frame_buffer.h b/webrtc-sys/include/livekit/video_frame_buffer.h index 9008e43..ca246d1 100644 --- a/webrtc-sys/include/livekit/video_frame_buffer.h +++ b/webrtc-sys/include/livekit/video_frame_buffer.h @@ -49,8 +49,9 @@ class VideoFrameBuffer { int width() const; int height() const; - // Takes ownership - std::unique_ptr to_i420(); + std::unique_ptr to_i420() const; + + // Requires ownership std::unique_ptr get_i420(); std::unique_ptr get_i420a(); std::unique_ptr get_i422(); @@ -131,7 +132,9 @@ class BiplanarYuv8Buffer : public BiplanarYuvBuffer { webrtc::BiplanarYuv8Buffer* buffer() const; }; -std::unique_ptr create_i420_buffer(int width, int height); +std::unique_ptr new_i420_buffer(int width, int height); +std::unique_ptr copy_i420_buffer( + const std::unique_ptr& i420); class I420Buffer : public PlanarYuv8Buffer { public: @@ -141,6 +144,12 @@ class I420Buffer : public PlanarYuv8Buffer { class I420ABuffer : public I420Buffer { public: explicit I420ABuffer(rtc::scoped_refptr buffer); + + int stride_a() const; + const uint8_t* data_a() const; + + private: + webrtc::I420ABufferInterface* buffer() const; }; class I422Buffer : public PlanarYuv8Buffer { diff --git a/webrtc-sys/include/livekit/webrtc.h b/webrtc-sys/include/livekit/webrtc.h index d9c43ad..378d8f5 100644 --- a/webrtc-sys/include/livekit/webrtc.h +++ b/webrtc-sys/include/livekit/webrtc.h @@ -18,6 +18,7 @@ #include "rtc_base/physical_socket_server.h" #include "rtc_base/ssl_adapter.h" +#include "rust/cxx.h" #ifdef WEBRTC_WIN #include "rtc_base/win32_socket_init.h" @@ -54,6 +55,8 @@ class RTCRuntime { #endif }; +rust::String create_random_uuid(); + std::shared_ptr create_rtc_runtime(); } // namespace livekit diff --git a/webrtc-sys/include/livekit/yuv_helper.h b/webrtc-sys/include/livekit/yuv_helper.h index 5a6ae56..173c865 100644 --- a/webrtc-sys/include/livekit/yuv_helper.h +++ b/webrtc-sys/include/livekit/yuv_helper.h @@ -17,12 +17,19 @@ #pragma once #include +#include +#include #include "api/video/yuv_helper.h" #include "webrtc-sys/src/yuv_helper.rs.h" namespace livekit { +#define THROW_ON_ERROR(ret) \ + if (ret != 0) { \ + throw std::runtime_error("libyuv error: " + std::to_string(ret)); \ + } + static void i420_to_argb(const uint8_t* src_y, int src_stride_y, const uint8_t* src_u, @@ -33,8 +40,9 @@ static void i420_to_argb(const uint8_t* src_y, int dst_stride_argb, int width, int height) { - webrtc::I420ToARGB(src_y, src_stride_y, src_u, src_stride_u, src_v, - src_stride_v, dst_argb, dst_stride_argb, width, height); + THROW_ON_ERROR(webrtc::I420ToARGB(src_y, src_stride_y, src_u, src_stride_u, + src_v, src_stride_v, dst_argb, + dst_stride_argb, width, height)); } static void i420_to_bgra(const uint8_t* src_y, @@ -47,8 +55,9 @@ static void i420_to_bgra(const uint8_t* src_y, int dst_stride_bgra, int width, int height) { - webrtc::I420ToBGRA(src_y, src_stride_y, src_u, src_stride_u, src_v, - src_stride_v, dst_bgra, dst_stride_bgra, width, height); + THROW_ON_ERROR(webrtc::I420ToBGRA(src_y, src_stride_y, src_u, src_stride_u, + src_v, src_stride_v, dst_bgra, + dst_stride_bgra, width, height)); } static void i420_to_abgr(const uint8_t* src_y, @@ -61,8 +70,9 @@ static void i420_to_abgr(const uint8_t* src_y, int dst_stride_abgr, int width, int height) { - webrtc::I420ToABGR(src_y, src_stride_y, src_u, src_stride_u, src_v, - src_stride_v, dst_abgr, dst_stride_abgr, width, height); + THROW_ON_ERROR(webrtc::I420ToABGR(src_y, src_stride_y, src_u, src_stride_u, + src_v, src_stride_v, dst_abgr, + dst_stride_abgr, width, height)); } static void i420_to_rgba(const uint8_t* src_y, @@ -75,8 +85,49 @@ static void i420_to_rgba(const uint8_t* src_y, int dst_stride_rgba, int width, int height) { - webrtc::I420ToRGBA(src_y, src_stride_y, src_u, src_stride_u, src_v, - src_stride_v, dst_rgba, dst_stride_rgba, width, height); + THROW_ON_ERROR(webrtc::I420ToRGBA(src_y, src_stride_y, src_u, src_stride_u, + src_v, src_stride_v, dst_rgba, + dst_stride_rgba, width, height)); +} + +static void argb_to_i420(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_u, + int dst_stride_u, + uint8_t* dst_v, + int dst_stride_v, + int width, + int height) { + THROW_ON_ERROR(webrtc::ARGBToI420(src_argb, src_stride_argb, dst_y, + dst_stride_y, dst_u, dst_stride_u, dst_v, + dst_stride_v, width, height)); +} + +static void abgr_to_i420(const uint8_t* src_abgr, + int src_stride_abgr, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_u, + int dst_stride_u, + uint8_t* dst_v, + int dst_stride_v, + int width, + int height) { + THROW_ON_ERROR(webrtc::ABGRToI420(src_abgr, src_stride_abgr, dst_y, + dst_stride_y, dst_u, dst_stride_u, dst_v, + dst_stride_v, width, height)); +} + +static void argb_to_rgb24(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_rgb24, + int dst_stride_rgb24, + int width, + int height) { + THROW_ON_ERROR(webrtc::ARGBToRGB24(src_argb, src_stride_argb, dst_rgb24, + dst_stride_rgb24, width, height)); } } // namespace livekit diff --git a/webrtc-sys/libwebrtc/configure.py b/webrtc-sys/libwebrtc/configure.py index fe1d194..c335ff9 100644 --- a/webrtc-sys/libwebrtc/configure.py +++ b/webrtc-sys/libwebrtc/configure.py @@ -1,6 +1,9 @@ +# Ignore this file if you have no intention of using a custom libwebrtc build. + import platform import subprocess +# Edit the target_os/target_cpu to match your platform. GN_ARGS = [ "is_debug=true", "treat_warnings_as_errors=false", @@ -18,3 +21,6 @@ GN_ARGS = [ cmd = ["gn", "gen", "out/Dev", "--args=" + ' '.join(GN_ARGS)] print("Executing:", cmd) subprocess.call(cmd, shell=platform.system() == "Windows") + +# Use this command when developing on libwebrtc: +# ninja -C out/Dev sdk:default_codec_factory_objc api api/task_queue:default_task_queue_factory api/audio_codecs:builtin_audio_decoder_factory sdk:videocapture_objc pc:peerconnection sdk:native_api callback_logger_objc default diff --git a/webrtc-sys/src/data_channel.cpp b/webrtc-sys/src/data_channel.cpp index 859cdd9..61aa30c 100644 --- a/webrtc-sys/src/data_channel.cpp +++ b/webrtc-sys/src/data_channel.cpp @@ -18,6 +18,8 @@ #include +#include "webrtc-sys/src/data_channel.rs.h" + namespace livekit { DataChannel::DataChannel( @@ -26,8 +28,8 @@ DataChannel::DataChannel( : rtc_runtime_(std::move(rtc_runtime)), data_channel_(std::move(data_channel)) {} -void DataChannel::register_observer(NativeDataChannelObserver& observer) const { - data_channel_->RegisterObserver(&observer); +void DataChannel::register_observer(NativeDataChannelObserver* observer) const { + data_channel_->RegisterObserver(observer); } void DataChannel::unregister_observer() const { @@ -58,7 +60,6 @@ std::unique_ptr create_data_channel_init( rtc_init->negotiated = init.negotiated; rtc_init->ordered = init.ordered; rtc_init->protocol = init.protocol.c_str(); - rtc_init->reliable = init.reliable; if (init.has_max_retransmit_time) rtc_init->maxRetransmitTime = init.max_retransmit_time; @@ -73,11 +74,16 @@ std::unique_ptr create_data_channel_init( } NativeDataChannelObserver::NativeDataChannelObserver( - rust::Box observer) - : observer_(std::move(observer)) {} + rust::Box observer, + DataChannel* dc) + : observer_(std::move(observer)), dc_(dc) {} + +NativeDataChannelObserver::~NativeDataChannelObserver() { + dc_->unregister_observer(); +} void NativeDataChannelObserver::OnStateChange() { - observer_->on_state_change(); + observer_->on_state_change(dc_->state()); } void NativeDataChannelObserver::OnMessage(const webrtc::DataBuffer& buffer) { @@ -93,8 +99,9 @@ void NativeDataChannelObserver::OnBufferedAmountChange( observer_->on_buffered_amount_change(sent_data_size); } -std::unique_ptr create_native_data_channel_observer( - rust::Box observer) { - return std::make_unique(std::move(observer)); +std::shared_ptr create_native_data_channel_observer( + rust::Box observer, + DataChannel* dc) { + return std::make_shared(std::move(observer), dc); } } // namespace livekit diff --git a/webrtc-sys/src/data_channel.rs b/webrtc-sys/src/data_channel.rs index b27e0a8..a5a6cca 100644 --- a/webrtc-sys/src/data_channel.rs +++ b/webrtc-sys/src/data_channel.rs @@ -1,5 +1,6 @@ use crate::impl_thread_safety; use std::slice; +use std::sync::Arc; #[cxx::bridge(namespace = "livekit")] pub mod ffi { @@ -14,9 +15,6 @@ pub mod ffi { #[derive(Debug)] pub struct DataChannelInit { - #[allow(deprecated)] - #[deprecated] - pub reliable: bool, pub ordered: bool, pub has_max_retransmit_time: bool, pub max_retransmit_time: i32, @@ -48,7 +46,7 @@ pub mod ffi { extern "Rust" { type DataChannelObserverWrapper; - fn on_state_change(self: &DataChannelObserverWrapper); + fn on_state_change(self: &DataChannelObserverWrapper, state: DataState); fn on_message(self: &DataChannelObserverWrapper, buffer: DataBuffer); fn on_buffered_amount_change(self: &DataChannelObserverWrapper, sent_data_size: u64); } @@ -61,11 +59,8 @@ pub mod ffi { type NativeDataChannelObserver; /// SAFETY - /// The observer must live as the datachannel uses it - unsafe fn register_observer( - self: &DataChannel, - observer: Pin<&mut NativeDataChannelObserver>, - ); + /// The observer must live as long as the datachannel uses it + unsafe fn register_observer(self: &DataChannel, observer: *mut NativeDataChannelObserver); fn unregister_observer(self: &DataChannel); fn send(self: &DataChannel, data: &DataBuffer) -> bool; @@ -74,11 +69,12 @@ pub mod ffi { fn close(self: &DataChannel); fn create_data_channel_init(init: DataChannelInit) -> UniquePtr; - fn create_native_data_channel_observer( + unsafe fn create_native_data_channel_observer( observer: Box, - ) -> UniquePtr; + dc: *mut DataChannel, + ) -> SharedPtr; - fn _unique_data_channel() -> UniquePtr; // Ignore + fn _shared_data_channel() -> SharedPtr; // Ignore } } @@ -87,37 +83,33 @@ impl_thread_safety!(ffi::NativeDataChannelObserver, Send + Sync); // DataChannelObserver -pub trait DataChannelObserver: Send { - fn on_state_change(&self); +pub trait DataChannelObserver: Send + Sync { + fn on_state_change(&self, state: ffi::DataState); fn on_message(&self, data: &[u8], is_binary: bool); fn on_buffered_amount_change(&self, sent_data_size: u64); } pub struct DataChannelObserverWrapper { - observer: *mut dyn DataChannelObserver, + observer: Arc, } impl DataChannelObserverWrapper { - /// SAFETY - /// DataChannelObserver must lives as long as DataChannelObserverWrapper does - pub unsafe fn new(observer: *mut dyn DataChannelObserver) -> Self { + pub fn new(observer: Arc) -> Self { Self { observer } } - fn on_state_change(&self) { - unsafe { - (*self.observer).on_state_change(); - } + fn on_state_change(&self, state: ffi::DataState) { + self.observer.on_state_change(state); } fn on_message(&self, buffer: ffi::DataBuffer) { unsafe { let data = slice::from_raw_parts(buffer.ptr, buffer.len); - (*self.observer).on_message(data, buffer.binary); + self.observer.on_message(data, buffer.binary); } } fn on_buffered_amount_change(&self, sent_data_size: u64) { - unsafe { (*self.observer).on_buffered_amount_change(sent_data_size) }; + self.observer.on_buffered_amount_change(sent_data_size); } } diff --git a/webrtc-sys/src/jsep.cpp b/webrtc-sys/src/jsep.cpp index 7032eae..aea2103 100644 --- a/webrtc-sys/src/jsep.cpp +++ b/webrtc-sys/src/jsep.cpp @@ -77,6 +77,10 @@ SessionDescription::SessionDescription( std::unique_ptr session_description) : session_description_(std::move(session_description)) {} +SdpType SessionDescription::sdp_type() const { + return static_cast(session_description_->GetType()); +} + rust::String SessionDescription::stringify() const { std::string str; session_description_->ToString(&str); diff --git a/webrtc-sys/src/jsep.rs b/webrtc-sys/src/jsep.rs index 10a0b46..a6aed82 100644 --- a/webrtc-sys/src/jsep.rs +++ b/webrtc-sys/src/jsep.rs @@ -3,7 +3,6 @@ use cxx::UniquePtr; use std::error::Error; use std::fmt::{Display, Formatter}; use std::mem::ManuallyDrop; -use std::str::FromStr; use crate::rtc_error::ffi::RTCError; @@ -59,6 +58,7 @@ pub mod ffi { fn candidate(self: &IceCandidate) -> String; fn stringify(self: &IceCandidate) -> String; + fn sdp_type(self: &SessionDescription) -> SdpType; fn stringify(self: &SessionDescription) -> String; fn clone(self: &SessionDescription) -> UniquePtr; @@ -115,20 +115,6 @@ impl ffi::SdpParseError { } } -impl FromStr for ffi::SdpType { - type Err = (); - - fn from_str(s: &str) -> Result { - match s { - "offer" => Ok(ffi::SdpType::Offer), - "pranswer" => Ok(ffi::SdpType::PrAnswer), - "answer" => Ok(ffi::SdpType::Answer), - "rollback" => Ok(ffi::SdpType::Rollback), - _ => Err(()), - } - } -} - // CreateSdpObserver pub struct CreateSdpObserverWrapper { diff --git a/webrtc-sys/src/media_stream.cpp b/webrtc-sys/src/media_stream.cpp index d5370c9..a0839a0 100644 --- a/webrtc-sys/src/media_stream.cpp +++ b/webrtc-sys/src/media_stream.cpp @@ -17,11 +17,15 @@ #include "livekit/media_stream.h" #include +#include #include #include "api/media_stream_interface.h" #include "api/video/video_frame.h" +#include "api/video/video_rotation.h" +#include "rtc_base/logging.h" #include "rtc_base/ref_counted_object.h" +#include "rtc_base/time_utils.h" namespace livekit { @@ -61,32 +65,28 @@ std::shared_ptr MediaStream::find_video_track( media_stream_->FindVideoTrack(track_id.c_str())); } -bool MediaStream::add_audio_track( - std::shared_ptr audio_track) const { - return media_stream_->AddTrack( - rtc::scoped_refptr( - static_cast(audio_track->get().get()))); +bool MediaStream::add_track(std::shared_ptr track) const { + if (track->kind() == webrtc::MediaStreamTrackInterface::kVideoKind) { + return media_stream_->AddTrack( + rtc::scoped_refptr( + static_cast(track->get().get()))); + } else { + return media_stream_->AddTrack( + rtc::scoped_refptr( + static_cast(track->get().get()))); + } } -bool MediaStream::add_video_track( - std::shared_ptr video_track) const { - return media_stream_->AddTrack( - rtc::scoped_refptr( - static_cast(video_track->get().get()))); -} - -bool MediaStream::remove_audio_track( - std::shared_ptr audio_track) const { - return media_stream_->RemoveTrack( - rtc::scoped_refptr( - static_cast(audio_track->get().get()))); -} - -bool MediaStream::remove_video_track( - std::shared_ptr video_track) const { - return media_stream_->RemoveTrack( - rtc::scoped_refptr( - static_cast(video_track->get().get()))); +bool MediaStream::remove_track(std::shared_ptr track) const { + if (track->kind() == webrtc::MediaStreamTrackInterface::kVideoKind) { + return media_stream_->RemoveTrack( + rtc::scoped_refptr( + static_cast(track->get().get()))); + } else { + return media_stream_->RemoveTrack( + rtc::scoped_refptr( + static_cast(track->get().get()))); + } } MediaStreamTrack::MediaStreamTrack( @@ -177,13 +177,13 @@ void NativeVideoFrameSink::OnConstraintsChanged( observer_->on_constraints_changed(cst); } -std::unique_ptr create_native_video_frame_sink( +std::unique_ptr new_native_video_frame_sink( rust::Box observer) { return std::make_unique(std::move(observer)); } NativeVideoTrackSource::NativeVideoTrackSource() - : rtc::AdaptedVideoTrackSource(1) {} + : rtc::AdaptedVideoTrackSource(4) {} NativeVideoTrackSource::~NativeVideoTrackSource() {} @@ -197,7 +197,6 @@ absl::optional NativeVideoTrackSource::needs_denoising() const { webrtc::MediaSourceInterface::SourceState NativeVideoTrackSource::state() const { - // TODO(theomonnom): expose source state to Rust return SourceState::kLive; } @@ -212,22 +211,23 @@ bool NativeVideoTrackSource::on_captured_frame( int64_t aligned_timestamp_us = timestamp_aligner_.TranslateTimestamp( frame.timestamp_us(), rtc::TimeMicros()); + rtc::scoped_refptr buffer = + frame.video_frame_buffer(); + int adapted_width, adapted_height, crop_width, crop_height, crop_x, crop_y; - if (!AdaptFrame(frame.width(), frame.height(), frame.timestamp_us(), + if (!AdaptFrame(buffer->width(), buffer->height(), aligned_timestamp_us, &adapted_width, &adapted_height, &crop_width, &crop_height, &crop_x, &crop_y)) { return false; } - // TODO(theomonnom): Should this be handled by the users? - rtc::scoped_refptr buffer = - frame.video_frame_buffer(); if (adapted_width != frame.width() || adapted_height != frame.height()) { buffer = buffer->CropAndScale(crop_x, crop_y, crop_width, crop_height, adapted_width, adapted_height); } - if (apply_rotation() && frame.rotation() != webrtc::kVideoRotation_0) { + webrtc::VideoRotation rotation = frame.rotation(); + if (apply_rotation() && rotation != webrtc::kVideoRotation_0) { // If the buffer is I420, rtc::AdaptedVideoTrackSource will handle the // rotation for us. buffer = buffer->ToI420(); @@ -235,7 +235,7 @@ bool NativeVideoTrackSource::on_captured_frame( OnFrame(webrtc::VideoFrame::Builder() .set_video_frame_buffer(buffer) - .set_rotation(frame.rotation()) + .set_rotation(rotation) .set_timestamp_us(aligned_timestamp_us) .build()); @@ -247,8 +247,15 @@ AdaptedVideoTrackSource::AdaptedVideoTrackSource( : source_(source) {} bool AdaptedVideoTrackSource::on_captured_frame( - std::unique_ptr frame) const { - return source_->on_captured_frame(frame->get()); + const std::unique_ptr& frame) const { + auto rtc_frame = frame->get(); + rtc_frame.set_timestamp_us(rtc::TimeMicros()); + + // auto buffer = webrtc::I420Buffer::Create(1280, 720); + // webrtc::I420Buffer::SetBlack(buffer.get()); + // rtc_frame.set_video_frame_buffer(buffer); + + return source_->on_captured_frame(rtc_frame); } rtc::scoped_refptr AdaptedVideoTrackSource::get() @@ -256,8 +263,8 @@ rtc::scoped_refptr AdaptedVideoTrackSource::get() return source_; } -std::unique_ptr create_adapted_video_track_source() { - return std::make_unique( +std::shared_ptr new_adapted_video_track_source() { + return std::make_shared( rtc::make_ref_counted()); } diff --git a/webrtc-sys/src/media_stream.rs b/webrtc-sys/src/media_stream.rs index 4f1fbd0..2afbd33 100644 --- a/webrtc-sys/src/media_stream.rs +++ b/webrtc-sys/src/media_stream.rs @@ -52,10 +52,8 @@ pub mod ffi { fn get_video_tracks(self: &MediaStream) -> Vec; fn find_audio_track(self: &MediaStream, track_id: String) -> SharedPtr; fn find_video_track(self: &MediaStream, track_id: String) -> SharedPtr; - fn add_audio_track(self: &MediaStream, audio_track: SharedPtr) -> bool; - fn add_video_track(self: &MediaStream, video_track: SharedPtr) -> bool; - fn remove_audio_track(self: &MediaStream, audio_track: SharedPtr) -> bool; - fn remove_video_track(self: &MediaStream, video_track: SharedPtr) -> bool; + fn add_track(self: &MediaStream, audio_track: SharedPtr) -> bool; + fn remove_track(self: &MediaStream, audio_track: SharedPtr) -> bool; fn kind(self: &MediaStreamTrack) -> String; fn id(self: &MediaStreamTrack) -> String; @@ -71,14 +69,19 @@ pub mod ffi { fn content_hint(self: &VideoTrack) -> ContentHint; fn set_content_hint(self: &VideoTrack, hint: ContentHint); - fn create_native_video_frame_sink( + fn new_native_video_frame_sink( observer: Box, ) -> UniquePtr; - fn on_captured_frame(self: &AdaptedVideoTrackSource, frame: UniquePtr) -> bool; + fn on_captured_frame(self: &AdaptedVideoTrackSource, frame: &UniquePtr) + -> bool; - unsafe fn media_to_video(track: *const MediaStreamTrack) -> *const VideoTrack; - unsafe fn media_to_audio(track: *const MediaStreamTrack) -> *const AudioTrack; + fn new_adapted_video_track_source() -> SharedPtr; + + fn video_to_media(track: SharedPtr) -> SharedPtr; + fn audio_to_media(track: SharedPtr) -> SharedPtr; + fn media_to_video(track: SharedPtr) -> SharedPtr; + fn media_to_audio(track: SharedPtr) -> SharedPtr; fn _shared_media_stream_track() -> SharedPtr; fn _shared_audio_track() -> SharedPtr; @@ -103,8 +106,9 @@ impl_thread_safety!(ffi::MediaStream, Send + Sync); impl_thread_safety!(ffi::AudioTrack, Send + Sync); impl_thread_safety!(ffi::VideoTrack, Send + Sync); impl_thread_safety!(ffi::NativeVideoFrameSink, Send + Sync); +impl_thread_safety!(ffi::AdaptedVideoTrackSource, Send + Sync); -pub trait VideoFrameSink: Send + Sync { +pub trait VideoFrameSink: Send { fn on_frame(&self, frame: UniquePtr); fn on_discarded_frame(&self); fn on_constraints_changed(&self, constraints: ffi::VideoTrackSourceConstraints); diff --git a/webrtc-sys/src/peer_connection.cpp b/webrtc-sys/src/peer_connection.cpp index 0aa6f46..44ffd32 100644 --- a/webrtc-sys/src/peer_connection.cpp +++ b/webrtc-sys/src/peer_connection.cpp @@ -70,7 +70,7 @@ void PeerConnection::set_remote_description( observer.observer); } -std::unique_ptr PeerConnection::create_data_channel( +std::shared_ptr PeerConnection::create_data_channel( rust::String label, std::unique_ptr init) const { auto result = @@ -80,7 +80,7 @@ std::unique_ptr PeerConnection::create_data_channel( throw std::runtime_error(serialize_error(to_error(result.error()))); } - return std::make_unique(rtc_runtime_, result.value()); + return std::make_shared(rtc_runtime_, result.value()); } std::shared_ptr PeerConnection::add_track( @@ -106,7 +106,7 @@ std::shared_ptr PeerConnection::add_transceiver( RtpTransceiverInit init) const { auto result = peer_connection_->AddTransceiver( track->get(), to_native_rtp_transceiver_init(init)); - if (result.ok()) + if (!result.ok()) throw std::runtime_error(serialize_error(to_error(result.error()))); return std::make_shared(result.value()); @@ -119,7 +119,7 @@ std::shared_ptr PeerConnection::add_transceiver_for_media( static_cast(media_type), to_native_rtp_transceiver_init(init)); - if (result.ok()) + if (!result.ok()) throw std::runtime_error(serialize_error(to_error(result.error()))); return std::make_shared(result.value()); @@ -158,12 +158,48 @@ void PeerConnection::add_ice_candidate( [&](const webrtc::RTCError& err) { observer.OnComplete(to_error(err)); }); } +std::unique_ptr PeerConnection::current_local_description() + const { + auto local_description = peer_connection_->current_local_description(); + if (local_description) + return std::make_unique(local_description->Clone()); + + return nullptr; +} + +std::unique_ptr PeerConnection::current_remote_description() + const { + auto remote_description = peer_connection_->current_remote_description(); + if (remote_description) + return std::make_unique(remote_description->Clone()); + + return nullptr; +} + +std::unique_ptr PeerConnection::pending_local_description() + const { + auto local_description = peer_connection_->pending_local_description(); + if (local_description) + return std::make_unique(local_description->Clone()); + + return nullptr; +} + +std::unique_ptr PeerConnection::pending_remote_description() + const { + auto remote_description = peer_connection_->pending_remote_description(); + if (remote_description) + return std::make_unique(remote_description->Clone()); + + return nullptr; +} + std::unique_ptr PeerConnection::local_description() const { auto local_description = peer_connection_->local_description(); if (local_description) return std::make_unique(local_description->Clone()); - return std::unique_ptr(); + return nullptr; } std::unique_ptr PeerConnection::remote_description() const { @@ -171,7 +207,12 @@ std::unique_ptr PeerConnection::remote_description() const { if (remote_description) return std::make_unique(remote_description->Clone()); - return std::unique_ptr(); + return nullptr; +} + +PeerConnectionState PeerConnection::connection_state() const { + return static_cast( + peer_connection_->peer_connection_state()); } SignalingState PeerConnection::signaling_state() const { @@ -188,7 +229,7 @@ IceConnectionState PeerConnection::ice_connection_state() const { peer_connection_->ice_connection_state()); } -void PeerConnection::close() { +void PeerConnection::close() const { peer_connection_->Close(); } @@ -213,7 +254,13 @@ create_native_add_ice_candidate_observer( NativePeerConnectionObserver::NativePeerConnectionObserver( std::shared_ptr rtc_runtime, rust::Box observer) - : rtc_runtime_(std::move(rtc_runtime)), observer_(std::move(observer)) {} + : rtc_runtime_(std::move(rtc_runtime)), observer_(std::move(observer)) { + RTC_LOG(LS_INFO) << "NativePeerConnectionObserver()"; +} + +NativePeerConnectionObserver::~NativePeerConnectionObserver() { + RTC_LOG(LS_INFO) << "~NativePeerConnectionObserver()"; +} void NativePeerConnectionObserver::OnSignalingChange( webrtc::PeerConnectionInterface::SignalingState new_state) { @@ -233,7 +280,7 @@ void NativePeerConnectionObserver::OnRemoveStream( void NativePeerConnectionObserver::OnDataChannel( rtc::scoped_refptr data_channel) { observer_->on_data_channel( - std::make_unique(rtc_runtime_, data_channel)); + std::make_shared(rtc_runtime_, data_channel)); } void NativePeerConnectionObserver::OnRenegotiationNeeded() { @@ -342,11 +389,11 @@ void NativePeerConnectionObserver::OnInterestingUsage(int usage_pattern) { observer_->on_interesting_usage(usage_pattern); } -std::unique_ptr +std::shared_ptr create_native_peer_connection_observer( std::shared_ptr rtc_runtime, rust::Box observer) { - return std::make_unique(rtc_runtime, + return std::make_shared(rtc_runtime, std::move(observer)); } } // namespace livekit diff --git a/webrtc-sys/src/peer_connection.rs b/webrtc-sys/src/peer_connection.rs index e4bc8d6..0401201 100644 --- a/webrtc-sys/src/peer_connection.rs +++ b/webrtc-sys/src/peer_connection.rs @@ -6,8 +6,9 @@ use crate::media_stream::ffi::MediaStream; use crate::rtc_error::ffi::RTCError; use crate::rtp_receiver::ffi::RtpReceiver; use crate::rtp_transceiver::ffi::RtpTransceiver; -use cxx::{SharedPtr, UniquePtr}; +use cxx::SharedPtr; use std::mem::ManuallyDrop; +use std::sync::Arc; #[cxx::bridge(namespace = "livekit")] pub mod ffi { @@ -183,7 +184,7 @@ pub mod ffi { self: &PeerConnection, label: String, init: UniquePtr, - ) -> Result>; + ) -> Result>; fn add_ice_candidate( self: &PeerConnection, @@ -191,9 +192,11 @@ pub mod ffi { observer: Pin<&mut NativeAddIceCandidateObserver>, ); - fn local_description(self: &PeerConnection) -> UniquePtr; + fn current_local_description(self: &PeerConnection) -> UniquePtr; - fn remote_description(self: &PeerConnection) -> UniquePtr; + fn current_remote_description(self: &PeerConnection) -> UniquePtr; + + fn connection_state(self: &PeerConnection) -> PeerConnectionState; fn signaling_state(self: &PeerConnection) -> SignalingState; @@ -201,18 +204,18 @@ pub mod ffi { fn ice_connection_state(self: &PeerConnection) -> IceConnectionState; - fn close(self: Pin<&mut PeerConnection>); + fn close(self: &PeerConnection); fn create_native_peer_connection_observer( rtc_runtime: SharedPtr, observer: Box, - ) -> UniquePtr; + ) -> SharedPtr; fn create_native_add_ice_candidate_observer( observer: Box, ) -> UniquePtr; - fn _unique_peer_connection() -> UniquePtr; // Ignore + fn _shared_peer_connection() -> SharedPtr; // Ignore } extern "Rust" { @@ -227,7 +230,7 @@ pub mod ffi { fn on_remove_stream(self: &PeerConnectionObserverWrapper, stream: SharedPtr); fn on_data_channel( self: &PeerConnectionObserverWrapper, - data_channel: UniquePtr, + data_channel: SharedPtr, ); fn on_renegotiation_needed(self: &PeerConnectionObserverWrapper); fn on_negotiation_needed_event(self: &PeerConnectionObserverWrapper, event: u32); @@ -325,7 +328,7 @@ pub trait PeerConnectionObserver: Send + Sync { fn on_signaling_change(&self, new_state: ffi::SignalingState); fn on_add_stream(&self, stream: SharedPtr); fn on_remove_stream(&self, stream: SharedPtr); - fn on_data_channel(&self, data_channel: UniquePtr); + fn on_data_channel(&self, data_channel: SharedPtr); fn on_renegotiation_needed(&self); fn on_negotiation_needed_event(&self, event: u32); fn on_ice_connection_change(&self, new_state: ffi::IceConnectionState); @@ -352,80 +355,57 @@ pub trait PeerConnectionObserver: Send + Sync { // Thread safety is handled inside PeerConnectionObserver pub struct PeerConnectionObserverWrapper { - observer: *mut dyn PeerConnectionObserver, + observer: Arc, } impl PeerConnectionObserverWrapper { - /// # Safety - /// PeerConnectionObserver must lives as long as PeerConnectionObserverWrapper does - pub unsafe fn new(observer: *mut dyn PeerConnectionObserver) -> Self { + pub fn new(observer: Arc) -> Self { Self { observer } } fn on_signaling_change(&self, new_state: ffi::SignalingState) { - unsafe { - (*self.observer).on_signaling_change(new_state); - } + self.observer.on_signaling_change(new_state); } fn on_add_stream(&self, stream: SharedPtr) { - unsafe { - (*self.observer).on_add_stream(stream); - } + self.observer.on_add_stream(stream); } fn on_remove_stream(&self, stream: SharedPtr) { - unsafe { - (*self.observer).on_remove_stream(stream); - } + self.observer.on_remove_stream(stream); } - fn on_data_channel(&self, data_channel: UniquePtr) { - unsafe { - (*self.observer).on_data_channel(data_channel); - } + fn on_data_channel(&self, data_channel: SharedPtr) { + self.observer.on_data_channel(data_channel); } fn on_renegotiation_needed(&self) { - unsafe { - (*self.observer).on_renegotiation_needed(); - } + self.observer.on_renegotiation_needed(); } fn on_negotiation_needed_event(&self, event: u32) { - unsafe { - (*self.observer).on_negotiation_needed_event(event); - } + self.observer.on_negotiation_needed_event(event); } fn on_ice_connection_change(&self, new_state: ffi::IceConnectionState) { - unsafe { - (*self.observer).on_ice_connection_change(new_state); - } + self.observer.on_ice_connection_change(new_state); } fn on_standardized_ice_connection_change(&self, new_state: ffi::IceConnectionState) { - unsafe { - (*self.observer).on_standardized_ice_connection_change(new_state); - } + self.observer + .on_standardized_ice_connection_change(new_state); } fn on_connection_change(&self, new_state: ffi::PeerConnectionState) { - unsafe { - (*self.observer).on_connection_change(new_state); - } + self.observer.on_connection_change(new_state); } fn on_ice_gathering_change(&self, new_state: ffi::IceGatheringState) { - unsafe { - (*self.observer).on_ice_gathering_change(new_state); - } + self.observer.on_ice_gathering_change(new_state); } fn on_ice_candidate(&self, candidate: SharedPtr) { - unsafe { - (*self.observer).on_ice_candidate(candidate); - } + self.observer.on_ice_candidate(candidate); } fn on_ice_candidate_error( @@ -436,9 +416,8 @@ impl PeerConnectionObserverWrapper { error_code: i32, error_text: String, ) { - unsafe { - (*self.observer).on_ice_candidate_error(address, port, url, error_code, error_text); - } + self.observer + .on_ice_candidate_error(address, port, url, error_code, error_text); } fn on_ice_candidates_removed(&self, removed: Vec) { @@ -448,21 +427,15 @@ impl PeerConnectionObserverWrapper { vec.push(v.ptr); } - unsafe { - (*self.observer).on_ice_candidates_removed(vec); - } + self.observer.on_ice_candidates_removed(vec); } fn on_ice_connection_receiving_change(&self, receiving: bool) { - unsafe { - (*self.observer).on_ice_connection_receiving_change(receiving); - } + self.observer.on_ice_connection_receiving_change(receiving); } fn on_ice_selected_candidate_pair_changed(&self, event: ffi::CandidatePairChangeEvent) { - unsafe { - (*self.observer).on_ice_selected_candidate_pair_changed(event); - } + self.observer.on_ice_selected_candidate_pair_changed(event); } fn on_add_track(&self, receiver: SharedPtr, streams: Vec) { @@ -472,26 +445,18 @@ impl PeerConnectionObserverWrapper { vec.push(v.ptr); } - unsafe { - (*self.observer).on_add_track(receiver, vec); - } + self.observer.on_add_track(receiver, vec); } fn on_track(&self, transceiver: SharedPtr) { - unsafe { - (*self.observer).on_track(transceiver); - } + self.observer.on_track(transceiver); } fn on_remove_track(&self, receiver: SharedPtr) { - unsafe { - (*self.observer).on_remove_track(receiver); - } + self.observer.on_remove_track(receiver); } fn on_interesting_usage(&self, usage_pattern: i32) { - unsafe { - (*self.observer).on_interesting_usage(usage_pattern); - } + self.observer.on_interesting_usage(usage_pattern); } } diff --git a/webrtc-sys/src/peer_connection_factory.cpp b/webrtc-sys/src/peer_connection_factory.cpp index 9b07f3c..6a2281c 100644 --- a/webrtc-sys/src/peer_connection_factory.cpp +++ b/webrtc-sys/src/peer_connection_factory.cpp @@ -20,11 +20,13 @@ #include "api/audio_codecs/builtin_audio_decoder_factory.h" #include "api/audio_codecs/builtin_audio_encoder_factory.h" +#include "api/peer_connection_interface.h" #include "api/rtc_event_log/rtc_event_log_factory.h" #include "api/task_queue/default_task_queue_factory.h" #include "api/video_codecs/builtin_video_decoder_factory.h" #include "api/video_codecs/builtin_video_encoder_factory.h" #include "livekit/rtc_error.h" +#include "livekit/rtp_parameters.h" #include "livekit/video_decoder_factory.h" #include "livekit/video_encoder_factory.h" #include "media/engine/webrtc_media_engine.h" @@ -73,10 +75,10 @@ PeerConnectionFactory::~PeerConnectionFactory() { RTC_LOG(LS_INFO) << "PeerConnectionFactory::~PeerConnectionFactory()"; } -std::unique_ptr PeerConnectionFactory::create_peer_connection( +std::shared_ptr PeerConnectionFactory::create_peer_connection( std::unique_ptr config, - NativePeerConnectionObserver& observer) const { - webrtc::PeerConnectionDependencies deps{&observer}; + NativePeerConnectionObserver* observer) const { + webrtc::PeerConnectionDependencies deps{observer}; auto result = peer_factory_->CreatePeerConnectionOrError(*config, std::move(deps)); @@ -84,12 +86,31 @@ std::unique_ptr PeerConnectionFactory::create_peer_connection( throw std::runtime_error(serialize_error(to_error(result.error()))); } - return std::make_unique(rtc_runtime_, result.value()); + return std::make_shared(rtc_runtime_, result.value()); } -std::unique_ptr create_peer_connection_factory( +std::shared_ptr PeerConnectionFactory::create_video_track( + rust::String label, + std::shared_ptr source) const { + return std::make_shared( + peer_factory_->CreateVideoTrack(label.c_str(), source->get().get())); +} + +RtpCapabilities PeerConnectionFactory::get_rtp_sender_capabilities( + MediaType type) const { + return to_rust_rtp_capabilities(peer_factory_->GetRtpSenderCapabilities( + static_cast(type))); +} + +RtpCapabilities PeerConnectionFactory::get_rtp_receiver_capabilities( + MediaType type) const { + return to_rust_rtp_capabilities(peer_factory_->GetRtpReceiverCapabilities( + static_cast(type))); +} + +std::shared_ptr create_peer_connection_factory( std::shared_ptr rtc_runtime) { - return std::make_unique(std::move(rtc_runtime)); + return std::make_shared(std::move(rtc_runtime)); } std::unique_ptr create_rtc_configuration( diff --git a/webrtc-sys/src/peer_connection_factory.rs b/webrtc-sys/src/peer_connection_factory.rs index f74b2db..a099c64 100644 --- a/webrtc-sys/src/peer_connection_factory.rs +++ b/webrtc-sys/src/peer_connection_factory.rs @@ -32,6 +32,17 @@ pub mod ffi { pub ice_transport_type: IceTransportsType, } + extern "C++" { + include!("livekit/media_stream.h"); + include!("livekit/webrtc.h"); + include!("livekit/rtp_parameters.h"); + + type AdaptedVideoTrackSource = crate::media_stream::ffi::AdaptedVideoTrackSource; + type VideoTrack = crate::media_stream::ffi::VideoTrack; + type RtpCapabilities = crate::rtp_parameters::ffi::RtpCapabilities; + type MediaType = crate::webrtc::ffi::MediaType; + } + unsafe extern "C++" { include!("livekit/peer_connection_factory.h"); @@ -44,7 +55,7 @@ pub mod ffi { fn create_peer_connection_factory( runtime: SharedPtr, - ) -> UniquePtr; + ) -> SharedPtr; fn create_rtc_configuration(conf: RTCConfiguration) -> UniquePtr; /// # Safety @@ -52,8 +63,24 @@ pub mod ffi { unsafe fn create_peer_connection( self: &PeerConnectionFactory, config: UniquePtr, - observer: Pin<&mut NativePeerConnectionObserver>, - ) -> Result>; + observer: *mut NativePeerConnectionObserver, + ) -> Result>; + + fn create_video_track( + self: &PeerConnectionFactory, + label: String, + source: SharedPtr, + ) -> SharedPtr; + + fn get_rtp_sender_capabilities( + self: &PeerConnectionFactory, + kind: MediaType, + ) -> RtpCapabilities; + + fn get_rtp_receiver_capabilities( + self: &PeerConnectionFactory, + kind: MediaType, + ) -> RtpCapabilities; } } diff --git a/webrtc-sys/src/rtp_parameters.cpp b/webrtc-sys/src/rtp_parameters.cpp index b198bfb..0289ffa 100644 --- a/webrtc-sys/src/rtp_parameters.cpp +++ b/webrtc-sys/src/rtp_parameters.cpp @@ -19,7 +19,7 @@ namespace livekit { webrtc::RtcpFeedback to_native_rtcp_feedback(RtcpFeedback feedback) { - webrtc::RtcpFeedback native; + webrtc::RtcpFeedback native{}; native.type = static_cast(feedback.feedback_type); if (feedback.has_message_type) native.message_type = @@ -30,14 +30,14 @@ webrtc::RtcpFeedback to_native_rtcp_feedback(RtcpFeedback feedback) { webrtc::RtpCodecCapability to_native_rtp_codec_capability( RtpCodecCapability capability) { - webrtc::RtpCodecCapability native; + webrtc::RtpCodecCapability native{}; // native.mime_type(); IGNORED native.name = capability.name.c_str(); native.kind = static_cast(capability.kind); if (capability.has_clock_rate) - native.clock_rate = native.clock_rate; + native.clock_rate = capability.clock_rate; if (capability.has_preferred_payload_type) native.preferred_payload_type = capability.preferred_payload_type; @@ -62,9 +62,7 @@ webrtc::RtpCodecCapability to_native_rtp_codec_capability( native.max_temporal_layer_extensions = capability.max_temporal_layer_extensions; - native.max_spatial_layer_extensions = capability.max_spatial_layer_extensions; - native.svc_multi_stream_support = capability.svc_multi_stream_support; return native; @@ -72,7 +70,7 @@ webrtc::RtpCodecCapability to_native_rtp_codec_capability( webrtc::RtpHeaderExtensionCapability to_native_rtp_header_extension_capability( RtpHeaderExtensionCapability header) { - webrtc::RtpHeaderExtensionCapability native; + webrtc::RtpHeaderExtensionCapability native{}; native.uri = header.uri.c_str(); if (header.has_preferred_id) @@ -86,7 +84,7 @@ webrtc::RtpHeaderExtensionCapability to_native_rtp_header_extension_capability( } webrtc::RtpExtension to_native_rtp_extension(RtpExtension ext) { - webrtc::RtpExtension native; + webrtc::RtpExtension native{}; native.uri = ext.uri.c_str(); native.id = ext.id; native.encrypt = ext.encrypt; @@ -94,7 +92,7 @@ webrtc::RtpExtension to_native_rtp_extension(RtpExtension ext) { } webrtc::RtpFecParameters to_rtp_fec_parameters(RtpFecParameters fec) { - webrtc::RtpFecParameters native; + webrtc::RtpFecParameters native{}; if (fec.has_ssrc) native.ssrc = fec.ssrc; @@ -104,7 +102,7 @@ webrtc::RtpFecParameters to_rtp_fec_parameters(RtpFecParameters fec) { } webrtc::RtpRtxParameters to_rtp_rtx_parameters(RtpRtxParameters rtx) { - webrtc::RtpRtxParameters native; + webrtc::RtpRtxParameters native{}; if (rtx.has_ssrc) native.ssrc = rtx.ssrc; @@ -113,7 +111,7 @@ webrtc::RtpRtxParameters to_rtp_rtx_parameters(RtpRtxParameters rtx) { webrtc::RtpEncodingParameters to_native_rtp_encoding_paramters( RtpEncodingParameters parameters) { - webrtc::RtpEncodingParameters native; + webrtc::RtpEncodingParameters native{}; native.rid = parameters.rid.c_str(); if (parameters.has_ssrc) @@ -147,7 +145,7 @@ webrtc::RtpEncodingParameters to_native_rtp_encoding_paramters( webrtc::RtpCodecParameters to_native_rtp_codec_parameters( RtpCodecParameters params) { - webrtc::RtpCodecParameters native; + webrtc::RtpCodecParameters native{}; native.name = params.name.c_str(); native.kind = static_cast(params.kind); native.payload_type = params.payload_type; @@ -174,7 +172,7 @@ webrtc::RtpCodecParameters to_native_rtp_codec_parameters( } webrtc::RtpCapabilities to_rtp_capabilities(RtpCapabilities capabilities) { - webrtc::RtpCapabilities native; + webrtc::RtpCapabilities native{}; for (auto codec : capabilities.codecs) native.codecs.push_back(to_native_rtp_codec_capability(codec)); @@ -189,7 +187,7 @@ webrtc::RtpCapabilities to_rtp_capabilities(RtpCapabilities capabilities) { } webrtc::RtcpParameters to_native_rtcp_paramaters(RtcpParameters params) { - webrtc::RtcpParameters native; + webrtc::RtcpParameters native{}; if (params.has_ssrc) native.ssrc = params.ssrc; @@ -200,7 +198,7 @@ webrtc::RtcpParameters to_native_rtcp_paramaters(RtcpParameters params) { } webrtc::RtpParameters to_native_rtp_parameters(RtpParameters params) { - webrtc::RtpParameters native; + webrtc::RtpParameters native{}; native.transaction_id = params.transaction_id.c_str(); native.mid = params.mid.c_str(); @@ -223,7 +221,7 @@ webrtc::RtpParameters to_native_rtp_parameters(RtpParameters params) { } RtcpFeedback to_rust_rtcp_feedback(webrtc::RtcpFeedback feedback) { - RtcpFeedback rust; + RtcpFeedback rust{}; rust.feedback_type = static_cast(feedback.type); if (feedback.message_type.has_value()) { @@ -237,7 +235,7 @@ RtcpFeedback to_rust_rtcp_feedback(webrtc::RtcpFeedback feedback) { RtpCodecCapability to_rust_rtp_codec_capability( webrtc::RtpCodecCapability capability) { - RtpCodecCapability rust; + RtpCodecCapability rust{}; rust.mime_type = capability.mime_type(); rust.name = capability.name; rust.kind = static_cast(capability.kind); @@ -262,7 +260,7 @@ RtpCodecCapability to_rust_rtp_codec_capability( rust.ptime = capability.ptime.value(); } - if (capability.num_channels.has_value()) { + if (capability.num_channels) { rust.has_num_channels = true; rust.num_channels = capability.num_channels.value(); } @@ -284,7 +282,7 @@ RtpCodecCapability to_rust_rtp_codec_capability( RtpHeaderExtensionCapability to_rust_rtp_header_extension_capability( webrtc::RtpHeaderExtensionCapability header) { - RtpHeaderExtensionCapability rust; + RtpHeaderExtensionCapability rust{}; rust.uri = header.uri; if (header.preferred_id.has_value()) { rust.has_preferred_id = true; @@ -297,7 +295,7 @@ RtpHeaderExtensionCapability to_rust_rtp_header_extension_capability( } RtpExtension to_rust_rtp_extension(webrtc::RtpExtension ext) { - RtpExtension rust; + RtpExtension rust{}; rust.uri = ext.uri; rust.id = ext.id; rust.encrypt = ext.encrypt; @@ -305,7 +303,7 @@ RtpExtension to_rust_rtp_extension(webrtc::RtpExtension ext) { } RtpFecParameters to_rust_rtp_fec_parameters(webrtc::RtpFecParameters fec) { - RtpFecParameters rust; + RtpFecParameters rust{}; if (fec.ssrc.has_value()) { rust.has_ssrc = true; rust.ssrc = fec.ssrc.value(); @@ -316,7 +314,7 @@ RtpFecParameters to_rust_rtp_fec_parameters(webrtc::RtpFecParameters fec) { } RtpRtxParameters to_rust_rtp_rtx_parameters(webrtc::RtpRtxParameters param) { - RtpRtxParameters rust; + RtpRtxParameters rust{}; if (param.ssrc.has_value()) { rust.has_ssrc = param.ssrc.has_value(); rust.ssrc = param.ssrc.value(); @@ -326,7 +324,7 @@ RtpRtxParameters to_rust_rtp_rtx_parameters(webrtc::RtpRtxParameters param) { RtpEncodingParameters to_rust_rtp_encoding_parameters( webrtc::RtpEncodingParameters params) { - RtpEncodingParameters rust; + RtpEncodingParameters rust{}; if (params.ssrc.has_value()) { rust.has_ssrc = params.ssrc.has_value(); rust.ssrc = params.ssrc.value(); @@ -372,7 +370,7 @@ RtpEncodingParameters to_rust_rtp_encoding_parameters( RtpCodecParameters to_rust_rtp_codec_parameters( webrtc::RtpCodecParameters params) { - RtpCodecParameters rust; + RtpCodecParameters rust{}; rust.mime_type = params.mime_type(); rust.name = params.name; rust.kind = static_cast(params.kind); @@ -407,7 +405,7 @@ RtpCodecParameters to_rust_rtp_codec_parameters( } RtpCapabilities to_rust_rtp_capabilities(webrtc::RtpCapabilities capabilities) { - RtpCapabilities rust; + RtpCapabilities rust{}; for (auto codec : capabilities.codecs) rust.codecs.push_back(to_rust_rtp_codec_capability(codec)); @@ -422,7 +420,7 @@ RtpCapabilities to_rust_rtp_capabilities(webrtc::RtpCapabilities capabilities) { } RtcpParameters to_rust_rtcp_parameters(webrtc::RtcpParameters params) { - RtcpParameters rust; + RtcpParameters rust{}; if (params.ssrc.has_value()) { rust.has_ssrc = true; rust.ssrc = params.ssrc.value(); @@ -435,7 +433,7 @@ RtcpParameters to_rust_rtcp_parameters(webrtc::RtcpParameters params) { } RtpParameters to_rust_rtp_parameters(webrtc::RtpParameters params) { - RtpParameters rust; + RtpParameters rust{}; rust.transaction_id = params.transaction_id; rust.mid = params.mid; diff --git a/webrtc-sys/src/rtp_parameters.rs b/webrtc-sys/src/rtp_parameters.rs index cc34365..4e0b9ca 100644 --- a/webrtc-sys/src/rtp_parameters.rs +++ b/webrtc-sys/src/rtp_parameters.rs @@ -1,5 +1,3 @@ -use cxx::UniquePtr; - pub const DEFAULT_BITRATE_PRIORITY: f64 = 1.0; #[cxx::bridge(namespace = "livekit")] @@ -12,34 +10,34 @@ pub mod ffi { pub value: String, } - #[repr(i32)] #[derive(Debug)] + #[repr(i32)] pub enum FecMechanism { - RED, - REDAndULPFEC, - FLEXFEC, + Red, + RedAndUlpfec, + FlexFec, } - #[repr(i32)] #[derive(Debug)] + #[repr(i32)] pub enum RtcpFeedbackType { - CCM, - LNTF, - NACK, - REMB, + Ccm, + Lntf, + Nack, + Remb, TransportCC, } - #[repr(i32)] #[derive(Debug)] + #[repr(i32)] pub enum RtcpFeedbackMessageType { - GenericNACK, - PLI, - FIR, + GenericNack, + Pli, + Fir, } - #[repr(i32)] #[derive(Debug)] + #[repr(i32)] pub enum DegradationPreference { Disabled, MaintainFramerate, @@ -86,8 +84,8 @@ pub mod ffi { pub direction: RtpTransceiverDirection, } - #[repr(i32)] #[derive(Debug)] + #[repr(i32)] pub enum RtpExtensionFilter { DiscardEncryptedExtension, PreferEncryptedExtension, diff --git a/webrtc-sys/src/rtp_sender.rs b/webrtc-sys/src/rtp_sender.rs index 01ed519..08a118e 100644 --- a/webrtc-sys/src/rtp_sender.rs +++ b/webrtc-sys/src/rtp_sender.rs @@ -1,3 +1,5 @@ +use crate::impl_thread_safety; + #[cxx::bridge(namespace = "livekit")] pub mod ffi { @@ -31,3 +33,5 @@ pub mod ffi { fn _shared_rtp_sender() -> SharedPtr; } } + +impl_thread_safety!(ffi::RtpSender, Send + Sync); diff --git a/webrtc-sys/src/rtp_transceiver.cpp b/webrtc-sys/src/rtp_transceiver.cpp index db67446..92996bd 100644 --- a/webrtc-sys/src/rtp_transceiver.cpp +++ b/webrtc-sys/src/rtp_transceiver.cpp @@ -21,7 +21,7 @@ namespace livekit { webrtc::RtpTransceiverInit to_native_rtp_transceiver_init( RtpTransceiverInit init) { { - webrtc::RtpTransceiverInit native; + webrtc::RtpTransceiverInit native{}; native.direction = static_cast(init.direction); native.stream_ids = std::vector(init.stream_ids.begin(), @@ -95,6 +95,7 @@ void RtpTransceiver::stop_standard() const { void RtpTransceiver::set_codec_preferences( rust::Vec codecs) const { std::vector std_codecs; + for (auto codec : codecs) std_codecs.push_back(to_native_rtp_codec_capability(codec)); diff --git a/webrtc-sys/src/rtp_transceiver.rs b/webrtc-sys/src/rtp_transceiver.rs index 19b097a..73fe2d5 100644 --- a/webrtc-sys/src/rtp_transceiver.rs +++ b/webrtc-sys/src/rtp_transceiver.rs @@ -24,6 +24,7 @@ pub mod ffi { crate::rtp_parameters::ffi::RtpHeaderExtensionCapability; type RtpSender = crate::rtp_sender::ffi::RtpSender; type RtpReceiver = crate::rtp_receiver::ffi::RtpReceiver; + type RTCError = crate::rtc_error::ffi::RTCError; } unsafe extern "C++" { diff --git a/webrtc-sys/src/video_frame.cpp b/webrtc-sys/src/video_frame.cpp index 322999b..e80b254 100644 --- a/webrtc-sys/src/video_frame.cpp +++ b/webrtc-sys/src/video_frame.cpp @@ -62,9 +62,8 @@ webrtc::VideoFrame VideoFrame::get() const { return frame_; } -void VideoFrameBuilder::set_video_frame_buffer( - std::unique_ptr buffer) { - builder_.set_video_frame_buffer(buffer->get()); +void VideoFrameBuilder::set_video_frame_buffer(const VideoFrameBuffer& buffer) { + builder_.set_video_frame_buffer(buffer.get()); // const & ref_counted } void VideoFrameBuilder::set_timestamp_us(int64_t timestamp_us) { @@ -83,7 +82,7 @@ std::unique_ptr VideoFrameBuilder::build() { return std::make_unique(builder_.build()); } -std::unique_ptr create_video_frame_builder() { +std::unique_ptr new_video_frame_builder() { return std::make_unique(); } diff --git a/webrtc-sys/src/video_frame.rs b/webrtc-sys/src/video_frame.rs index 6bf6f48..d18ff21 100644 --- a/webrtc-sys/src/video_frame.rs +++ b/webrtc-sys/src/video_frame.rs @@ -21,7 +21,6 @@ pub mod ffi { include!("livekit/video_frame.h"); type VideoFrame; - type VideoFrameBuilder; fn width(self: &VideoFrame) -> i32; fn height(self: &VideoFrame) -> i32; @@ -32,18 +31,18 @@ pub mod ffi { fn transport_frame_id(self: &VideoFrame) -> u32; fn timestamp(self: &VideoFrame) -> u32; fn rotation(self: &VideoFrame) -> VideoRotation; - fn video_frame_buffer(self: &VideoFrame) -> UniquePtr; + unsafe fn video_frame_buffer(self: &VideoFrame) -> UniquePtr; - fn set_video_frame_buffer( - self: Pin<&mut VideoFrameBuilder>, - buffer: UniquePtr, - ); + // VideoFrameBuilder + type VideoFrameBuilder; + fn new_video_frame_builder() -> UniquePtr; fn set_timestamp_us(self: Pin<&mut VideoFrameBuilder>, timestamp_us: i64); fn set_rotation(self: Pin<&mut VideoFrameBuilder>, rotation: VideoRotation); fn set_id(self: Pin<&mut VideoFrameBuilder>, id: u16); + fn set_video_frame_buffer(self: Pin<&mut VideoFrameBuilder>, buffer: &VideoFrameBuffer); + fn build(self: Pin<&mut VideoFrameBuilder>) -> UniquePtr; - fn create_video_frame_builder() -> UniquePtr; } } diff --git a/webrtc-sys/src/video_frame_buffer.cpp b/webrtc-sys/src/video_frame_buffer.cpp index b8d6bcd..bb8ff62 100644 --- a/webrtc-sys/src/video_frame_buffer.cpp +++ b/webrtc-sys/src/video_frame_buffer.cpp @@ -34,7 +34,7 @@ int VideoFrameBuffer::height() const { return buffer_->height(); } -std::unique_ptr VideoFrameBuffer::to_i420() { +std::unique_ptr VideoFrameBuffer::to_i420() const { return std::make_unique(buffer_->ToI420()); } @@ -187,11 +187,16 @@ webrtc::BiplanarYuv8Buffer* BiplanarYuv8Buffer::buffer() const { return static_cast(buffer_.get()); } -std::unique_ptr create_i420_buffer(int width, int height) { +std::unique_ptr new_i420_buffer(int width, int height) { return std::make_unique( webrtc::I420Buffer::Create(width, height)); } +std::unique_ptr copy_i420_buffer( + const std::unique_ptr& i420) { + return std::make_unique(webrtc::I420Buffer::Copy(*i420->get())); +} + I420Buffer::I420Buffer(rtc::scoped_refptr buffer) : PlanarYuv8Buffer(buffer) {} @@ -199,6 +204,18 @@ I420ABuffer::I420ABuffer( rtc::scoped_refptr buffer) : I420Buffer(buffer) {} +int I420ABuffer::stride_a() const { + return buffer()->StrideA(); +} + +const uint8_t* I420ABuffer::data_a() const { + return buffer()->DataA(); +} + +webrtc::I420ABufferInterface* I420ABuffer::buffer() const { + return static_cast(buffer_.get()); +} + I422Buffer::I422Buffer(rtc::scoped_refptr buffer) : PlanarYuv8Buffer(buffer) {} diff --git a/webrtc-sys/src/video_frame_buffer.rs b/webrtc-sys/src/video_frame_buffer.rs index 489e4d3..7bd67dd 100644 --- a/webrtc-sys/src/video_frame_buffer.rs +++ b/webrtc-sys/src/video_frame_buffer.rs @@ -34,9 +34,12 @@ pub mod ffi { fn width(self: &VideoFrameBuffer) -> i32; fn height(self: &VideoFrameBuffer) -> i32; + /// # SAFETY + /// If the buffer type is I420, the buffer must be cloned before + unsafe fn to_i420(self: &VideoFrameBuffer) -> UniquePtr; + /// # SAFETY /// The functions require ownership - unsafe fn to_i420(self: Pin<&mut VideoFrameBuffer>) -> UniquePtr; unsafe fn get_i420(self: Pin<&mut VideoFrameBuffer>) -> UniquePtr; unsafe fn get_i420a(self: Pin<&mut VideoFrameBuffer>) -> UniquePtr; unsafe fn get_i422(self: Pin<&mut VideoFrameBuffer>) -> UniquePtr; @@ -66,7 +69,11 @@ pub mod ffi { fn data_y(self: &BiplanarYuv8Buffer) -> *const u8; fn data_uv(self: &BiplanarYuv8Buffer) -> *const u8; - fn create_i420_buffer(width: i32, height: i32) -> UniquePtr; + fn stride_a(self: &I420ABuffer) -> i32; + fn data_a(self: &I420ABuffer) -> *const u8; + + fn new_i420_buffer(width: i32, height: i32) -> UniquePtr; + fn copy_i420_buffer(i420: &UniquePtr) -> UniquePtr; unsafe fn yuv_to_vfb(yuv: *const PlanarYuvBuffer) -> *const VideoFrameBuffer; unsafe fn biyuv_to_vfb(yuv: *const BiplanarYuvBuffer) -> *const VideoFrameBuffer; diff --git a/webrtc-sys/src/webrtc.cpp b/webrtc-sys/src/webrtc.cpp index a4c0f63..e7c9ccc 100644 --- a/webrtc-sys/src/webrtc.cpp +++ b/webrtc-sys/src/webrtc.cpp @@ -16,10 +16,12 @@ #include "livekit/webrtc.h" +#include "rtc_base/helpers.h" #include "rtc_base/logging.h" namespace livekit { RTCRuntime::RTCRuntime() { + // rtc::LogMessage::LogToDebug(rtc::LS_INFO); RTC_LOG(LS_INFO) << "RTCRuntime()"; RTC_CHECK(rtc::InitializeSSL()) << "Failed to InitializeSSL()"; @@ -57,6 +59,10 @@ rtc::Thread* RTCRuntime::signaling_thread() const { return signaling_thread_.get(); } +rust::String create_random_uuid() { + return rtc::CreateRandomUuid(); +} + std::shared_ptr create_rtc_runtime() { return std::make_shared(); } diff --git a/webrtc-sys/src/webrtc.rs b/webrtc-sys/src/webrtc.rs index 1fc995f..0223121 100644 --- a/webrtc-sys/src/webrtc.rs +++ b/webrtc-sys/src/webrtc.rs @@ -33,9 +33,10 @@ pub mod ffi { unsafe extern "C++" { include!("livekit/webrtc.h"); - type RTCRuntime; + fn create_random_uuid() -> String; + fn create_rtc_runtime() -> SharedPtr; } } diff --git a/webrtc-sys/src/yuv_helper.rs b/webrtc-sys/src/yuv_helper.rs index 420d25d..495b4bf 100644 --- a/webrtc-sys/src/yuv_helper.rs +++ b/webrtc-sys/src/yuv_helper.rs @@ -14,7 +14,7 @@ pub mod ffi { dst_stride_argb: i32, width: i32, height: i32, - ); + ) -> Result<()>; unsafe fn i420_to_bgra( src_y: *const u8, @@ -27,7 +27,7 @@ pub mod ffi { dst_stride_bgra: i32, width: i32, height: i32, - ); + ) -> Result<()>; unsafe fn i420_to_abgr( src_y: *const u8, @@ -40,7 +40,7 @@ pub mod ffi { dst_stride_abgr: i32, width: i32, height: i32, - ); + ) -> Result<()>; unsafe fn i420_to_rgba( src_y: *const u8, @@ -53,6 +53,41 @@ pub mod ffi { dst_stride_rgba: i32, width: i32, height: i32, - ); + ) -> Result<()>; + + unsafe fn argb_to_i420( + src_argb: *const u8, + src_stride_argb: i32, + dst_y: *mut u8, + dst_stride_y: i32, + dst_u: *mut u8, + dst_stride_u: i32, + dst_v: *mut u8, + dst_stride_v: i32, + width: i32, + height: i32, + ) -> Result<()>; + + unsafe fn abgr_to_i420( + src_abgr: *const u8, + src_stride_abgr: i32, + dst_y: *mut u8, + dst_stride_y: i32, + dst_u: *mut u8, + dst_stride_u: i32, + dst_v: *mut u8, + dst_stride_v: i32, + width: i32, + height: i32, + ) -> Result<()>; + + unsafe fn argb_to_rgb24( + src_argb: *const u8, + src_stride_argb: i32, + dst_rgb24: *mut u8, + dst_stride_rgb24: i32, + width: i32, + height: i32, + ) -> Result<()>; } }