talksik/client-sdk-rust
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: video publishing (#42)

Browse Source 
- Prepare webrtc abstraction ( for future wasm support )
- Added track publish support for videos
  - Added LogoTrack example to simple_room demo
- Lot of cleanup
- There are compiler warnings I'll solve on our v1 release
This commit is contained in:
Théo Monnom
2023-03-18 03:25:16 +01:00
committed by GitHub
parent 4443eae434
commit cad6d36201
123 changed files with 8534 additions and 4601 deletions
Download Patch File Download Diff File Expand all files Collapse all files
webrtc-sys/include/livekit/data_channel.h
+10 -5
View File
@@ -37,7 +37,7 @@ class DataChannel {
std::shared_ptr<RTCRuntime> rtc_runtime,
rtc::scoped_refptr<webrtc::DataChannelInterface> data_channel);
void register_observer(NativeDataChannelObserver& observer) const;
void register_observer(NativeDataChannelObserver* observer) const;
void unregister_observer() const;
bool send(const DataBuffer& buffer) const;
rust::String label() const;
@@ -52,14 +52,17 @@ class DataChannel {
std::unique_ptr<NativeDataChannelInit> create_data_channel_init(
DataChannelInit init);
static std::unique_ptr<DataChannel> _unique_data_channel() {
static std::shared_ptr<DataChannel> _shared_data_channel() {
return nullptr; // Ignore
}
class NativeDataChannelObserver : public webrtc::DataChannelObserver {
public:
explicit NativeDataChannelObserver(
rust::Box<DataChannelObserverWrapper> observer);
rust::Box<DataChannelObserverWrapper> observer,
DataChannel* dc);
~NativeDataChannelObserver();
void OnStateChange() override;
void OnMessage(const webrtc::DataBuffer& buffer) override;
@@ -67,8 +70,10 @@ class NativeDataChannelObserver : public webrtc::DataChannelObserver {
private:
rust::Box<DataChannelObserverWrapper> observer_;
DataChannel* dc_;
};
std::unique_ptr<NativeDataChannelObserver> create_native_data_channel_observer(
rust::Box<DataChannelObserverWrapper> observer);
std::shared_ptr<NativeDataChannelObserver> create_native_data_channel_observer(
rust::Box<DataChannelObserverWrapper> observer,
DataChannel* dc);
} // namespace livekit
webrtc-sys/include/livekit/jsep.h
+1
View File
@@ -65,6 +65,7 @@ class SessionDescription {
explicit SessionDescription(
std::unique_ptr<webrtc::SessionDescriptionInterface> session_description);
SdpType sdp_type() const;
rust::String stringify() const;
std::unique_ptr<SessionDescription> clone() const;
std::unique_ptr<webrtc::SessionDescriptionInterface> release();
webrtc-sys/include/livekit/media_stream.h
+22 -11
View File
@@ -26,6 +26,7 @@
#include "rtc_base/synchronization/mutex.h"
#include "rtc_base/timestamp_aligner.h"
#include "rust/cxx.h"
#include "system_wrappers/include/clock.h"
namespace livekit {
class MediaStream;
@@ -50,10 +51,8 @@ class MediaStream {
std::shared_ptr<AudioTrack> find_audio_track(rust::String track_id) const;
std::shared_ptr<VideoTrack> find_video_track(rust::String track_id) const;
bool add_audio_track(std::shared_ptr<AudioTrack> audio_track) const;
bool add_video_track(std::shared_ptr<VideoTrack> video_track) const;
bool remove_audio_track(std::shared_ptr<AudioTrack> audio_track) const;
bool remove_video_track(std::shared_ptr<VideoTrack> video_track) const;
bool add_track(std::shared_ptr<MediaStreamTrack> track) const;
bool remove_track(std::shared_ptr<MediaStreamTrack> track) const;
private:
rtc::scoped_refptr<webrtc::MediaStreamInterface> media_stream_;
@@ -121,7 +120,7 @@ class NativeVideoFrameSink
rust::Box<VideoFrameSinkWrapper> observer_;
};
std::unique_ptr<NativeVideoFrameSink> create_native_video_frame_sink(
std::unique_ptr<NativeVideoFrameSink> new_native_video_frame_sink(
rust::Box<VideoFrameSinkWrapper> observer);
// Native impl of the WebRTC interface
@@ -146,7 +145,7 @@ class AdaptedVideoTrackSource {
public:
AdaptedVideoTrackSource(rtc::scoped_refptr<NativeVideoTrackSource> source);
bool on_captured_frame(std::unique_ptr<VideoFrame> frame)
bool on_captured_frame(const std::unique_ptr<VideoFrame>& frame)
const; // frames pushed from Rust (+interior mutability)
rtc::scoped_refptr<NativeVideoTrackSource> get() const;
@@ -155,14 +154,26 @@ class AdaptedVideoTrackSource {
rtc::scoped_refptr<NativeVideoTrackSource> source_;
};
std::unique_ptr<AdaptedVideoTrackSource> create_adapted_video_track_source();
std::shared_ptr<AdaptedVideoTrackSource> new_adapted_video_track_source();
static const VideoTrack* media_to_video(const MediaStreamTrack* track) {
return static_cast<const VideoTrack*>(track);
static std::shared_ptr<MediaStreamTrack> video_to_media(
std::shared_ptr<VideoTrack> track) {
return track;
}
static const AudioTrack* media_to_audio(const MediaStreamTrack* track) {
return static_cast<const AudioTrack*>(track);
static std::shared_ptr<MediaStreamTrack> audio_to_media(
std::shared_ptr<AudioTrack> track) {
return track;
}
static std::shared_ptr<VideoTrack> media_to_video(
std::shared_ptr<MediaStreamTrack> track) {
return std::static_pointer_cast<VideoTrack>(track);
}
static std::shared_ptr<AudioTrack> media_to_audio(
std::shared_ptr<MediaStreamTrack> track) {
return std::static_pointer_cast<AudioTrack>(track);
}
static std::shared_ptr<MediaStreamTrack> _shared_media_stream_track() {
webrtc-sys/include/livekit/peer_connection.h
+16 -4
View File
@@ -57,7 +57,7 @@ class PeerConnection {
void set_remote_description(std::unique_ptr<SessionDescription> desc,
NativeSetRemoteSdpObserverHandle& observer) const;
std::unique_ptr<DataChannel> create_data_channel(
std::shared_ptr<DataChannel> create_data_channel(
rust::String label,
std::unique_ptr<NativeDataChannelInit> init) const;
@@ -84,24 +84,34 @@ class PeerConnection {
rust::Vec<RtpTransceiverPtr> get_transceivers() const;
std::unique_ptr<SessionDescription> current_local_description() const;
std::unique_ptr<SessionDescription> current_remote_description() const;
std::unique_ptr<SessionDescription> pending_local_description() const;
std::unique_ptr<SessionDescription> pending_remote_description() const;
std::unique_ptr<SessionDescription> local_description() const;
std::unique_ptr<SessionDescription> remote_description() const;
PeerConnectionState connection_state() const;
SignalingState signaling_state() const;
IceGatheringState ice_gathering_state() const;
IceConnectionState ice_connection_state() const;
void close();
void close() const;
private:
std::shared_ptr<RTCRuntime> rtc_runtime_;
rtc::scoped_refptr<webrtc::PeerConnectionInterface> peer_connection_;
};
static std::unique_ptr<PeerConnection> _unique_peer_connection() {
static std::shared_ptr<PeerConnection> _shared_peer_connection() {
return nullptr; // Ignore
}
@@ -126,6 +136,8 @@ class NativePeerConnectionObserver : public webrtc::PeerConnectionObserver {
std::shared_ptr<RTCRuntime> rtc_runtime,
rust::Box<PeerConnectionObserverWrapper> observer);
~NativePeerConnectionObserver();
void OnSignalingChange(
webrtc::PeerConnectionInterface::SignalingState new_state) override;
@@ -188,7 +200,7 @@ class NativePeerConnectionObserver : public webrtc::PeerConnectionObserver {
rust::Box<PeerConnectionObserverWrapper> observer_;
};
std::unique_ptr<NativePeerConnectionObserver>
std::shared_ptr<NativePeerConnectionObserver>
create_native_peer_connection_observer(
std::shared_ptr<RTCRuntime> rtc_runtime,
rust::Box<PeerConnectionObserverWrapper> observer);
webrtc-sys/include/livekit/peer_connection_factory.h
+14 -3
View File
@@ -17,7 +17,10 @@
#pragma once
#include "api/peer_connection_interface.h"
#include "media_stream.h"
#include "peer_connection.h"
#include "rtp_parameters.h"
#include "rust/cxx.h"
#include "webrtc.h"
namespace livekit {
@@ -35,16 +38,24 @@ class PeerConnectionFactory {
explicit PeerConnectionFactory(std::shared_ptr<RTCRuntime> rtc_runtime);
~PeerConnectionFactory();
std::unique_ptr<PeerConnection> create_peer_connection(
std::shared_ptr<PeerConnection> create_peer_connection(
std::unique_ptr<NativeRTCConfiguration> config,
NativePeerConnectionObserver& observer) const;
NativePeerConnectionObserver* observer) const;
std::shared_ptr<VideoTrack> create_video_track(
rust::String label,
std::shared_ptr<AdaptedVideoTrackSource> source) const;
RtpCapabilities get_rtp_sender_capabilities(MediaType type) const;
RtpCapabilities get_rtp_receiver_capabilities(MediaType type) const;
private:
std::shared_ptr<RTCRuntime> rtc_runtime_;
rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface> peer_factory_;
};
std::unique_ptr<PeerConnectionFactory> create_peer_connection_factory(
std::shared_ptr<PeerConnectionFactory> create_peer_connection_factory(
std::shared_ptr<RTCRuntime> rtc_runtime);
std::unique_ptr<NativeRTCConfiguration> create_rtc_configuration(
RTCConfiguration conf);
webrtc-sys/include/livekit/video_frame.h
+2 -2
View File
@@ -57,7 +57,7 @@ class VideoFrameBuilder {
VideoFrameBuilder() = default;
// TODO(theomonnom): other setters?
void set_video_frame_buffer(std::unique_ptr<VideoFrameBuffer> buffer);
void set_video_frame_buffer(const VideoFrameBuffer& buffer);
void set_timestamp_us(int64_t timestamp_us);
void set_rotation(VideoRotation rotation);
void set_id(uint16_t id);
@@ -67,6 +67,6 @@ class VideoFrameBuilder {
webrtc::VideoFrame::Builder builder_;
};
std::unique_ptr<VideoFrameBuilder> create_video_frame_builder();
std::unique_ptr<VideoFrameBuilder> new_video_frame_builder();
} // namespace livekit
webrtc-sys/include/livekit/video_frame_buffer.h
+12 -3
View File
@@ -49,8 +49,9 @@ class VideoFrameBuffer {
int width() const;
int height() const;
// Takes ownership
std::unique_ptr<I420Buffer> to_i420();
std::unique_ptr<I420Buffer> to_i420() const;
// Requires ownership
std::unique_ptr<I420Buffer> get_i420();
std::unique_ptr<I420ABuffer> get_i420a();
std::unique_ptr<I422Buffer> get_i422();
@@ -131,7 +132,9 @@ class BiplanarYuv8Buffer : public BiplanarYuvBuffer {
webrtc::BiplanarYuv8Buffer* buffer() const;
};
std::unique_ptr<I420Buffer> create_i420_buffer(int width, int height);
std::unique_ptr<I420Buffer> new_i420_buffer(int width, int height);
std::unique_ptr<I420Buffer> copy_i420_buffer(
const std::unique_ptr<I420Buffer>& i420);
class I420Buffer : public PlanarYuv8Buffer {
public:
@@ -141,6 +144,12 @@ class I420Buffer : public PlanarYuv8Buffer {
class I420ABuffer : public I420Buffer {
public:
explicit I420ABuffer(rtc::scoped_refptr<webrtc::I420ABufferInterface> buffer);
int stride_a() const;
const uint8_t* data_a() const;
private:
webrtc::I420ABufferInterface* buffer() const;
};
class I422Buffer : public PlanarYuv8Buffer {
webrtc-sys/include/livekit/webrtc.h
+3
View File
@@ -18,6 +18,7 @@
#include "rtc_base/physical_socket_server.h"
#include "rtc_base/ssl_adapter.h"
#include "rust/cxx.h"
#ifdef WEBRTC_WIN
#include "rtc_base/win32_socket_init.h"
@@ -54,6 +55,8 @@ class RTCRuntime {
#endif
};
rust::String create_random_uuid();
std::shared_ptr<RTCRuntime> create_rtc_runtime();
} // namespace livekit
webrtc-sys/include/livekit/yuv_helper.h
+59 -8
View File
@@ -17,12 +17,19 @@
#pragma once
#include <memory>
#include <stdexcept>
#include <string>
#include "api/video/yuv_helper.h"
#include "webrtc-sys/src/yuv_helper.rs.h"
namespace livekit {
#define THROW_ON_ERROR(ret) \
if (ret != 0) { \
throw std::runtime_error("libyuv error: " + std::to_string(ret)); \
}
static void i420_to_argb(const uint8_t* src_y,
int src_stride_y,
const uint8_t* src_u,
@@ -33,8 +40,9 @@ static void i420_to_argb(const uint8_t* src_y,
int dst_stride_argb,
int width,
int height) {
webrtc::I420ToARGB(src_y, src_stride_y, src_u, src_stride_u, src_v,
src_stride_v, dst_argb, dst_stride_argb, width, height);
THROW_ON_ERROR(webrtc::I420ToARGB(src_y, src_stride_y, src_u, src_stride_u,
src_v, src_stride_v, dst_argb,
dst_stride_argb, width, height));
}
static void i420_to_bgra(const uint8_t* src_y,
@@ -47,8 +55,9 @@ static void i420_to_bgra(const uint8_t* src_y,
int dst_stride_bgra,
int width,
int height) {
webrtc::I420ToBGRA(src_y, src_stride_y, src_u, src_stride_u, src_v,
src_stride_v, dst_bgra, dst_stride_bgra, width, height);
THROW_ON_ERROR(webrtc::I420ToBGRA(src_y, src_stride_y, src_u, src_stride_u,
src_v, src_stride_v, dst_bgra,
dst_stride_bgra, width, height));
}
static void i420_to_abgr(const uint8_t* src_y,
@@ -61,8 +70,9 @@ static void i420_to_abgr(const uint8_t* src_y,
int dst_stride_abgr,
int width,
int height) {
webrtc::I420ToABGR(src_y, src_stride_y, src_u, src_stride_u, src_v,
src_stride_v, dst_abgr, dst_stride_abgr, width, height);
THROW_ON_ERROR(webrtc::I420ToABGR(src_y, src_stride_y, src_u, src_stride_u,
src_v, src_stride_v, dst_abgr,
dst_stride_abgr, width, height));
}
static void i420_to_rgba(const uint8_t* src_y,
@@ -75,8 +85,49 @@ static void i420_to_rgba(const uint8_t* src_y,
int dst_stride_rgba,
int width,
int height) {
webrtc::I420ToRGBA(src_y, src_stride_y, src_u, src_stride_u, src_v,
src_stride_v, dst_rgba, dst_stride_rgba, width, height);
THROW_ON_ERROR(webrtc::I420ToRGBA(src_y, src_stride_y, src_u, src_stride_u,
src_v, src_stride_v, dst_rgba,
dst_stride_rgba, width, height));
}
static void argb_to_i420(const uint8_t* src_argb,
int src_stride_argb,
uint8_t* dst_y,
int dst_stride_y,
uint8_t* dst_u,
int dst_stride_u,
uint8_t* dst_v,
int dst_stride_v,
int width,
int height) {
THROW_ON_ERROR(webrtc::ARGBToI420(src_argb, src_stride_argb, dst_y,
dst_stride_y, dst_u, dst_stride_u, dst_v,
dst_stride_v, width, height));
}
static void abgr_to_i420(const uint8_t* src_abgr,
int src_stride_abgr,
uint8_t* dst_y,
int dst_stride_y,
uint8_t* dst_u,
int dst_stride_u,
uint8_t* dst_v,
int dst_stride_v,
int width,
int height) {
THROW_ON_ERROR(webrtc::ABGRToI420(src_abgr, src_stride_abgr, dst_y,
dst_stride_y, dst_u, dst_stride_u, dst_v,
dst_stride_v, width, height));
}
static void argb_to_rgb24(const uint8_t* src_argb,
int src_stride_argb,
uint8_t* dst_rgb24,
int dst_stride_rgb24,
int width,
int height) {
THROW_ON_ERROR(webrtc::ARGBToRGB24(src_argb, src_stride_argb, dst_rgb24,
dst_stride_rgb24, width, height));
}
} // namespace livekit