/* * Copyright 2023 LiveKit * * Licensed under the Apache License, Version 2.0 (the “License”); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an “AS IS” BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #include #include "api/media_stream_interface.h" #include "api/video/video_frame.h" #include "common_audio/resampler/include/push_resampler.h" #include "common_audio/ring_buffer.h" #include "livekit/helper.h" #include "livekit/video_frame.h" #include "media/base/adapted_video_track_source.h" #include "pc/local_audio_source.h" #include "rtc_base/synchronization/mutex.h" #include "rtc_base/timestamp_aligner.h" #include "rust/cxx.h" #include "system_wrappers/include/clock.h" namespace livekit { class MediaStream; class MediaStreamTrack; class VideoTrack; class AudioTrack; class NativeVideoFrameSink; class NativeAudioSink; class AudioTrackSource; class AdaptedVideoTrackSource; } // namespace livekit #include "webrtc-sys/src/media_stream.rs.h" namespace livekit { class MediaStream { public: explicit MediaStream(rtc::scoped_refptr stream); rust::String id() const; rust::Vec get_video_tracks() const; rust::Vec get_audio_tracks() const; std::shared_ptr find_audio_track(rust::String track_id) const; std::shared_ptr find_video_track(rust::String track_id) const; bool add_track(std::shared_ptr track) const; bool remove_track(std::shared_ptr track) const; private: rtc::scoped_refptr media_stream_; }; class MediaStreamTrack { protected: explicit MediaStreamTrack( rtc::scoped_refptr track); public: static std::shared_ptr from( rtc::scoped_refptr track); rust::String kind() const; rust::String id() const; bool enabled() const; bool set_enabled(bool enable) const; TrackState state() const; rtc::scoped_refptr get() const { return track_; } protected: rtc::scoped_refptr track_; }; class AudioTrack : public MediaStreamTrack { public: explicit AudioTrack(rtc::scoped_refptr track); void add_sink(NativeAudioSink& sink) const; void remove_sink(NativeAudioSink& sink) const; private: webrtc::AudioTrackInterface* track() const { return static_cast(track_.get()); } }; class NativeAudioSink : public webrtc::AudioTrackSinkInterface { public: explicit NativeAudioSink(rust::Box observer); void OnData(const void* audio_data, int bits_per_sample, int sample_rate, size_t number_of_channels, size_t number_of_frames) override; private: rust::Box observer_; }; std::unique_ptr new_native_audio_sink( rust::Box observer); class NativeAudioTrackSource : public webrtc::LocalAudioSource { public: NativeAudioTrackSource(); SourceState state() const override; bool remote() const override; const cricket::AudioOptions options() const override; void AddSink(webrtc::AudioTrackSinkInterface* sink) override; void RemoveSink(webrtc::AudioTrackSinkInterface* sink) override; // AudioFrame should always contain 10 ms worth of data (see index.md of acm) void on_captured_frame(const int16_t* audio_data, int sample_rate, size_t number_of_channels, size_t number_of_frames); private: webrtc::Mutex mutex_; std::vector sinks_; cricket::AudioOptions options_{}; }; class AudioTrackSource { public: AudioTrackSource(rtc::scoped_refptr source); void on_captured_frame(const int16_t* audio_data, int sample_rate, size_t number_of_channels, size_t number_of_frames) const; rtc::scoped_refptr get() const; private: rtc::scoped_refptr source_; }; std::shared_ptr new_audio_track_source(); class VideoTrack : public MediaStreamTrack { public: explicit VideoTrack(rtc::scoped_refptr track); void add_sink(NativeVideoFrameSink& sink) const; void remove_sink(NativeVideoFrameSink& sink) const; void set_should_receive(bool should_receive) const; bool should_receive() const; ContentHint content_hint() const; void set_content_hint(ContentHint hint) const; private: webrtc::VideoTrackInterface* track() const { return static_cast(track_.get()); } }; class NativeVideoFrameSink : public rtc::VideoSinkInterface { public: explicit NativeVideoFrameSink(rust::Box observer); void OnFrame(const webrtc::VideoFrame& frame) override; void OnDiscardedFrame() override; void OnConstraintsChanged( const webrtc::VideoTrackSourceConstraints& constraints) override; private: rust::Box observer_; }; std::unique_ptr new_native_video_frame_sink( rust::Box observer); // Native impl of the WebRTC interface class NativeVideoTrackSource : public rtc::AdaptedVideoTrackSource { public: NativeVideoTrackSource(); ~NativeVideoTrackSource() override; bool is_screencast() const override; absl::optional needs_denoising() const override; SourceState state() const override; bool remote() const override; bool on_captured_frame(const webrtc::VideoFrame& frame); private: webrtc::Mutex mutex_; rtc::TimestampAligner timestamp_aligner_; }; class AdaptedVideoTrackSource { public: AdaptedVideoTrackSource(rtc::scoped_refptr source); bool on_captured_frame(const std::unique_ptr& frame) const; // frames pushed from Rust (+interior mutability) rtc::scoped_refptr get() const; private: rtc::scoped_refptr source_; }; std::shared_ptr new_adapted_video_track_source(); static std::shared_ptr video_to_media( std::shared_ptr track) { return track; } static std::shared_ptr audio_to_media( std::shared_ptr track) { return track; } static std::shared_ptr media_to_video( std::shared_ptr track) { return std::static_pointer_cast(track); } static std::shared_ptr media_to_audio( std::shared_ptr track) { return std::static_pointer_cast(track); } static std::shared_ptr _shared_media_stream_track() { return nullptr; // Ignore } static std::shared_ptr _shared_audio_track() { return nullptr; // Ignore } static std::shared_ptr _shared_video_track() { return nullptr; // Ignore } static std::shared_ptr _shared_media_stream() { return nullptr; // Ignore } } // namespace livekit