feat: audio support (#45)
- Receive/Send audio frames - LocalAudioTrack
This commit is contained in:
Théo Monnom
@@ -0,0 +1,129 @@
|
||||
/*
|
||||
* Copyright 2023 LiveKit
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the “License”);
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an “AS IS” BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <atomic>
|
||||
|
||||
#include "api/task_queue/task_queue_factory.h"
|
||||
#include "modules/audio_device/include/audio_device.h"
|
||||
#include "rtc_base/synchronization/mutex.h"
|
||||
#include "rtc_base/task_queue.h"
|
||||
#include "rtc_base/task_utils/repeating_task.h"
|
||||
|
||||
namespace livekit {
|
||||
|
||||
class AudioDevice : public webrtc::AudioDeviceModule {
|
||||
public:
|
||||
AudioDevice(webrtc::TaskQueueFactory* task_queue_factory);
|
||||
~AudioDevice() override;
|
||||
|
||||
int32_t ActiveAudioLayer(AudioLayer* audioLayer) const override;
|
||||
int32_t RegisterAudioCallback(webrtc::AudioTransport* transport) override;
|
||||
|
||||
int32_t Init() override;
|
||||
int32_t Terminate() override;
|
||||
bool Initialized() const override;
|
||||
|
||||
int16_t PlayoutDevices() override;
|
||||
int16_t RecordingDevices() override;
|
||||
int32_t PlayoutDeviceName(uint16_t index,
|
||||
char name[webrtc::kAdmMaxDeviceNameSize],
|
||||
char guid[webrtc::kAdmMaxGuidSize]) override;
|
||||
|
||||
int32_t RecordingDeviceName(uint16_t index,
|
||||
char name[webrtc::kAdmMaxDeviceNameSize],
|
||||
char guid[webrtc::kAdmMaxGuidSize]) override;
|
||||
|
||||
int32_t SetPlayoutDevice(uint16_t index) override;
|
||||
int32_t SetPlayoutDevice(WindowsDeviceType device) override;
|
||||
int32_t SetRecordingDevice(uint16_t index) override;
|
||||
int32_t SetRecordingDevice(WindowsDeviceType device) override;
|
||||
|
||||
int32_t PlayoutIsAvailable(bool* available) override;
|
||||
int32_t InitPlayout() override;
|
||||
bool PlayoutIsInitialized() const override;
|
||||
int32_t RecordingIsAvailable(bool* available) override;
|
||||
int32_t InitRecording() override;
|
||||
bool RecordingIsInitialized() const override;
|
||||
|
||||
int32_t StartPlayout() override;
|
||||
int32_t StopPlayout() override;
|
||||
bool Playing() const override;
|
||||
int32_t StartRecording() override;
|
||||
int32_t StopRecording() override;
|
||||
bool Recording() const override;
|
||||
|
||||
int32_t InitSpeaker() override;
|
||||
bool SpeakerIsInitialized() const override;
|
||||
int32_t InitMicrophone() override;
|
||||
bool MicrophoneIsInitialized() const override;
|
||||
|
||||
int32_t SpeakerVolumeIsAvailable(bool* available) override;
|
||||
int32_t SetSpeakerVolume(uint32_t volume) override;
|
||||
int32_t SpeakerVolume(uint32_t* volume) const override;
|
||||
int32_t MaxSpeakerVolume(uint32_t* maxVolume) const override;
|
||||
int32_t MinSpeakerVolume(uint32_t* minVolume) const override;
|
||||
|
||||
int32_t MicrophoneVolumeIsAvailable(bool* available) override;
|
||||
int32_t SetMicrophoneVolume(uint32_t volume) override;
|
||||
int32_t MicrophoneVolume(uint32_t* volume) const override;
|
||||
int32_t MaxMicrophoneVolume(uint32_t* maxVolume) const override;
|
||||
int32_t MinMicrophoneVolume(uint32_t* minVolume) const override;
|
||||
|
||||
int32_t SpeakerMuteIsAvailable(bool* available) override;
|
||||
int32_t SetSpeakerMute(bool enable) override;
|
||||
int32_t SpeakerMute(bool* enabled) const override;
|
||||
|
||||
int32_t MicrophoneMuteIsAvailable(bool* available) override;
|
||||
int32_t SetMicrophoneMute(bool enable) override;
|
||||
int32_t MicrophoneMute(bool* enabled) const override;
|
||||
|
||||
int32_t StereoPlayoutIsAvailable(bool* available) const override;
|
||||
int32_t SetStereoPlayout(bool enable) override;
|
||||
int32_t StereoPlayout(bool* enabled) const override;
|
||||
int32_t StereoRecordingIsAvailable(bool* available) const override;
|
||||
int32_t SetStereoRecording(bool enable) override;
|
||||
int32_t StereoRecording(bool* enabled) const override;
|
||||
|
||||
int32_t PlayoutDelay(uint16_t* delayMS) const override;
|
||||
|
||||
bool BuiltInAECIsAvailable() const override;
|
||||
bool BuiltInAGCIsAvailable() const override;
|
||||
bool BuiltInNSIsAvailable() const override;
|
||||
|
||||
int32_t EnableBuiltInAEC(bool enable) override;
|
||||
int32_t EnableBuiltInAGC(bool enable) override;
|
||||
int32_t EnableBuiltInNS(bool enable) override;
|
||||
|
||||
#if defined(WEBRTC_IOS)
|
||||
int GetPlayoutAudioParameters(AudioParameters* params) const override;
|
||||
int GetRecordAudioParameters(AudioParameters* params) const override;
|
||||
#endif // WEBRTC_IOS
|
||||
|
||||
int32_t SetAudioDeviceSink(webrtc::AudioDeviceSink* sink) const override;
|
||||
|
||||
private:
|
||||
mutable webrtc::Mutex mutex_;
|
||||
webrtc::TaskQueueFactory* task_queue_factory_;
|
||||
std::unique_ptr<rtc::TaskQueue> audio_queue_;
|
||||
webrtc::RepeatingTaskHandle audio_task_;
|
||||
std::vector<int16_t> data_;
|
||||
webrtc::AudioTransport* audio_transport_;
|
||||
std::atomic<bool> playing_{false};
|
||||
std::atomic<bool> initialized_{false};
|
||||
};
|
||||
} // namespace livekit
|
||||
@@ -20,9 +20,12 @@
|
||||
|
||||
#include "api/media_stream_interface.h"
|
||||
#include "api/video/video_frame.h"
|
||||
#include "common_audio/resampler/include/push_resampler.h"
|
||||
#include "common_audio/ring_buffer.h"
|
||||
#include "livekit/helper.h"
|
||||
#include "livekit/video_frame.h"
|
||||
#include "media/base/adapted_video_track_source.h"
|
||||
#include "pc/local_audio_source.h"
|
||||
#include "rtc_base/synchronization/mutex.h"
|
||||
#include "rtc_base/timestamp_aligner.h"
|
||||
#include "rust/cxx.h"
|
||||
@@ -34,6 +37,8 @@ class MediaStreamTrack;
|
||||
class VideoTrack;
|
||||
class AudioTrack;
|
||||
class NativeVideoFrameSink;
|
||||
class NativeAudioSink;
|
||||
class AudioTrackSource;
|
||||
class AdaptedVideoTrackSource;
|
||||
} // namespace livekit
|
||||
#include "webrtc-sys/src/media_stream.rs.h"
|
||||
@@ -86,8 +91,73 @@ class MediaStreamTrack {
|
||||
class AudioTrack : public MediaStreamTrack {
|
||||
public:
|
||||
explicit AudioTrack(rtc::scoped_refptr<webrtc::AudioTrackInterface> track);
|
||||
|
||||
void add_sink(NativeAudioSink& sink) const;
|
||||
void remove_sink(NativeAudioSink& sink) const;
|
||||
|
||||
private:
|
||||
webrtc::AudioTrackInterface* track() const {
|
||||
return static_cast<webrtc::AudioTrackInterface*>(track_.get());
|
||||
}
|
||||
};
|
||||
|
||||
class NativeAudioSink : public webrtc::AudioTrackSinkInterface {
|
||||
public:
|
||||
explicit NativeAudioSink(rust::Box<AudioSinkWrapper> observer);
|
||||
void OnData(const void* audio_data,
|
||||
int bits_per_sample,
|
||||
int sample_rate,
|
||||
size_t number_of_channels,
|
||||
size_t number_of_frames) override;
|
||||
|
||||
private:
|
||||
rust::Box<AudioSinkWrapper> observer_;
|
||||
};
|
||||
|
||||
std::unique_ptr<NativeAudioSink> new_native_audio_sink(
|
||||
rust::Box<AudioSinkWrapper> observer);
|
||||
|
||||
class NativeAudioTrackSource : public webrtc::LocalAudioSource {
|
||||
public:
|
||||
NativeAudioTrackSource();
|
||||
|
||||
SourceState state() const override;
|
||||
bool remote() const override;
|
||||
|
||||
const cricket::AudioOptions options() const override;
|
||||
|
||||
void AddSink(webrtc::AudioTrackSinkInterface* sink) override;
|
||||
void RemoveSink(webrtc::AudioTrackSinkInterface* sink) override;
|
||||
|
||||
// AudioFrame should always contain 10 ms worth of data (see index.md of acm)
|
||||
void on_captured_frame(const int16_t* audio_data,
|
||||
int sample_rate,
|
||||
size_t number_of_channels,
|
||||
size_t number_of_frames);
|
||||
|
||||
private:
|
||||
webrtc::Mutex mutex_;
|
||||
std::vector<webrtc::AudioTrackSinkInterface*> sinks_;
|
||||
cricket::AudioOptions options_{};
|
||||
};
|
||||
|
||||
class AudioTrackSource {
|
||||
public:
|
||||
AudioTrackSource(rtc::scoped_refptr<NativeAudioTrackSource> source);
|
||||
|
||||
void on_captured_frame(const int16_t* audio_data,
|
||||
int sample_rate,
|
||||
size_t number_of_channels,
|
||||
size_t number_of_frames) const;
|
||||
|
||||
rtc::scoped_refptr<NativeAudioTrackSource> get() const;
|
||||
|
||||
private:
|
||||
rtc::scoped_refptr<NativeAudioTrackSource> source_;
|
||||
};
|
||||
|
||||
std::shared_ptr<AudioTrackSource> new_audio_track_source();
|
||||
|
||||
class VideoTrack : public MediaStreamTrack {
|
||||
public:
|
||||
explicit VideoTrack(rtc::scoped_refptr<webrtc::VideoTrackInterface> track);
|
||||
@@ -131,7 +201,7 @@ class NativeVideoTrackSource : public rtc::AdaptedVideoTrackSource {
|
||||
|
||||
bool is_screencast() const override;
|
||||
absl::optional<bool> needs_denoising() const override;
|
||||
webrtc::MediaSourceInterface::SourceState state() const override;
|
||||
SourceState state() const override;
|
||||
bool remote() const override;
|
||||
|
||||
bool on_captured_frame(const webrtc::VideoFrame& frame);
|
||||
|
||||
@@ -46,6 +46,10 @@ class PeerConnectionFactory {
|
||||
rust::String label,
|
||||
std::shared_ptr<AdaptedVideoTrackSource> source) const;
|
||||
|
||||
std::shared_ptr<AudioTrack> create_audio_track(
|
||||
rust::String label,
|
||||
std::shared_ptr<AudioTrackSource> source) const;
|
||||
|
||||
RtpCapabilities get_rtp_sender_capabilities(MediaType type) const;
|
||||
|
||||
RtpCapabilities get_rtp_receiver_capabilities(MediaType type) const;
|
||||
|
||||
Reference in New Issue
Block a user