Release camera/mic when muted, restartTrack when unmuted (#39)

* re-structure tracks

* publication.track is now a LocalTrack or RemoteTrack

* restartTrack when unmute

* stop/start on restartTrack

* move to mute/unmute

* format

* remove comment
This commit is contained in:
Hiroshi Horie
2021-11-30 15:58:19 +07:00
committed by GitHub
parent 9e0a3b19fe
commit 8037d80674
18 changed files with 279 additions and 239 deletions
+2 -3
View File
@@ -104,10 +104,9 @@ class _ControlsWidgetState extends State<ControlsWidget> {
await FlutterBackground.initialize(androidConfig: androidConfig);
await FlutterBackground.enableBackgroundExecution();
final screenTrack =
await LocalVideoTrack.createScreenTrack(); // Defaults to camera
final screenShareTrack = await LocalVideoTrack.createScreenShareTrack();
await widget.room.localParticipant.publishVideoTrack(
screenTrack,
screenShareTrack,
);
} catch (e) {
print('could not publish video: $e');
+1 -1
View File
@@ -16,10 +16,10 @@ export 'src/room.dart';
export 'src/track/local_audio_track.dart';
export 'src/track/local_track_publication.dart';
export 'src/track/local_video_track.dart';
export 'src/track/local_track.dart';
export 'src/track/options.dart';
export 'src/track/remote_track_publication.dart';
export 'src/track/track.dart';
export 'src/track/track_publication.dart';
export 'src/track/video_track.dart';
export 'src/types.dart';
export 'src/widget/video_track_renderer.dart';
+1 -1
View File
@@ -279,7 +279,7 @@ extension LocalParticipantTrackSourceExt on LocalParticipant {
final track = await LocalAudioTrack.create();
await publishAudioTrack(track);
} else if (source == TrackSource.screenShareVideo) {
final track = await LocalVideoTrack.createScreenTrack();
final track = await LocalVideoTrack.createScreenShareTrack();
await publishVideoTrack(track);
}
}
+2 -2
View File
@@ -104,10 +104,10 @@ class RemoteParticipant extends Participant {
final Track track;
if (pub.kind == lk_models.TrackType.AUDIO) {
// audio track
track = RemoteAudioTrack(pub.source, pub.name, mediaTrack, stream);
track = RemoteAudioTrack(pub.name, pub.source, stream, mediaTrack);
} else {
// video track
track = RemoteVideoTrack(pub.source, pub.name, mediaTrack, stream);
track = RemoteVideoTrack(pub.name, pub.source, stream, mediaTrack);
}
await track.start();
@@ -1,16 +1,14 @@
// import 'package:audio_session/audio_session.dart' as _as;
import 'dart:io';
import 'package:flutter/foundation.dart';
import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
import 'package:synchronized/synchronized.dart' as sync;
import '../logger.dart';
import '../proto/livekit_models.pb.dart' as lk_models;
import '../support/native_audio.dart';
import '../types.dart';
import 'local_audio_track.dart';
import 'track.dart';
import 'local_track.dart';
enum AudioTrackState {
none,
@@ -22,7 +20,7 @@ enum AudioTrackState {
typedef ConfigureNativeAudioFunc = Future<NativeAudioConfiguration> Function(
AudioTrackState state);
abstract class AudioTrack extends Track {
mixin AudioManagementMixin on AudioTrack {
// it's possible to set custom function here to customize audio session configuration
static ConfigureNativeAudioFunc nativeAudioConfigurationForAudioTrackState =
defaultNativeAudioConfigurationFunc;
@@ -32,20 +30,6 @@ abstract class AudioTrack extends Track {
static int _localTrackCount = 0;
static int _remoteTrackCount = 0;
rtc.MediaStream? mediaStream;
AudioTrack(
TrackSource source,
String name,
rtc.MediaStreamTrack track,
this.mediaStream,
) : super(
lk_models.TrackType.AUDIO,
source,
name,
track,
);
/// Start playing audio track. On web platform, create an audio element and
/// start playback
@override
@@ -149,17 +133,6 @@ Future<NativeAudioConfiguration> defaultNativeAudioConfigurationFunc(
);
}
// TODO: .record category causes exception in WebRTC lib for unknown reason
// if (this == AudioTrackState.localOnly) {
// return NativeAudioConfiguration(
// iosCategory: IosAudioCategory.record,
// iosCategoryOptions: {
// // IosAudioCategoryOption.allowBluetooth,
// },
// iosMode: IosAudioMode.spokenAudio,
// );
// }
return NativeAudioConfiguration(
appleAudioCategory: AppleAudioCategory.soloAmbient,
appleAudioCategoryOptions: {},
+25 -30
View File
@@ -2,50 +2,45 @@ import 'dart:async';
import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
import '../exceptions.dart';
import '../logger.dart';
import '../proto/livekit_models.pb.dart' as lk_models;
import '../types.dart';
import 'audio_track.dart';
import 'audio_management.dart';
import 'local_track.dart';
import 'options.dart';
class LocalAudioTrack extends AudioTrack with LocalTrack {
class LocalAudioTrack extends LocalTrack with AudioTrack, AudioManagementMixin {
// Options used for this track
@override
covariant LocalAudioTrackOptions currentOptions;
// private constructor
LocalAudioTrack._(
TrackSource source,
String name,
rtc.MediaStreamTrack track,
TrackSource source,
rtc.MediaStream stream,
) : super(source, name, track, stream);
rtc.MediaStreamTrack track,
this.currentOptions,
) : super(
name,
lk_models.TrackType.AUDIO,
source,
stream,
track,
);
/// Creates a new audio track from the default audio input device.
static Future<LocalAudioTrack> create(
[LocalAudioTrackOptions? options]) async {
final audioConstraints = options?.toMediaConstraintsMap() ??
const LocalAudioTrackOptions().toMediaConstraintsMap();
final stream =
await rtc.navigator.mediaDevices.getUserMedia(<String, dynamic>{
'audio': audioConstraints,
'video': false,
});
if (stream.getAudioTracks().isEmpty) throw TrackCreateException();
static Future<LocalAudioTrack> create([
LocalAudioTrackOptions? options,
]) async {
options ??= const LocalAudioTrackOptions();
final stream = await LocalTrack.createStream(options);
return LocalAudioTrack._(
TrackSource.microphone,
'',
stream.getAudioTracks().first,
TrackSource.microphone,
stream,
stream.getAudioTracks().first,
options,
);
}
@override
Future<bool> stop() async {
final didStop = await super.stop();
if (didStop) {
logger.fine('Stopping mediaStreamTrack...');
await mediaStreamTrack.stop();
}
return didStop;
}
}
+103 -1
View File
@@ -1,13 +1,41 @@
import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
import 'package:meta/meta.dart';
import '../exceptions.dart';
import '../internal/events.dart';
import '../logger.dart';
import '../proto/livekit_models.pb.dart' as lk_models;
import '../types.dart';
import 'options.dart';
import 'track.dart';
mixin LocalTrack on Track {
mixin VideoTrack on Track {}
mixin AudioTrack on Track {}
abstract class LocalTrack extends Track {
// Options used for this track
abstract LocalTrackOptions currentOptions;
LocalTrack(
String name,
lk_models.TrackType kind,
TrackSource source,
rtc.MediaStream mediaStream,
rtc.MediaStreamTrack mediaStreamTrack,
) : super(
name,
kind,
source,
mediaStream,
mediaStreamTrack,
);
// only local tracks can set muted
Future<void> mute() async {
logger.fine('LocalTrack.mute() muted: $muted');
if (muted) return;
await disable();
await stop();
updateMuted(true);
events.emit(TrackMuteUpdatedEvent(track: this, muted: muted));
}
@@ -15,8 +43,82 @@ mixin LocalTrack on Track {
Future<void> unmute() async {
logger.fine('LocalTrack.unmute() muted: $muted');
if (!muted) return;
await restartTrack();
await enable();
updateMuted(false);
events.emit(TrackMuteUpdatedEvent(track: this, muted: muted));
}
@override
Future<bool> stop() async {
final didStop = await super.stop();
if (didStop) {
logger.fine('Stopping mediaStreamTrack...');
await mediaStreamTrack.stop();
await mediaStream.dispose();
}
return didStop;
}
/// Creates a [rtc.MediaStream] from LocalTrackOptions.
@internal
static Future<rtc.MediaStream> createStream(
LocalTrackOptions options,
) async {
final constraints = <String, dynamic>{
'audio': options is LocalAudioTrackOptions
? options.toMediaConstraintsMap()
: false,
'video': options is LocalVideoTrackOptions
? options.toMediaConstraintsMap()
: false,
};
final rtc.MediaStream stream;
if (options is ScreenShareTrackOptions) {
stream = await rtc.navigator.mediaDevices.getDisplayMedia(constraints);
} else {
// options is CameraVideoTrackOptions
stream = await rtc.navigator.mediaDevices.getUserMedia(constraints);
}
// Check if the stream looks good
if ((options is LocalVideoTrackOptions &&
stream.getVideoTracks().isEmpty) ||
(options is LocalAudioTrackOptions &&
stream.getAudioTracks().isEmpty)) {
throw TrackCreateException(
'Failed to create stream, at least 1 video or audio track should exist');
}
return stream;
}
/// Restarts the track with new options. This is useful when switching between
/// front and back cameras.
Future<void> restartTrack([
LocalTrackOptions? options,
]) async {
if (sender == null) throw TrackCreateException('could not restart track');
if (options != null && currentOptions.runtimeType != options.runtimeType) {
throw Exception('options must be a ${currentOptions.runtimeType}');
}
currentOptions = options ?? currentOptions;
// stop if not already stopped...
await stop();
// create new track with options
final newStream = await LocalTrack.createStream(currentOptions);
final newTrack = newStream.getTracks().first;
// replace track on sender
await sender?.replaceTrack(newTrack);
// set new stream & track to this object
updateMediaStreamAndTrack(newStream, newTrack);
// mark as started
await start();
}
}
@@ -10,6 +10,9 @@ import 'track_publication.dart';
class LocalTrackPublication extends TrackPublication {
final LocalParticipant _participant;
@override
covariant LocalTrack? track;
LocalTrackPublication(
lk_models.TrackInfo info,
Track track,
+22 -80
View File
@@ -1,73 +1,45 @@
import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
import '../exceptions.dart';
import '../logger.dart';
import '../proto/livekit_models.pb.dart' as lk_models;
import '../types.dart';
import 'local_track.dart';
import 'options.dart';
import 'track.dart';
import 'video_track.dart';
/// A video track from the local device. Use static methods in this class to create
/// video tracks.
class LocalVideoTrack extends VideoTrack with LocalTrack {
//
class LocalVideoTrack extends LocalTrack with VideoTrack {
// Options used for this track
//
LocalVideoTrackOptions currentOptions;
@override
covariant LocalVideoTrackOptions currentOptions;
//
// Private constructor
//
LocalVideoTrack._(
TrackSource source,
String name,
rtc.MediaStreamTrack mediaTrack,
TrackSource source,
rtc.MediaStream stream,
rtc.MediaStreamTrack track,
this.currentOptions,
) : super(source, name, mediaTrack, stream);
rtc.RTCRtpSender? get sender => transceiver?.sender;
/// Restarts the track with new options. This is useful when switching between
/// front and back cameras.
Future<void> restartTrack([
LocalVideoTrackOptions? options,
]) async {
if (sender == null) throw TrackCreateException('could not restart track');
if (options != null && currentOptions.runtimeType != options.runtimeType) {
throw Exception('options must be a ${currentOptions.runtimeType}');
}
currentOptions = options ?? currentOptions;
// stop the current track & stream
await mediaStreamTrack.stop();
await mediaStream.dispose();
// create new track with options
final newStream = await _createStream(currentOptions);
final newTrack = newStream.getVideoTracks().first;
// replace track on sender
await sender?.replaceTrack(newTrack);
// set new stream & track to this object
setMediaStream(newStream);
mediaStreamTrack = newTrack;
}
) : super(
name,
lk_models.TrackType.VIDEO,
source,
stream,
track,
);
/// Creates a LocalVideoTrack from camera input.
static Future<LocalVideoTrack> createCameraTrack([
CameraTrackOptions? options,
]) async {
options ??= const CameraTrackOptions();
final stream = await _createStream(options);
final stream = await LocalTrack.createStream(options);
return LocalVideoTrack._(
TrackSource.camera,
Track.cameraName,
stream.getVideoTracks().first,
TrackSource.camera,
stream,
stream.getVideoTracks().first,
options,
);
}
@@ -76,49 +48,19 @@ class LocalVideoTrack extends VideoTrack with LocalTrack {
///
/// Note: Android requires a foreground service to be started prior to
/// creating a screen track. Refer to the example app for an implementation.
static Future<LocalVideoTrack> createScreenTrack([
ScreenTrackOptions? options,
static Future<LocalVideoTrack> createScreenShareTrack([
ScreenShareTrackOptions? options,
]) async {
options ??= const ScreenTrackOptions();
final stream = await _createStream(options);
options ??= const ScreenShareTrackOptions();
final stream = await LocalTrack.createStream(options);
return LocalVideoTrack._(
TrackSource.screenShareVideo,
Track.screenShareName,
stream.getVideoTracks().first,
TrackSource.screenShareVideo,
stream,
stream.getVideoTracks().first,
options,
);
}
static Future<rtc.MediaStream> _createStream(
LocalVideoTrackOptions options,
) async {
final constraints = <String, dynamic>{
'audio': false,
'video': options.toMediaConstraintsMap(),
};
final rtc.MediaStream stream;
if (options is ScreenTrackOptions) {
stream = await rtc.navigator.mediaDevices.getDisplayMedia(constraints);
} else {
// options is CameraVideoTrackOptions
stream = await rtc.navigator.mediaDevices.getUserMedia(constraints);
}
if (stream.getVideoTracks().isEmpty) throw TrackCreateException();
return stream;
}
@override
Future<bool> stop() async {
final didStop = await super.stop();
if (didStop) {
logger.fine('Stopping mediaStreamTrack...');
await mediaStreamTrack.stop();
}
return didStop;
}
}
//
+13 -4
View File
@@ -43,12 +43,19 @@ class CameraTrackOptions extends LocalVideoTrackOptions {
);
}
class ScreenTrackOptions extends LocalVideoTrackOptions {
const ScreenTrackOptions();
class ScreenShareTrackOptions extends LocalVideoTrackOptions {
const ScreenShareTrackOptions();
}
/// Base class for track options.
abstract class LocalTrackOptions {
const LocalTrackOptions();
// All subclasses must be able to report constraints
Map<String, dynamic> toMediaConstraintsMap();
}
/// Options when creating a LocalVideoTrack.
abstract class LocalVideoTrackOptions {
abstract class LocalVideoTrackOptions extends LocalTrackOptions {
// final LocalVideoTrackType type;
final VideoParameters params;
@@ -56,6 +63,7 @@ abstract class LocalVideoTrackOptions {
this.params = VideoParameters.presetQHD169,
});
@override
Map<String, dynamic> toMediaConstraintsMap() => <String, dynamic>{
'mandatory': params.toMediaConstraintsMap(),
};
@@ -235,7 +243,7 @@ class VideoParameters {
}
/// Options when creating an LocalAudioTrack. Placeholder for now.
class LocalAudioTrackOptions {
class LocalAudioTrackOptions extends LocalTrackOptions {
final bool noiseSuppression;
final bool echoCancellation;
final bool autoGainControl;
@@ -250,6 +258,7 @@ class LocalAudioTrackOptions {
this.typingNoiseDetection = true,
});
@override
Map<String, dynamic> toMediaConstraintsMap() => <String, dynamic>{
'optional': <Map<String, dynamic>>[
<String, dynamic>{'echoCancellation': echoCancellation},
+16 -7
View File
@@ -1,16 +1,27 @@
import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
import '../proto/livekit_models.pb.dart' as lk_models;
import '../types.dart';
import '_audio_api.dart' if (dart.library.html) '_audio_html.dart' as audio;
import 'audio_track.dart';
import 'audio_management.dart';
import 'local_track.dart';
import 'remote_track.dart';
class RemoteAudioTrack extends AudioTrack {
class RemoteAudioTrack extends RemoteTrack
with AudioTrack, AudioManagementMixin {
//
RemoteAudioTrack(
TrackSource source,
String name,
rtc.MediaStreamTrack track,
TrackSource source,
rtc.MediaStream stream,
) : super(source, name, track, stream);
rtc.MediaStreamTrack track,
) : super(
name,
lk_models.TrackType.AUDIO,
source,
stream,
track,
);
@override
Future<bool> start() async {
@@ -18,7 +29,6 @@ class RemoteAudioTrack extends AudioTrack {
if (didStart) {
// web support
audio.startAudio(getCid(), mediaStreamTrack);
await enable();
}
return didStart;
}
@@ -29,7 +39,6 @@ class RemoteAudioTrack extends AudioTrack {
if (didStop) {
// web support
audio.stopAudio(getCid());
await disable();
}
return didStop;
}
+39
View File
@@ -0,0 +1,39 @@
import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
import '../proto/livekit_models.pb.dart' as lk_models;
import '../types.dart';
import 'track.dart';
abstract class RemoteTrack extends Track {
RemoteTrack(
String name,
lk_models.TrackType kind,
TrackSource source,
rtc.MediaStream stream,
rtc.MediaStreamTrack track,
) : super(
name,
kind,
source,
stream,
track,
);
@override
Future<bool> start() async {
final didStart = await super.start();
if (didStart) {
await enable();
}
return didStart;
}
@override
Future<bool> stop() async {
final didStop = await super.stop();
if (didStop) {
await disable();
}
return didStop;
}
}
@@ -12,6 +12,7 @@ import '../participant/remote_participant.dart';
import '../proto/livekit_models.pb.dart' as lk_models;
import '../proto/livekit_rtc.pb.dart' as lk_rtc;
import '../utils.dart';
import 'remote_track.dart';
import 'track.dart';
import 'track_publication.dart';
@@ -31,6 +32,9 @@ class RendererVisibility {
/// Represents a track publication from a RemoteParticipant. Provides methods to
/// control if we should subscribe to the track, and its quality (for video).
class RemoteTrackPublication extends TrackPublication {
@override
covariant RemoteTrack? track;
final RemoteParticipant _participant;
bool _enabled = true;
lk_rtc.VideoQuality _videoQuality = lk_rtc.VideoQuality.HIGH;
+13 -23
View File
@@ -1,32 +1,22 @@
import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
import 'package:livekit_client/src/track/local_track.dart';
import '../track/video_track.dart';
import '../proto/livekit_models.pb.dart' as lk_models;
import '../types.dart';
import 'remote_track.dart';
class RemoteVideoTrack extends VideoTrack {
class RemoteVideoTrack extends RemoteTrack with VideoTrack {
//
RemoteVideoTrack(
TrackSource source,
String name,
rtc.MediaStreamTrack mediaTrack,
TrackSource source,
rtc.MediaStream stream,
) : super(source, name, mediaTrack, stream);
@override
Future<bool> start() async {
final didStart = await super.start();
if (didStart) {
await enable();
}
return didStart;
}
@override
Future<bool> stop() async {
final didStop = await super.stop();
if (didStop) {
await disable();
}
return didStop;
}
rtc.MediaStreamTrack track,
) : super(
name,
lk_models.TrackType.VIDEO,
source,
stream,
track,
);
}
+25 -3
View File
@@ -5,6 +5,7 @@ import 'package:uuid/uuid.dart';
import '../events.dart';
import '../extensions.dart';
import '../internal/events.dart';
import '../logger.dart';
import '../managers/event.dart';
import '../proto/livekit_models.pb.dart' as lk_models;
@@ -23,7 +24,14 @@ abstract class Track extends DisposableChangeNotifier
final String name;
final lk_models.TrackType kind;
final TrackSource source;
rtc.MediaStreamTrack mediaStreamTrack;
// read only
rtc.MediaStream get mediaStream => _mediaStream;
rtc.MediaStream _mediaStream;
// read only
rtc.MediaStreamTrack get mediaStreamTrack => _mediaStreamTrack;
rtc.MediaStreamTrack _mediaStreamTrack;
String? sid;
rtc.RTCRtpTransceiver? transceiver;
@@ -36,11 +44,14 @@ abstract class Track extends DisposableChangeNotifier
bool _muted = false;
bool get muted => _muted;
rtc.RTCRtpSender? get sender => transceiver?.sender;
Track(
this.name,
this.kind,
this.source,
this.name,
this.mediaStreamTrack,
this._mediaStream,
this._mediaStreamTrack,
) {
// Any event emitted will trigger ChangeNotifier
events.listen((event) {
@@ -128,4 +139,15 @@ abstract class Track extends DisposableChangeNotifier
@internal
void updateMuted(bool muted) => _muted = muted;
@internal
void updateMediaStreamAndTrack(
rtc.MediaStream stream, rtc.MediaStreamTrack track) {
_mediaStream = stream;
_mediaStreamTrack = track;
events.emit(TrackStreamUpdatedEvent(
track: this,
stream: stream,
));
}
}
+6 -6
View File
@@ -18,8 +18,8 @@ abstract class TrackPublication extends Disposable {
final lk_models.TrackType kind;
final TrackSource source;
Track? _track;
Track? get track => _track;
abstract Track? track;
// Track? get track => _track;
// metadata-muted
bool _muted = false;
@@ -28,7 +28,7 @@ abstract class TrackPublication extends Disposable {
bool simulcasted = false;
TrackDimension? dimension;
bool get subscribed => _track != null;
bool get subscribed => track != null;
TrackPublication.fromInfo(lk_models.TrackInfo info)
: sid = info.sid,
@@ -66,10 +66,10 @@ abstract class TrackPublication extends Disposable {
// Intended for internal use only.
@internal
Future<bool> updateTrack(Track? newValue) async {
if (_track == newValue) return false;
if (track == newValue) return false;
// dispose previous track (if exists)
await _track?.dispose();
_track = newValue;
await track?.dispose();
track = newValue;
return true;
}
}
-47
View File
@@ -1,47 +0,0 @@
import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
import 'package:meta/meta.dart';
import '../internal/events.dart';
import '../proto/livekit_models.pb.dart' as lk_models;
import '../types.dart';
import 'track.dart';
/// A video track will notify when its mediaTrack has changed.
abstract class VideoTrack extends Track {
rtc.MediaStream _mediaStream;
VideoTrack(
TrackSource source,
String name,
rtc.MediaStreamTrack mediaTrack,
this._mediaStream,
) : super(
lk_models.TrackType.VIDEO,
source,
name,
mediaTrack,
);
rtc.MediaStream get mediaStream => _mediaStream;
/// internal use
/// {@nodoc}
@internal
void setMediaStream(rtc.MediaStream stream) {
_mediaStream = stream;
events.emit(TrackStreamUpdatedEvent(
track: this,
stream: stream,
));
}
@override
Future<bool> stop() async {
final didStop = await super.stop();
if (didStop) {
await _mediaStream.dispose();
}
// _mediaStream = null;
return didStop;
}
}
+1 -1
View File
@@ -1,6 +1,7 @@
import 'package:flutter/foundation.dart';
import 'package:flutter/material.dart';
import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
import 'package:livekit_client/src/track/local_track.dart';
import 'package:visibility_detector/visibility_detector.dart';
import '../events.dart';
@@ -9,7 +10,6 @@ import '../internal/events.dart';
import '../logger.dart';
import '../managers/event.dart';
import '../track/local_video_track.dart';
import '../track/video_track.dart';
/// Widget that renders a [VideoTrack].
class VideoTrackRenderer extends StatefulWidget {