talksik/client-sdk-flutter
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: web/native device consistency management (#243)

Browse Source 
* fix: Force audio settings to be consistent.

* chore: applyAudioSettings for onUnpublish.

* revert audio output select for web.

* add RoomHardwareManagementMethods to Room.

* update.

* fix analyze.

* update.

* update.

* chore: Reset sinkId for web when remoteTrack replays.

* rename setAudioOutput to setSinkId for flutter web.

* chore: Make sure AudioElement has `setSinkId` method.
This commit is contained in:
CloudWebRTC
2023-03-07 17:54:06 +08:00
committed by GitHub
parent 70e6589981
commit db2e25d424
12 changed files with 275 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files
example/lib/widgets/controls.dart
+50 -60
View File
@@ -33,7 +33,6 @@ class _ControlsWidgetState extends State<ControlsWidget> {
List<MediaDevice>? _audioInputs;
List<MediaDevice>? _audioOutputs;
List<MediaDevice>? _videoInputs;
MediaDevice? _selectedVideoInput;
StreamSubscription? _subscription;
@@ -61,7 +60,6 @@ class _ControlsWidgetState extends State<ControlsWidget> {
_audioInputs = devices.where((d) => d.kind == 'audioinput').toList();
_audioOutputs = devices.where((d) => d.kind == 'audiooutput').toList();
_videoInputs = devices.where((d) => d.kind == 'videoinput').toList();
_selectedVideoInput = _videoInputs?.first;
setState(() {});
}
@@ -94,23 +92,18 @@ class _ControlsWidgetState extends State<ControlsWidget> {
}
void _selectAudioOutput(MediaDevice device) async {
await Hardware.instance.selectAudioOutput(device);
await widget.room.setAudioOutputDevice(device);
setState(() {});
}
void _selectAudioInput(MediaDevice device) async {
await Hardware.instance.selectAudioInput(device);
await widget.room.setAudioInputDevice(device);
setState(() {});
}
void _selectVideoInput(MediaDevice device) async {
final track = participant.videoTracks.firstOrNull?.track;
if (track == null) return;
if (_selectedVideoInput?.deviceId != device.deviceId) {
await track.switchCamera(device.deviceId);
_selectedVideoInput = device;
setState(() {});
}
await widget.room.setVideoInputDevice(device);
setState(() {});
}
void _toggleCamera() async {
@@ -293,8 +286,7 @@ class _ControlsWidgetState extends State<ControlsWidget> {
value: device,
child: ListTile(
leading: (device.deviceId ==
Hardware
.instance.selectedAudioInput?.deviceId)
widget.room.selectedAudioInputDeviceId)
? const Icon(
EvaIcons.checkmarkSquare,
color: Colors.white,
@@ -317,45 +309,43 @@ class _ControlsWidgetState extends State<ControlsWidget> {
icon: const Icon(EvaIcons.micOff),
tooltip: 'un-mute audio',
),
if (!lkPlatformIs(PlatformType.web))
PopupMenuButton<MediaDevice>(
icon: const Icon(Icons.volume_up),
itemBuilder: (BuildContext context) {
return [
const PopupMenuItem<MediaDevice>(
value: null,
child: ListTile(
leading: Icon(
EvaIcons.speaker,
color: Colors.white,
),
title: Text('Select Audio Output'),
PopupMenuButton<MediaDevice>(
icon: const Icon(Icons.volume_up),
itemBuilder: (BuildContext context) {
return [
const PopupMenuItem<MediaDevice>(
value: null,
child: ListTile(
leading: Icon(
EvaIcons.speaker,
color: Colors.white,
),
title: Text('Select Audio Output'),
),
if (_audioOutputs != null)
..._audioOutputs!.map((device) {
return PopupMenuItem<MediaDevice>(
value: device,
child: ListTile(
leading: (device.deviceId ==
Hardware
.instance.selectedAudioOutput?.deviceId)
? const Icon(
EvaIcons.checkmarkSquare,
color: Colors.white,
)
: const Icon(
EvaIcons.square,
color: Colors.white,
),
title: Text(device.label),
),
onTap: () => _selectAudioOutput(device),
);
}).toList()
];
},
),
),
if (_audioOutputs != null)
..._audioOutputs!.map((device) {
return PopupMenuItem<MediaDevice>(
value: device,
child: ListTile(
leading: (device.deviceId ==
widget.room.selectedAudioOutputDeviceId)
? const Icon(
EvaIcons.checkmarkSquare,
color: Colors.white,
)
: const Icon(
EvaIcons.square,
color: Colors.white,
),
title: Text(device.label),
),
onTap: () => _selectAudioOutput(device),
);
}).toList()
];
},
),
if (participant.isCameraEnabled())
PopupMenuButton<MediaDevice>(
icon: const Icon(EvaIcons.video),
@@ -377,16 +367,16 @@ class _ControlsWidgetState extends State<ControlsWidget> {
return PopupMenuItem<MediaDevice>(
value: device,
child: ListTile(
leading:
(device.deviceId == _selectedVideoInput?.deviceId)
? const Icon(
EvaIcons.checkmarkSquare,
color: Colors.white,
)
: const Icon(
EvaIcons.square,
color: Colors.white,
),
leading: (device.deviceId ==
widget.room.selectedVideoInputDeviceId)
? const Icon(
EvaIcons.checkmarkSquare,
color: Colors.white,
)
: const Icon(
EvaIcons.square,
color: Colors.white,
),
title: Text(device.label),
),
onTap: () => _selectVideoInput(device),
example/lib/widgets/participant.dart
+5 -65
View File
@@ -82,20 +82,12 @@ abstract class _ParticipantWidgetState<T extends ParticipantWidget>
VideoTrack? get activeVideoTrack;
TrackPublication? get videoPublication;
TrackPublication? get firstAudioPublication;
List<MediaDevice>? _audioOutputs;
MediaDevice? _selectedAudioDevice;
@override
void initState() {
super.initState();
widget.participant.addListener(_onParticipantChanged);
_onParticipantChanged();
Hardware.instance.audioOutputs().then((value) {
setState(() {
_audioOutputs = value;
_selectedAudioDevice = _audioOutputs?.firstOrNull;
});
});
}
@override
@@ -116,14 +108,6 @@ abstract class _ParticipantWidgetState<T extends ParticipantWidget>
// since the updated values are computed properties.
void _onParticipantChanged() => setState(() {});
void _onSelectAudioOutput(MediaDevice device) {
var audioTrack = firstAudioPublication?.track as RemoteAudioTrack;
audioTrack.setAudioOutput(device.deviceId);
setState(() {
_selectedAudioDevice = device;
});
}
// Widgets to show above the info bar
List<Widget> extraWidgets(bool isScreenShare) => [];
@@ -146,8 +130,10 @@ abstract class _ParticipantWidgetState<T extends ParticipantWidget>
InkWell(
onTap: () => setState(() => _visible = !_visible),
child: activeVideoTrack != null && !activeVideoTrack!.muted
? VideoTrackRenderer(activeVideoTrack!,
fit: RTCVideoViewObjectFit.RTCVideoViewObjectFitContain)
? VideoTrackRenderer(
activeVideoTrack!,
fit: RTCVideoViewObjectFit.RTCVideoViewObjectFitContain,
)
: const NoVideoWidget(),
),
@@ -158,9 +144,7 @@ abstract class _ParticipantWidgetState<T extends ParticipantWidget>
crossAxisAlignment: CrossAxisAlignment.stretch,
mainAxisSize: MainAxisSize.min,
children: [
...extraWidgets(
widget.isScreenShare,
),
...extraWidgets(widget.isScreenShare),
ParticipantInfoWidget(
title: widget.participant.name.isNotEmpty
? '${widget.participant.name} (${widget.participant.identity})'
@@ -227,13 +211,6 @@ class _RemoteParticipantWidgetState
pub: firstAudioPublication!,
icon: EvaIcons.volumeUp,
),
if (lkPlatformIs(PlatformType.web))
RemoteTrackAudioOutputSelectMenuWidget(
audioOutputs: _audioOutputs ?? [],
selected: _selectedAudioDevice,
onSelected: _onSelectAudioOutput,
icon: EvaIcons.speaker,
),
],
),
];
@@ -276,40 +253,3 @@ class RemoteTrackPublicationMenuWidget extends StatelessWidget {
),
);
}
class RemoteTrackAudioOutputSelectMenuWidget extends StatelessWidget {
final IconData icon;
final List<MediaDevice> audioOutputs;
final MediaDevice? selected;
final Function(MediaDevice) onSelected;
const RemoteTrackAudioOutputSelectMenuWidget({
required this.audioOutputs,
required this.onSelected,
required this.selected,
required this.icon,
Key? key,
}) : super(key: key);
@override
Widget build(BuildContext context) => Material(
color: Colors.black.withOpacity(0.3),
child: PopupMenuButton<Function>(
tooltip: 'Select AudioOutput',
icon: Icon(icon),
onSelected: (value) => value(),
itemBuilder: (BuildContext context) => <PopupMenuEntry<Function>>[
...audioOutputs
.map((e) => PopupMenuItem(
child: ListTile(
trailing: (e.deviceId == selected?.deviceId)
? const Icon(Icons.check, color: Colors.white)
: null,
title: Text(e.label),
),
value: () => onSelected(e),
))
.toList(),
],
),
);
}
lib/src/core/room.dart
+100
View File
@@ -2,6 +2,7 @@ import 'dart:async';
import 'package:collection/collection.dart';
import 'package:flutter/foundation.dart';
import 'package:livekit_client/src/hardware/hardware.dart';
import 'package:livekit_client/src/support/app_state.dart';
import 'package:meta/meta.dart';
@@ -343,6 +344,7 @@ class Room extends DisposableChangeNotifier with EventsEmittable<RoomEvent> {
event.stream,
trackSid,
receiver: event.receiver,
audioOutputOptions: roomOptions.defaultAudioOutputOptions,
);
} on TrackSubscriptionExceptionEvent catch (event) {
logger.severe('addSubscribedMediaTrack() throwed ${event}');
@@ -640,3 +642,101 @@ extension RoomDebugMethods on Room {
switchCandidate: switchCandidate);
}
}
/// Room extension methods for managing audio, video.
extension RoomHardwareManagementMethods on Room {
/// Get current audio output device.
String? get selectedAudioOutputDeviceId =>
roomOptions.defaultAudioOutputOptions.deviceId ??
Hardware.instance.selectedAudioOutput?.deviceId;
/// Get current audio input device.
String? get selectedAudioInputDeviceId =>
roomOptions.defaultAudioCaptureOptions.deviceId ??
Hardware.instance.selectedAudioInput?.deviceId;
/// Get current video input device.
String? get selectedVideoInputDeviceId =>
roomOptions.defaultCameraCaptureOptions.deviceId ??
Hardware.instance.selectedVideoInput?.deviceId;
/// Get mobile device's speaker status.
bool? get speakerOn => roomOptions.defaultAudioOutputOptions.speakerOn;
/// Set audio output device.
Future<void> setAudioOutputDevice(MediaDevice device) async {
if (lkPlatformIs(PlatformType.web)) {
participants.forEach((_, participant) {
for (var audioTrack in participant.audioTracks) {
audioTrack.track?.setSinkId(device.deviceId);
}
});
Hardware.instance.selectedAudioOutput = device;
} else {
await Hardware.instance.selectAudioOutput(device);
}
engine.roomOptions = engine.roomOptions.copyWith(
defaultAudioOutputOptions: roomOptions.defaultAudioOutputOptions.copyWith(
deviceId: device.deviceId,
),
);
}
/// Set audio input device.
Future<void> setAudioInputDevice(MediaDevice device) async {
if (lkPlatformIs(PlatformType.web) && localParticipant != null) {
for (var audioTrack in localParticipant!.audioTracks) {
await audioTrack.track?.setDeviceId(device.deviceId);
}
Hardware.instance.selectedAudioInput = device;
} else {
await Hardware.instance.selectAudioInput(device);
}
engine.roomOptions = engine.roomOptions.copyWith(
defaultAudioCaptureOptions:
roomOptions.defaultAudioCaptureOptions.copyWith(
deviceId: device.deviceId,
),
);
}
/// Set video input device.
Future<void> setVideoInputDevice(MediaDevice device) async {
final track = localParticipant?.videoTracks.firstOrNull?.track;
if (track == null) return;
if (selectedVideoInputDeviceId != device.deviceId) {
await track.switchCamera(device.deviceId);
Hardware.instance.selectedVideoInput = device;
}
engine.roomOptions = engine.roomOptions.copyWith(
defaultCameraCaptureOptions:
roomOptions.defaultCameraCaptureOptions.copyWith(
deviceId: device.deviceId,
),
);
}
Future<void> setSpeakerOn(bool speakerOn) async {
if (lkPlatformIs(PlatformType.iOS) || lkPlatformIs(PlatformType.android)) {
await Hardware.instance.setSpeakerphoneOn(speakerOn);
engine.roomOptions = engine.roomOptions.copyWith(
defaultAudioOutputOptions:
roomOptions.defaultAudioOutputOptions.copyWith(
speakerOn: speakerOn,
),
);
}
}
/// Apply audio output device settings.
@internal
Future<void> applyAudioSpeakerSettings() async {
if (roomOptions.defaultAudioOutputOptions.speakerOn != null) {
if (lkPlatformIs(PlatformType.iOS) ||
lkPlatformIs(PlatformType.android)) {
await Hardware.instance.setSpeakerphoneOn(
roomOptions.defaultAudioOutputOptions.speakerOn!);
}
}
}
}
lib/src/hardware/hardware.dart
+12 -4
View File
@@ -3,6 +3,8 @@ import 'dart:async';
import 'package:collection/collection.dart';
import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
import '../logger.dart';
class MediaDevice {
const MediaDevice(this.deviceId, this.label, this.kind);
@@ -40,6 +42,7 @@ class Hardware {
devices.firstWhereOrNull((element) => element.kind == 'audiooutput');
selectedVideoInput ??=
devices.firstWhereOrNull((element) => element.kind == 'videoinput');
speakerOn = true;
});
}
@@ -54,6 +57,8 @@ class Hardware {
MediaDevice? selectedVideoInput;
bool? speakerOn;
Future<List<MediaDevice>> enumerateDevices({String? type}) async {
var infos = await rtc.navigator.mediaDevices.enumerateDevices();
var devices =
@@ -78,16 +83,18 @@ class Hardware {
Future<void> selectAudioOutput(MediaDevice device) async {
if (rtc.WebRTC.platformIsWeb) {
throw UnimplementedError('selectAudioOutput not support on web');
logger.warning('selectAudioOutput not support on web');
return;
}
selectedAudioOutput = device;
await rtc.Helper.selectAudioOutput(device.deviceId);
}
Future<void> selectAudioInput(MediaDevice device) async {
if (rtc.WebRTC.platformIsWeb || rtc.WebRTC.platformIsIOS) {
throw UnimplementedError(
if (rtc.WebRTC.platformIsWeb) {
logger.warning(
'selectAudioInput is only supported on Android/Windows/macOS');
return;
}
selectedAudioInput = device;
await rtc.Helper.selectAudioInput(device.deviceId);
@@ -95,9 +102,10 @@ class Hardware {
Future<void> setSpeakerphoneOn(bool enable) async {
if (rtc.WebRTC.platformIsMobile) {
speakerOn = enable;
await rtc.Helper.setSpeakerphoneOn(enable);
} else {
throw UnimplementedError('setSpeakerphoneOn only support on iOS/Android');
logger.warning('setSpeakerphoneOn only support on iOS/Android');
}
}
lib/src/options.dart
+34
View File
@@ -67,6 +67,8 @@ class RoomOptions {
/// Default options used when publishing a [LocalAudioTrack].
final AudioPublishOptions defaultAudioPublishOptions;
final AudioOutputOptions defaultAudioOutputOptions;
/// AdaptiveStream lets LiveKit automatically manage quality of subscribed
/// video tracks to optimize for bandwidth and CPU.
/// When attached video elements are visible, it'll choose an appropriate
@@ -92,10 +94,42 @@ class RoomOptions {
this.defaultAudioCaptureOptions = const AudioCaptureOptions(),
this.defaultVideoPublishOptions = const VideoPublishOptions(),
this.defaultAudioPublishOptions = const AudioPublishOptions(),
this.defaultAudioOutputOptions = const AudioOutputOptions(),
this.adaptiveStream = false,
this.dynacast = false,
this.stopLocalTrackOnUnpublish = true,
});
RoomOptions copyWith({
CameraCaptureOptions? defaultCameraCaptureOptions,
ScreenShareCaptureOptions? defaultScreenShareCaptureOptions,
AudioCaptureOptions? defaultAudioCaptureOptions,
VideoPublishOptions? defaultVideoPublishOptions,
AudioPublishOptions? defaultAudioPublishOptions,
AudioOutputOptions? defaultAudioOutputOptions,
bool? adaptiveStream,
bool? dynacast,
bool? stopLocalTrackOnUnpublish,
}) {
return RoomOptions(
defaultCameraCaptureOptions:
defaultCameraCaptureOptions ?? this.defaultCameraCaptureOptions,
defaultScreenShareCaptureOptions: defaultScreenShareCaptureOptions ??
this.defaultScreenShareCaptureOptions,
defaultAudioCaptureOptions:
defaultAudioCaptureOptions ?? this.defaultAudioCaptureOptions,
defaultVideoPublishOptions:
defaultVideoPublishOptions ?? this.defaultVideoPublishOptions,
defaultAudioPublishOptions:
defaultAudioPublishOptions ?? this.defaultAudioPublishOptions,
defaultAudioOutputOptions:
defaultAudioOutputOptions ?? this.defaultAudioOutputOptions,
adaptiveStream: adaptiveStream ?? this.adaptiveStream,
dynacast: dynacast ?? this.dynacast,
stopLocalTrackOnUnpublish:
stopLocalTrackOnUnpublish ?? this.stopLocalTrackOnUnpublish,
);
}
}
/// Options used when publishing video.
lib/src/participant/local.dart
+3
View File
@@ -88,6 +88,7 @@ class LocalParticipant extends Participant<LocalTrackPublication> {
// did publish
await track.onPublish();
await room.applyAudioSpeakerSettings();
[events, room.events].emit(LocalTrackPublishedEvent(
participant: this,
@@ -228,6 +229,7 @@ class LocalParticipant extends Participant<LocalTrackPublication> {
// did unpublish
await track.onUnpublish();
await room.applyAudioSpeakerSettings();
}
if (notify) {
@@ -319,6 +321,7 @@ class LocalParticipant extends Participant<LocalTrackPublication> {
await publication.mute();
}
}
await room.applyAudioSpeakerSettings();
return publication;
} else if (enabled) {
if (source == TrackSource.camera) {
lib/src/participant/remote.dart
+13
View File
@@ -8,6 +8,8 @@ import '../extensions.dart';
import '../logger.dart';
import '../proto/livekit_models.pb.dart' as lk_models;
import '../publication/remote.dart';
import '../support/platform.dart';
import '../track/options.dart';
import '../track/remote/audio.dart';
import '../track/remote/remote.dart';
import '../track/remote/video.dart';
@@ -75,6 +77,7 @@ class RemoteParticipant extends Participant<RemoteTrackPublication> {
rtc.MediaStream stream,
String trackSid, {
rtc.RTCRtpReceiver? receiver,
AudioOutputOptions audioOutputOptions = const AudioOutputOptions(),
}) async {
logger.fine('addSubscribedMediaTrack()');
@@ -123,6 +126,16 @@ class RemoteParticipant extends Participant<RemoteTrackPublication> {
}
await track.start();
/// Apply audio output selection for the web.
if (pub.kind == lk_models.TrackType.AUDIO &&
lkPlatformIs(PlatformType.web)) {
if (audioOutputOptions.deviceId != null) {
await (track as RemoteAudioTrack)
.setSinkId(audioOutputOptions.deviceId!);
}
}
await pub.updateTrack(track);
await pub.updateSubscriptionAllowed(true);
addTrackPublication(pub);
lib/src/track/local/audio.dart
+10
View File
@@ -15,6 +15,16 @@ class LocalAudioTrack extends LocalTrack
@override
covariant AudioCaptureOptions currentOptions;
Future<void> setDeviceId(String deviceId) async {
if (currentOptions.deviceId == deviceId) {
return;
}
currentOptions = currentOptions.copyWith(deviceId: deviceId);
if (!muted) {
await restartTrack();
}
}
// private constructor
@internal
LocalAudioTrack(
lib/src/track/options.dart
+36
View File
@@ -229,4 +229,40 @@ class AudioCaptureOptions extends LocalTrackOptions {
}
return constraints;
}
AudioCaptureOptions copyWith({
String? deviceId,
bool? noiseSuppression,
bool? echoCancellation,
bool? autoGainControl,
bool? highPassFilter,
bool? typingNoiseDetection,
}) {
return AudioCaptureOptions(
deviceId: deviceId ?? this.deviceId,
noiseSuppression: noiseSuppression ?? this.noiseSuppression,
echoCancellation: echoCancellation ?? this.echoCancellation,
autoGainControl: autoGainControl ?? this.autoGainControl,
highPassFilter: highPassFilter ?? this.highPassFilter,
typingNoiseDetection: typingNoiseDetection ?? this.typingNoiseDetection,
);
}
}
class AudioOutputOptions {
/// The deviceId of the output device to use.
final String? deviceId;
/// If true, the audio will be played on the speaker.
/// for mobile only
final bool? speakerOn;
const AudioOutputOptions({this.deviceId, this.speakerOn});
AudioOutputOptions copyWith({String? deviceId, bool? speakerOn}) {
return AudioOutputOptions(
deviceId: deviceId ?? this.deviceId,
speakerOn: speakerOn ?? this.speakerOn,
);
}
}
lib/src/track/remote/audio.dart
+7 -3
View File
@@ -11,7 +11,7 @@ import '../web/_audio_api.dart' if (dart.library.html) '../web/_audio_html.dart'
class RemoteAudioTrack extends RemoteTrack
with AudioTrack, RemoteAudioManagementMixin {
//
String? _deviceId;
RemoteAudioTrack(String name, TrackSource source, rtc.MediaStream stream,
rtc.MediaStreamTrack track,
{rtc.RTCRtpReceiver? receiver})
@@ -30,6 +30,9 @@ class RemoteAudioTrack extends RemoteTrack
if (didStart) {
// web support
audio.startAudio(getCid(), mediaStreamTrack);
if (_deviceId != null) {
audio.setSinkId(getCid(), _deviceId!);
}
}
return didStart;
}
@@ -44,7 +47,8 @@ class RemoteAudioTrack extends RemoteTrack
return didStop;
}
Future<void> setAudioOutput(String deviceId) async {
audio.setAudioOutput(getCid(), deviceId);
Future<void> setSinkId(String deviceId) async {
audio.setSinkId(getCid(), deviceId);
_deviceId = deviceId;
}
}
lib/src/track/web/_audio_api.dart
+1 -1
View File
@@ -8,6 +8,6 @@ void stopAudio(String id) {
// do nothing
}
void setAudioOutput(String id, String deviceId) {
void setSinkId(String id, String deviceId) {
// do nothing
}
lib/src/track/web/_audio_html.dart
+4 -3
View File
@@ -1,5 +1,5 @@
import 'dart:html' as html;
import 'dart:js_util' as jsutil;
import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
// ignore: implementation_imports
@@ -52,9 +52,10 @@ html.DivElement findOrCreateAudioContainer() {
return div as html.DivElement;
}
void setAudioOutput(String id, String deviceId) {
void setSinkId(String id, String deviceId) {
final audioElement = html.document.getElementById(audioPrefix + id);
if (audioElement is html.AudioElement) {
if (audioElement is html.AudioElement &&
jsutil.hasProperty(audioElement, 'setSinkId')) {
audioElement.setSinkId(deviceId);
}
}