feat: web/native device consistency management (#243)

* fix: Force audio settings to be consistent. * chore: applyAudioSettings for onUnpublish. * revert audio output select for web. * add RoomHardwareManagementMethods to Room. * update. * fix analyze. * update. * update. * chore: Reset sinkId for web when remoteTrack replays. * rename setAudioOutput to setSinkId for flutter web. * chore: Make sure AudioElement has `setSinkId` method.
2023-03-07 17:54:06 +08:00
parent 70e6589981
commit db2e25d424
12 changed files with 275 additions and 136 deletions
@@ -33,7 +33,6 @@ class _ControlsWidgetState extends State<ControlsWidget> {
  List<MediaDevice>? _audioInputs;
  List<MediaDevice>? _audioOutputs;
  List<MediaDevice>? _videoInputs;
-  MediaDevice? _selectedVideoInput;

  StreamSubscription? _subscription;

@@ -61,7 +60,6 @@ class _ControlsWidgetState extends State<ControlsWidget> {
    _audioInputs = devices.where((d) => d.kind == 'audioinput').toList();
    _audioOutputs = devices.where((d) => d.kind == 'audiooutput').toList();
    _videoInputs = devices.where((d) => d.kind == 'videoinput').toList();
-    _selectedVideoInput = _videoInputs?.first;
    setState(() {});
  }

@@ -94,24 +92,19 @@ class _ControlsWidgetState extends State<ControlsWidget> {
  }

  void _selectAudioOutput(MediaDevice device) async {
-    await Hardware.instance.selectAudioOutput(device);
+    await widget.room.setAudioOutputDevice(device);
    setState(() {});
  }

  void _selectAudioInput(MediaDevice device) async {
-    await Hardware.instance.selectAudioInput(device);
+    await widget.room.setAudioInputDevice(device);
    setState(() {});
  }

  void _selectVideoInput(MediaDevice device) async {
-    final track = participant.videoTracks.firstOrNull?.track;
-    if (track == null) return;
-    if (_selectedVideoInput?.deviceId != device.deviceId) {
-      await track.switchCamera(device.deviceId);
-      _selectedVideoInput = device;
+    await widget.room.setVideoInputDevice(device);
    setState(() {});
  }
-  }

  void _toggleCamera() async {
    //
@@ -293,8 +286,7 @@ class _ControlsWidgetState extends State<ControlsWidget> {
                        value: device,
                        child: ListTile(
                          leading: (device.deviceId ==
-                                  Hardware
-                                      .instance.selectedAudioInput?.deviceId)
+                                  widget.room.selectedAudioInputDeviceId)
                              ? const Icon(
                                  EvaIcons.checkmarkSquare,
                                  color: Colors.white,
@@ -317,7 +309,6 @@ class _ControlsWidgetState extends State<ControlsWidget> {
              icon: const Icon(EvaIcons.micOff),
              tooltip: 'un-mute audio',
            ),
-          if (!lkPlatformIs(PlatformType.web))
          PopupMenuButton<MediaDevice>(
            icon: const Icon(Icons.volume_up),
            itemBuilder: (BuildContext context) {
@@ -338,8 +329,7 @@ class _ControlsWidgetState extends State<ControlsWidget> {
                      value: device,
                      child: ListTile(
                        leading: (device.deviceId ==
-                                  Hardware
-                                      .instance.selectedAudioOutput?.deviceId)
+                                widget.room.selectedAudioOutputDeviceId)
                            ? const Icon(
                                EvaIcons.checkmarkSquare,
                                color: Colors.white,
@@ -377,8 +367,8 @@ class _ControlsWidgetState extends State<ControlsWidget> {
                      return PopupMenuItem<MediaDevice>(
                        value: device,
                        child: ListTile(
-                          leading:
-                              (device.deviceId == _selectedVideoInput?.deviceId)
+                          leading: (device.deviceId ==
+                                  widget.room.selectedVideoInputDeviceId)
                              ? const Icon(
                                  EvaIcons.checkmarkSquare,
                                  color: Colors.white,
@@ -82,20 +82,12 @@ abstract class _ParticipantWidgetState<T extends ParticipantWidget>
  VideoTrack? get activeVideoTrack;
  TrackPublication? get videoPublication;
  TrackPublication? get firstAudioPublication;
-  List<MediaDevice>? _audioOutputs;
-  MediaDevice? _selectedAudioDevice;

  @override
  void initState() {
    super.initState();
    widget.participant.addListener(_onParticipantChanged);
    _onParticipantChanged();
-    Hardware.instance.audioOutputs().then((value) {
-      setState(() {
-        _audioOutputs = value;
-        _selectedAudioDevice = _audioOutputs?.firstOrNull;
-      });
-    });
  }

  @override
@@ -116,14 +108,6 @@ abstract class _ParticipantWidgetState<T extends ParticipantWidget>
  // since the updated values are computed properties.
  void _onParticipantChanged() => setState(() {});

-  void _onSelectAudioOutput(MediaDevice device) {
-    var audioTrack = firstAudioPublication?.track as RemoteAudioTrack;
-    audioTrack.setAudioOutput(device.deviceId);
-    setState(() {
-      _selectedAudioDevice = device;
-    });
-  }
-
  // Widgets to show above the info bar
  List<Widget> extraWidgets(bool isScreenShare) => [];

@@ -146,8 +130,10 @@ abstract class _ParticipantWidgetState<T extends ParticipantWidget>
            InkWell(
              onTap: () => setState(() => _visible = !_visible),
              child: activeVideoTrack != null && !activeVideoTrack!.muted
-                  ? VideoTrackRenderer(activeVideoTrack!,
-                      fit: RTCVideoViewObjectFit.RTCVideoViewObjectFitContain)
+                  ? VideoTrackRenderer(
+                      activeVideoTrack!,
+                      fit: RTCVideoViewObjectFit.RTCVideoViewObjectFitContain,
+                    )
                  : const NoVideoWidget(),
            ),

@@ -158,9 +144,7 @@ abstract class _ParticipantWidgetState<T extends ParticipantWidget>
                crossAxisAlignment: CrossAxisAlignment.stretch,
                mainAxisSize: MainAxisSize.min,
                children: [
-                  ...extraWidgets(
-                    widget.isScreenShare,
-                  ),
+                  ...extraWidgets(widget.isScreenShare),
                  ParticipantInfoWidget(
                    title: widget.participant.name.isNotEmpty
                        ? '${widget.participant.name} (${widget.participant.identity})'
@@ -227,13 +211,6 @@ class _RemoteParticipantWidgetState
                pub: firstAudioPublication!,
                icon: EvaIcons.volumeUp,
              ),
-            if (lkPlatformIs(PlatformType.web))
-              RemoteTrackAudioOutputSelectMenuWidget(
-                audioOutputs: _audioOutputs ?? [],
-                selected: _selectedAudioDevice,
-                onSelected: _onSelectAudioOutput,
-                icon: EvaIcons.speaker,
-              ),
          ],
        ),
      ];
@@ -276,40 +253,3 @@ class RemoteTrackPublicationMenuWidget extends StatelessWidget {
        ),
      );
 }
-
-class RemoteTrackAudioOutputSelectMenuWidget extends StatelessWidget {
-  final IconData icon;
-  final List<MediaDevice> audioOutputs;
-  final MediaDevice? selected;
-  final Function(MediaDevice) onSelected;
-  const RemoteTrackAudioOutputSelectMenuWidget({
-    required this.audioOutputs,
-    required this.onSelected,
-    required this.selected,
-    required this.icon,
-    Key? key,
-  }) : super(key: key);
-
-  @override
-  Widget build(BuildContext context) => Material(
-        color: Colors.black.withOpacity(0.3),
-        child: PopupMenuButton<Function>(
-          tooltip: 'Select AudioOutput',
-          icon: Icon(icon),
-          onSelected: (value) => value(),
-          itemBuilder: (BuildContext context) => <PopupMenuEntry<Function>>[
-            ...audioOutputs
-                .map((e) => PopupMenuItem(
-                      child: ListTile(
-                        trailing: (e.deviceId == selected?.deviceId)
-                            ? const Icon(Icons.check, color: Colors.white)
-                            : null,
-                        title: Text(e.label),
-                      ),
-                      value: () => onSelected(e),
-                    ))
-                .toList(),
-          ],
-        ),
-      );
-}
@@ -2,6 +2,7 @@ import 'dart:async';

 import 'package:collection/collection.dart';
 import 'package:flutter/foundation.dart';
+import 'package:livekit_client/src/hardware/hardware.dart';
 import 'package:livekit_client/src/support/app_state.dart';
 import 'package:meta/meta.dart';

@@ -343,6 +344,7 @@ class Room extends DisposableChangeNotifier with EventsEmittable<RoomEvent> {
          event.stream,
          trackSid,
          receiver: event.receiver,
+          audioOutputOptions: roomOptions.defaultAudioOutputOptions,
        );
      } on TrackSubscriptionExceptionEvent catch (event) {
        logger.severe('addSubscribedMediaTrack() throwed ${event}');
@@ -640,3 +642,101 @@ extension RoomDebugMethods on Room {
        switchCandidate: switchCandidate);
  }
 }
+
+/// Room extension methods for managing audio, video.
+extension RoomHardwareManagementMethods on Room {
+  /// Get current audio output device.
+  String? get selectedAudioOutputDeviceId =>
+      roomOptions.defaultAudioOutputOptions.deviceId ??
+      Hardware.instance.selectedAudioOutput?.deviceId;
+
+  /// Get current audio input device.
+  String? get selectedAudioInputDeviceId =>
+      roomOptions.defaultAudioCaptureOptions.deviceId ??
+      Hardware.instance.selectedAudioInput?.deviceId;
+
+  /// Get current video input device.
+  String? get selectedVideoInputDeviceId =>
+      roomOptions.defaultCameraCaptureOptions.deviceId ??
+      Hardware.instance.selectedVideoInput?.deviceId;
+
+  /// Get mobile device's speaker status.
+  bool? get speakerOn => roomOptions.defaultAudioOutputOptions.speakerOn;
+
+  /// Set audio output device.
+  Future<void> setAudioOutputDevice(MediaDevice device) async {
+    if (lkPlatformIs(PlatformType.web)) {
+      participants.forEach((_, participant) {
+        for (var audioTrack in participant.audioTracks) {
+          audioTrack.track?.setSinkId(device.deviceId);
+        }
+      });
+      Hardware.instance.selectedAudioOutput = device;
+    } else {
+      await Hardware.instance.selectAudioOutput(device);
+    }
+    engine.roomOptions = engine.roomOptions.copyWith(
+      defaultAudioOutputOptions: roomOptions.defaultAudioOutputOptions.copyWith(
+        deviceId: device.deviceId,
+      ),
+    );
+  }
+
+  /// Set audio input device.
+  Future<void> setAudioInputDevice(MediaDevice device) async {
+    if (lkPlatformIs(PlatformType.web) && localParticipant != null) {
+      for (var audioTrack in localParticipant!.audioTracks) {
+        await audioTrack.track?.setDeviceId(device.deviceId);
+      }
+      Hardware.instance.selectedAudioInput = device;
+    } else {
+      await Hardware.instance.selectAudioInput(device);
+    }
+    engine.roomOptions = engine.roomOptions.copyWith(
+      defaultAudioCaptureOptions:
+          roomOptions.defaultAudioCaptureOptions.copyWith(
+        deviceId: device.deviceId,
+      ),
+    );
+  }
+
+  /// Set video input device.
+  Future<void> setVideoInputDevice(MediaDevice device) async {
+    final track = localParticipant?.videoTracks.firstOrNull?.track;
+    if (track == null) return;
+    if (selectedVideoInputDeviceId != device.deviceId) {
+      await track.switchCamera(device.deviceId);
+      Hardware.instance.selectedVideoInput = device;
+    }
+    engine.roomOptions = engine.roomOptions.copyWith(
+      defaultCameraCaptureOptions:
+          roomOptions.defaultCameraCaptureOptions.copyWith(
+        deviceId: device.deviceId,
+      ),
+    );
+  }
+
+  Future<void> setSpeakerOn(bool speakerOn) async {
+    if (lkPlatformIs(PlatformType.iOS) || lkPlatformIs(PlatformType.android)) {
+      await Hardware.instance.setSpeakerphoneOn(speakerOn);
+      engine.roomOptions = engine.roomOptions.copyWith(
+        defaultAudioOutputOptions:
+            roomOptions.defaultAudioOutputOptions.copyWith(
+          speakerOn: speakerOn,
+        ),
+      );
+    }
+  }
+
+  /// Apply audio output device settings.
+  @internal
+  Future<void> applyAudioSpeakerSettings() async {
+    if (roomOptions.defaultAudioOutputOptions.speakerOn != null) {
+      if (lkPlatformIs(PlatformType.iOS) ||
+          lkPlatformIs(PlatformType.android)) {
+        await Hardware.instance.setSpeakerphoneOn(
+            roomOptions.defaultAudioOutputOptions.speakerOn!);
+      }
+    }
+  }
+}
@@ -3,6 +3,8 @@ import 'dart:async';
 import 'package:collection/collection.dart';
 import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;

+import '../logger.dart';
+
 class MediaDevice {
  const MediaDevice(this.deviceId, this.label, this.kind);

@@ -40,6 +42,7 @@ class Hardware {
          devices.firstWhereOrNull((element) => element.kind == 'audiooutput');
      selectedVideoInput ??=
          devices.firstWhereOrNull((element) => element.kind == 'videoinput');
+      speakerOn = true;
    });
  }

@@ -54,6 +57,8 @@ class Hardware {

  MediaDevice? selectedVideoInput;

+  bool? speakerOn;
+
  Future<List<MediaDevice>> enumerateDevices({String? type}) async {
    var infos = await rtc.navigator.mediaDevices.enumerateDevices();
    var devices =
@@ -78,16 +83,18 @@ class Hardware {

  Future<void> selectAudioOutput(MediaDevice device) async {
    if (rtc.WebRTC.platformIsWeb) {
-      throw UnimplementedError('selectAudioOutput not support on web');
+      logger.warning('selectAudioOutput not support on web');
+      return;
    }
    selectedAudioOutput = device;
    await rtc.Helper.selectAudioOutput(device.deviceId);
  }

  Future<void> selectAudioInput(MediaDevice device) async {
-    if (rtc.WebRTC.platformIsWeb || rtc.WebRTC.platformIsIOS) {
-      throw UnimplementedError(
+    if (rtc.WebRTC.platformIsWeb) {
+      logger.warning(
          'selectAudioInput is only supported on Android/Windows/macOS');
+      return;
    }
    selectedAudioInput = device;
    await rtc.Helper.selectAudioInput(device.deviceId);
@@ -95,9 +102,10 @@ class Hardware {

  Future<void> setSpeakerphoneOn(bool enable) async {
    if (rtc.WebRTC.platformIsMobile) {
+      speakerOn = enable;
      await rtc.Helper.setSpeakerphoneOn(enable);
    } else {
-      throw UnimplementedError('setSpeakerphoneOn only support on iOS/Android');
+      logger.warning('setSpeakerphoneOn only support on iOS/Android');
    }
  }

@@ -67,6 +67,8 @@ class RoomOptions {
  /// Default options used when publishing a [LocalAudioTrack].
  final AudioPublishOptions defaultAudioPublishOptions;

+  final AudioOutputOptions defaultAudioOutputOptions;
+
  /// AdaptiveStream lets LiveKit automatically manage quality of subscribed
  /// video tracks to optimize for bandwidth and CPU.
  /// When attached video elements are visible, it'll choose an appropriate
@@ -92,10 +94,42 @@ class RoomOptions {
    this.defaultAudioCaptureOptions = const AudioCaptureOptions(),
    this.defaultVideoPublishOptions = const VideoPublishOptions(),
    this.defaultAudioPublishOptions = const AudioPublishOptions(),
+    this.defaultAudioOutputOptions = const AudioOutputOptions(),
    this.adaptiveStream = false,
    this.dynacast = false,
    this.stopLocalTrackOnUnpublish = true,
  });
+
+  RoomOptions copyWith({
+    CameraCaptureOptions? defaultCameraCaptureOptions,
+    ScreenShareCaptureOptions? defaultScreenShareCaptureOptions,
+    AudioCaptureOptions? defaultAudioCaptureOptions,
+    VideoPublishOptions? defaultVideoPublishOptions,
+    AudioPublishOptions? defaultAudioPublishOptions,
+    AudioOutputOptions? defaultAudioOutputOptions,
+    bool? adaptiveStream,
+    bool? dynacast,
+    bool? stopLocalTrackOnUnpublish,
+  }) {
+    return RoomOptions(
+      defaultCameraCaptureOptions:
+          defaultCameraCaptureOptions ?? this.defaultCameraCaptureOptions,
+      defaultScreenShareCaptureOptions: defaultScreenShareCaptureOptions ??
+          this.defaultScreenShareCaptureOptions,
+      defaultAudioCaptureOptions:
+          defaultAudioCaptureOptions ?? this.defaultAudioCaptureOptions,
+      defaultVideoPublishOptions:
+          defaultVideoPublishOptions ?? this.defaultVideoPublishOptions,
+      defaultAudioPublishOptions:
+          defaultAudioPublishOptions ?? this.defaultAudioPublishOptions,
+      defaultAudioOutputOptions:
+          defaultAudioOutputOptions ?? this.defaultAudioOutputOptions,
+      adaptiveStream: adaptiveStream ?? this.adaptiveStream,
+      dynacast: dynacast ?? this.dynacast,
+      stopLocalTrackOnUnpublish:
+          stopLocalTrackOnUnpublish ?? this.stopLocalTrackOnUnpublish,
+    );
+  }
 }

 /// Options used when publishing video.
@@ -88,6 +88,7 @@ class LocalParticipant extends Participant<LocalTrackPublication> {

    // did publish
    await track.onPublish();
+    await room.applyAudioSpeakerSettings();

    [events, room.events].emit(LocalTrackPublishedEvent(
      participant: this,
@@ -228,6 +229,7 @@ class LocalParticipant extends Participant<LocalTrackPublication> {

      // did unpublish
      await track.onUnpublish();
+      await room.applyAudioSpeakerSettings();
    }

    if (notify) {
@@ -319,6 +321,7 @@ class LocalParticipant extends Participant<LocalTrackPublication> {
          await publication.mute();
        }
      }
+      await room.applyAudioSpeakerSettings();
      return publication;
    } else if (enabled) {
      if (source == TrackSource.camera) {
@@ -8,6 +8,8 @@ import '../extensions.dart';
 import '../logger.dart';
 import '../proto/livekit_models.pb.dart' as lk_models;
 import '../publication/remote.dart';
+import '../support/platform.dart';
+import '../track/options.dart';
 import '../track/remote/audio.dart';
 import '../track/remote/remote.dart';
 import '../track/remote/video.dart';
@@ -75,6 +77,7 @@ class RemoteParticipant extends Participant<RemoteTrackPublication> {
    rtc.MediaStream stream,
    String trackSid, {
    rtc.RTCRtpReceiver? receiver,
+    AudioOutputOptions audioOutputOptions = const AudioOutputOptions(),
  }) async {
    logger.fine('addSubscribedMediaTrack()');

@@ -123,6 +126,16 @@ class RemoteParticipant extends Participant<RemoteTrackPublication> {
    }

    await track.start();
+
+    /// Apply audio output selection for the web.
+    if (pub.kind == lk_models.TrackType.AUDIO &&
+        lkPlatformIs(PlatformType.web)) {
+      if (audioOutputOptions.deviceId != null) {
+        await (track as RemoteAudioTrack)
+            .setSinkId(audioOutputOptions.deviceId!);
+      }
+    }
+
    await pub.updateTrack(track);
    await pub.updateSubscriptionAllowed(true);
    addTrackPublication(pub);
@@ -15,6 +15,16 @@ class LocalAudioTrack extends LocalTrack
  @override
  covariant AudioCaptureOptions currentOptions;

+  Future<void> setDeviceId(String deviceId) async {
+    if (currentOptions.deviceId == deviceId) {
+      return;
+    }
+    currentOptions = currentOptions.copyWith(deviceId: deviceId);
+    if (!muted) {
+      await restartTrack();
+    }
+  }
+
  // private constructor
  @internal
  LocalAudioTrack(
@@ -229,4 +229,40 @@ class AudioCaptureOptions extends LocalTrackOptions {
    }
    return constraints;
  }
+
+  AudioCaptureOptions copyWith({
+    String? deviceId,
+    bool? noiseSuppression,
+    bool? echoCancellation,
+    bool? autoGainControl,
+    bool? highPassFilter,
+    bool? typingNoiseDetection,
+  }) {
+    return AudioCaptureOptions(
+      deviceId: deviceId ?? this.deviceId,
+      noiseSuppression: noiseSuppression ?? this.noiseSuppression,
+      echoCancellation: echoCancellation ?? this.echoCancellation,
+      autoGainControl: autoGainControl ?? this.autoGainControl,
+      highPassFilter: highPassFilter ?? this.highPassFilter,
+      typingNoiseDetection: typingNoiseDetection ?? this.typingNoiseDetection,
+    );
+  }
+}
+
+class AudioOutputOptions {
+  /// The deviceId of the output device to use.
+  final String? deviceId;
+
+  /// If true, the audio will be played on the speaker.
+  /// for mobile only
+  final bool? speakerOn;
+
+  const AudioOutputOptions({this.deviceId, this.speakerOn});
+
+  AudioOutputOptions copyWith({String? deviceId, bool? speakerOn}) {
+    return AudioOutputOptions(
+      deviceId: deviceId ?? this.deviceId,
+      speakerOn: speakerOn ?? this.speakerOn,
+    );
+  }
 }
@@ -11,7 +11,7 @@ import '../web/_audio_api.dart' if (dart.library.html) '../web/_audio_html.dart'

 class RemoteAudioTrack extends RemoteTrack
    with AudioTrack, RemoteAudioManagementMixin {
-  //
+  String? _deviceId;
  RemoteAudioTrack(String name, TrackSource source, rtc.MediaStream stream,
      rtc.MediaStreamTrack track,
      {rtc.RTCRtpReceiver? receiver})
@@ -30,6 +30,9 @@ class RemoteAudioTrack extends RemoteTrack
    if (didStart) {
      // web support
      audio.startAudio(getCid(), mediaStreamTrack);
+      if (_deviceId != null) {
+        audio.setSinkId(getCid(), _deviceId!);
+      }
    }
    return didStart;
  }
@@ -44,7 +47,8 @@ class RemoteAudioTrack extends RemoteTrack
    return didStop;
  }

-  Future<void> setAudioOutput(String deviceId) async {
-    audio.setAudioOutput(getCid(), deviceId);
+  Future<void> setSinkId(String deviceId) async {
+    audio.setSinkId(getCid(), deviceId);
+    _deviceId = deviceId;
  }
 }
@@ -8,6 +8,6 @@ void stopAudio(String id) {
  // do nothing
 }

-void setAudioOutput(String id, String deviceId) {
+void setSinkId(String id, String deviceId) {
  // do nothing
 }
@@ -1,5 +1,5 @@
 import 'dart:html' as html;
-
+import 'dart:js_util' as jsutil;
 import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;

 // ignore: implementation_imports
@@ -52,9 +52,10 @@ html.DivElement findOrCreateAudioContainer() {
  return div as html.DivElement;
 }

-void setAudioOutput(String id, String deviceId) {
+void setSinkId(String id, String deviceId) {
  final audioElement = html.document.getElementById(audioPrefix + id);
-  if (audioElement is html.AudioElement) {
+  if (audioElement is html.AudioElement &&
+      jsutil.hasProperty(audioElement, 'setSinkId')) {
    audioElement.setSinkId(deviceId);
  }
 }