feat: web/native device consistency management (#243)

* fix: Force audio settings to be consistent.

* chore: applyAudioSettings for onUnpublish.

* revert audio output select for web.

* add RoomHardwareManagementMethods to Room.

* update.

* fix analyze.

* update.

* update.

* chore: Reset sinkId for web when remoteTrack replays.

* rename setAudioOutput to setSinkId for flutter web.

* chore: Make sure AudioElement has `setSinkId` method.

lib/src/core/room.dart

@@ -2,6 +2,7 @@ import 'dart:async';
 
 import 'package:collection/collection.dart';
 import 'package:flutter/foundation.dart';
+import 'package:livekit_client/src/hardware/hardware.dart';
 import 'package:livekit_client/src/support/app_state.dart';
 import 'package:meta/meta.dart';
 
@@ -343,6 +344,7 @@ class Room extends DisposableChangeNotifier with EventsEmittable<RoomEvent> {
           event.stream,
           trackSid,
           receiver: event.receiver,
+          audioOutputOptions: roomOptions.defaultAudioOutputOptions,
         );
       } on TrackSubscriptionExceptionEvent catch (event) {
         logger.severe('addSubscribedMediaTrack() throwed ${event}');
@@ -640,3 +642,101 @@ extension RoomDebugMethods on Room {
         switchCandidate: switchCandidate);
   }
 }
+
+/// Room extension methods for managing audio, video.
+extension RoomHardwareManagementMethods on Room {
+  /// Get current audio output device.
+  String? get selectedAudioOutputDeviceId =>
+      roomOptions.defaultAudioOutputOptions.deviceId ??
+      Hardware.instance.selectedAudioOutput?.deviceId;
+
+  /// Get current audio input device.
+  String? get selectedAudioInputDeviceId =>
+      roomOptions.defaultAudioCaptureOptions.deviceId ??
+      Hardware.instance.selectedAudioInput?.deviceId;
+
+  /// Get current video input device.
+  String? get selectedVideoInputDeviceId =>
+      roomOptions.defaultCameraCaptureOptions.deviceId ??
+      Hardware.instance.selectedVideoInput?.deviceId;
+
+  /// Get mobile device's speaker status.
+  bool? get speakerOn => roomOptions.defaultAudioOutputOptions.speakerOn;
+
+  /// Set audio output device.
+  Future<void> setAudioOutputDevice(MediaDevice device) async {
+    if (lkPlatformIs(PlatformType.web)) {
+      participants.forEach((_, participant) {
+        for (var audioTrack in participant.audioTracks) {
+          audioTrack.track?.setSinkId(device.deviceId);
+        }
+      });
+      Hardware.instance.selectedAudioOutput = device;
+    } else {
+      await Hardware.instance.selectAudioOutput(device);
+    }
+    engine.roomOptions = engine.roomOptions.copyWith(
+      defaultAudioOutputOptions: roomOptions.defaultAudioOutputOptions.copyWith(
+        deviceId: device.deviceId,
+      ),
+    );
+  }
+
+  /// Set audio input device.
+  Future<void> setAudioInputDevice(MediaDevice device) async {
+    if (lkPlatformIs(PlatformType.web) && localParticipant != null) {
+      for (var audioTrack in localParticipant!.audioTracks) {
+        await audioTrack.track?.setDeviceId(device.deviceId);
+      }
+      Hardware.instance.selectedAudioInput = device;
+    } else {
+      await Hardware.instance.selectAudioInput(device);
+    }
+    engine.roomOptions = engine.roomOptions.copyWith(
+      defaultAudioCaptureOptions:
+          roomOptions.defaultAudioCaptureOptions.copyWith(
+        deviceId: device.deviceId,
+      ),
+    );
+  }
+
+  /// Set video input device.
+  Future<void> setVideoInputDevice(MediaDevice device) async {
+    final track = localParticipant?.videoTracks.firstOrNull?.track;
+    if (track == null) return;
+    if (selectedVideoInputDeviceId != device.deviceId) {
+      await track.switchCamera(device.deviceId);
+      Hardware.instance.selectedVideoInput = device;
+    }
+    engine.roomOptions = engine.roomOptions.copyWith(
+      defaultCameraCaptureOptions:
+          roomOptions.defaultCameraCaptureOptions.copyWith(
+        deviceId: device.deviceId,
+      ),
+    );
+  }
+
+  Future<void> setSpeakerOn(bool speakerOn) async {
+    if (lkPlatformIs(PlatformType.iOS) || lkPlatformIs(PlatformType.android)) {
+      await Hardware.instance.setSpeakerphoneOn(speakerOn);
+      engine.roomOptions = engine.roomOptions.copyWith(
+        defaultAudioOutputOptions:
+            roomOptions.defaultAudioOutputOptions.copyWith(
+          speakerOn: speakerOn,
+        ),
+      );
+    }
+  }
+
+  /// Apply audio output device settings.
+  @internal
+  Future<void> applyAudioSpeakerSettings() async {
+    if (roomOptions.defaultAudioOutputOptions.speakerOn != null) {
+      if (lkPlatformIs(PlatformType.iOS) ||
+          lkPlatformIs(PlatformType.android)) {
+        await Hardware.instance.setSpeakerphoneOn(
+            roomOptions.defaultAudioOutputOptions.speakerOn!);
+      }
+    }
+  }
+}

lib/src/hardware/hardware.dart

@@ -3,6 +3,8 @@ import 'dart:async';
 import 'package:collection/collection.dart';
 import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
 
+import '../logger.dart';
+
 class MediaDevice {
   const MediaDevice(this.deviceId, this.label, this.kind);
 
@@ -40,6 +42,7 @@ class Hardware {
           devices.firstWhereOrNull((element) => element.kind == 'audiooutput');
       selectedVideoInput ??=
           devices.firstWhereOrNull((element) => element.kind == 'videoinput');
+      speakerOn = true;
     });
   }
 
@@ -54,6 +57,8 @@ class Hardware {
 
   MediaDevice? selectedVideoInput;
 
+  bool? speakerOn;
+
   Future<List<MediaDevice>> enumerateDevices({String? type}) async {
     var infos = await rtc.navigator.mediaDevices.enumerateDevices();
     var devices =
@@ -78,16 +83,18 @@ class Hardware {
 
   Future<void> selectAudioOutput(MediaDevice device) async {
     if (rtc.WebRTC.platformIsWeb) {
-      throw UnimplementedError('selectAudioOutput not support on web');
+      logger.warning('selectAudioOutput not support on web');
+      return;
     }
     selectedAudioOutput = device;
     await rtc.Helper.selectAudioOutput(device.deviceId);
   }
 
   Future<void> selectAudioInput(MediaDevice device) async {
-    if (rtc.WebRTC.platformIsWeb || rtc.WebRTC.platformIsIOS) {
-      throw UnimplementedError(
+    if (rtc.WebRTC.platformIsWeb) {
+      logger.warning(
           'selectAudioInput is only supported on Android/Windows/macOS');
+      return;
     }
     selectedAudioInput = device;
     await rtc.Helper.selectAudioInput(device.deviceId);
@@ -95,9 +102,10 @@ class Hardware {
 
   Future<void> setSpeakerphoneOn(bool enable) async {
     if (rtc.WebRTC.platformIsMobile) {
+      speakerOn = enable;
       await rtc.Helper.setSpeakerphoneOn(enable);
     } else {
-      throw UnimplementedError('setSpeakerphoneOn only support on iOS/Android');
+      logger.warning('setSpeakerphoneOn only support on iOS/Android');
     }
   }
 

lib/src/options.dart

@@ -67,6 +67,8 @@ class RoomOptions {
   /// Default options used when publishing a [LocalAudioTrack].
   final AudioPublishOptions defaultAudioPublishOptions;
 
+  final AudioOutputOptions defaultAudioOutputOptions;
+
   /// AdaptiveStream lets LiveKit automatically manage quality of subscribed
   /// video tracks to optimize for bandwidth and CPU.
   /// When attached video elements are visible, it'll choose an appropriate
@@ -92,10 +94,42 @@ class RoomOptions {
     this.defaultAudioCaptureOptions = const AudioCaptureOptions(),
     this.defaultVideoPublishOptions = const VideoPublishOptions(),
     this.defaultAudioPublishOptions = const AudioPublishOptions(),
+    this.defaultAudioOutputOptions = const AudioOutputOptions(),
     this.adaptiveStream = false,
     this.dynacast = false,
     this.stopLocalTrackOnUnpublish = true,
   });
+
+  RoomOptions copyWith({
+    CameraCaptureOptions? defaultCameraCaptureOptions,
+    ScreenShareCaptureOptions? defaultScreenShareCaptureOptions,
+    AudioCaptureOptions? defaultAudioCaptureOptions,
+    VideoPublishOptions? defaultVideoPublishOptions,
+    AudioPublishOptions? defaultAudioPublishOptions,
+    AudioOutputOptions? defaultAudioOutputOptions,
+    bool? adaptiveStream,
+    bool? dynacast,
+    bool? stopLocalTrackOnUnpublish,
+  }) {
+    return RoomOptions(
+      defaultCameraCaptureOptions:
+          defaultCameraCaptureOptions ?? this.defaultCameraCaptureOptions,
+      defaultScreenShareCaptureOptions: defaultScreenShareCaptureOptions ??
+          this.defaultScreenShareCaptureOptions,
+      defaultAudioCaptureOptions:
+          defaultAudioCaptureOptions ?? this.defaultAudioCaptureOptions,
+      defaultVideoPublishOptions:
+          defaultVideoPublishOptions ?? this.defaultVideoPublishOptions,
+      defaultAudioPublishOptions:
+          defaultAudioPublishOptions ?? this.defaultAudioPublishOptions,
+      defaultAudioOutputOptions:
+          defaultAudioOutputOptions ?? this.defaultAudioOutputOptions,
+      adaptiveStream: adaptiveStream ?? this.adaptiveStream,
+      dynacast: dynacast ?? this.dynacast,
+      stopLocalTrackOnUnpublish:
+          stopLocalTrackOnUnpublish ?? this.stopLocalTrackOnUnpublish,
+    );
+  }
 }
 
 /// Options used when publishing video.

lib/src/participant/local.dart

@@ -88,6 +88,7 @@ class LocalParticipant extends Participant<LocalTrackPublication> {
 
     // did publish
     await track.onPublish();
+    await room.applyAudioSpeakerSettings();
 
     [events, room.events].emit(LocalTrackPublishedEvent(
       participant: this,
@@ -228,6 +229,7 @@ class LocalParticipant extends Participant<LocalTrackPublication> {
 
       // did unpublish
       await track.onUnpublish();
+      await room.applyAudioSpeakerSettings();
     }
 
     if (notify) {
@@ -319,6 +321,7 @@ class LocalParticipant extends Participant<LocalTrackPublication> {
           await publication.mute();
         }
       }
+      await room.applyAudioSpeakerSettings();
       return publication;
     } else if (enabled) {
       if (source == TrackSource.camera) {

lib/src/participant/remote.dart

@@ -8,6 +8,8 @@ import '../extensions.dart';
 import '../logger.dart';
 import '../proto/livekit_models.pb.dart' as lk_models;
 import '../publication/remote.dart';
+import '../support/platform.dart';
+import '../track/options.dart';
 import '../track/remote/audio.dart';
 import '../track/remote/remote.dart';
 import '../track/remote/video.dart';
@@ -75,6 +77,7 @@ class RemoteParticipant extends Participant<RemoteTrackPublication> {
     rtc.MediaStream stream,
     String trackSid, {
     rtc.RTCRtpReceiver? receiver,
+    AudioOutputOptions audioOutputOptions = const AudioOutputOptions(),
   }) async {
     logger.fine('addSubscribedMediaTrack()');
 
@@ -123,6 +126,16 @@ class RemoteParticipant extends Participant<RemoteTrackPublication> {
     }
 
     await track.start();
+
+    /// Apply audio output selection for the web.
+    if (pub.kind == lk_models.TrackType.AUDIO &&
+        lkPlatformIs(PlatformType.web)) {
+      if (audioOutputOptions.deviceId != null) {
+        await (track as RemoteAudioTrack)
+            .setSinkId(audioOutputOptions.deviceId!);
+      }
+    }
+
     await pub.updateTrack(track);
     await pub.updateSubscriptionAllowed(true);
     addTrackPublication(pub);

lib/src/track/local/audio.dart

@@ -15,6 +15,16 @@ class LocalAudioTrack extends LocalTrack
   @override
   covariant AudioCaptureOptions currentOptions;
 
+  Future<void> setDeviceId(String deviceId) async {
+    if (currentOptions.deviceId == deviceId) {
+      return;
+    }
+    currentOptions = currentOptions.copyWith(deviceId: deviceId);
+    if (!muted) {
+      await restartTrack();
+    }
+  }
+
   // private constructor
   @internal
   LocalAudioTrack(

lib/src/track/options.dart

@@ -229,4 +229,40 @@ class AudioCaptureOptions extends LocalTrackOptions {
     }
     return constraints;
   }
+
+  AudioCaptureOptions copyWith({
+    String? deviceId,
+    bool? noiseSuppression,
+    bool? echoCancellation,
+    bool? autoGainControl,
+    bool? highPassFilter,
+    bool? typingNoiseDetection,
+  }) {
+    return AudioCaptureOptions(
+      deviceId: deviceId ?? this.deviceId,
+      noiseSuppression: noiseSuppression ?? this.noiseSuppression,
+      echoCancellation: echoCancellation ?? this.echoCancellation,
+      autoGainControl: autoGainControl ?? this.autoGainControl,
+      highPassFilter: highPassFilter ?? this.highPassFilter,
+      typingNoiseDetection: typingNoiseDetection ?? this.typingNoiseDetection,
+    );
+  }
+}
+
+class AudioOutputOptions {
+  /// The deviceId of the output device to use.
+  final String? deviceId;
+
+  /// If true, the audio will be played on the speaker.
+  /// for mobile only
+  final bool? speakerOn;
+
+  const AudioOutputOptions({this.deviceId, this.speakerOn});
+
+  AudioOutputOptions copyWith({String? deviceId, bool? speakerOn}) {
+    return AudioOutputOptions(
+      deviceId: deviceId ?? this.deviceId,
+      speakerOn: speakerOn ?? this.speakerOn,
+    );
+  }
 }

lib/src/track/remote/audio.dart

@@ -11,7 +11,7 @@ import '../web/_audio_api.dart' if (dart.library.html) '../web/_audio_html.dart'
 
 class RemoteAudioTrack extends RemoteTrack
     with AudioTrack, RemoteAudioManagementMixin {
-  //
+  String? _deviceId;
   RemoteAudioTrack(String name, TrackSource source, rtc.MediaStream stream,
       rtc.MediaStreamTrack track,
       {rtc.RTCRtpReceiver? receiver})
@@ -30,6 +30,9 @@ class RemoteAudioTrack extends RemoteTrack
     if (didStart) {
       // web support
       audio.startAudio(getCid(), mediaStreamTrack);
+      if (_deviceId != null) {
+        audio.setSinkId(getCid(), _deviceId!);
+      }
     }
     return didStart;
   }
@@ -44,7 +47,8 @@ class RemoteAudioTrack extends RemoteTrack
     return didStop;
   }
 
-  Future<void> setAudioOutput(String deviceId) async {
-    audio.setAudioOutput(getCid(), deviceId);
+  Future<void> setSinkId(String deviceId) async {
+    audio.setSinkId(getCid(), deviceId);
+    _deviceId = deviceId;
   }
 }

lib/src/track/web/_audio_api.dart

@@ -8,6 +8,6 @@ void stopAudio(String id) {
   // do nothing
 }
 
-void setAudioOutput(String id, String deviceId) {
+void setSinkId(String id, String deviceId) {
   // do nothing
 }

lib/src/track/web/_audio_html.dart

@@ -1,5 +1,5 @@
 import 'dart:html' as html;
-
+import 'dart:js_util' as jsutil;
 import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
 
 // ignore: implementation_imports
@@ -52,9 +52,10 @@ html.DivElement findOrCreateAudioContainer() {
   return div as html.DivElement;
 }
 
-void setAudioOutput(String id, String deviceId) {
+void setSinkId(String id, String deviceId) {
   final audioElement = html.document.getElementById(audioPrefix + id);
-  if (audioElement is html.AudioElement) {
+  if (audioElement is html.AudioElement &&
+      jsutil.hasProperty(audioElement, 'setSinkId')) {
     audioElement.setSinkId(deviceId);
   }
 }