Better audio management (#10)

* add audio_session package

* `AudioManager` initial design

* try to integrate audio manager

* configure audio session

* `DisposeAware`

disposing if already published causes exception.

* organize

* guard flutter_webrtc calls

* websocket async fix

* use `ConnectionState` instead of `_isClosed` and `isReconnecting`

* change create audio track defaults

* update protos

* protocol 3 speaker updates

* fix exception

* emit `RoomDisconnectedEvent` only once

* fix exception

* keep track of local / remote audio tracks

* explicit types

* manage track state

* re-structure audio management

* change defaults

* unpublish all

* use experimental build

* change defaults

* configuring is optional

* call native `RTCAudioSession.setConfiguration`

* defaults adjustment

* iOS only for now

* use lib 92.4515.07

* clean up

* revert audio options for now

* remove pod source

* rename apple related audio

* `createListener` method

* organize native audio

* `SpeakingChangedEvent` only on `Participant`

* refactoring

* fix ios compile

* fix configure audio only for iOS logic

* minor fix & clean up

* change dispose logic

* format protos

* fix unpublishTrack

* `createListener` into a mixin

* simplify

* use flutter_webrtc master

* unpublish for example

* update ios icon

* android icon

* web icon

* favicon

lib/src/track/audio_track.dart

@@ -1,29 +1,174 @@
-import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
+// import 'package:audio_session/audio_session.dart' as _as;
+import 'dart:io';
 
+import 'package:flutter/foundation.dart';
+import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
+import 'package:synchronized/synchronized.dart' as sync;
+
+import '../logger.dart';
 import '../proto/livekit_models.pb.dart' as lk_models;
+import '../support/native_audio.dart';
 import '_audio_api.dart' if (dart.library.html) '_audio_html.dart' as audio;
 import 'local_audio_track.dart';
 import 'track.dart';
 
+enum AudioTrackState {
+  none,
+  remoteOnly,
+  localOnly,
+  localAndRemote,
+}
+
+typedef ConfigureNativeAudioFunc = Future<NativeAudioConfiguration> Function(AudioTrackState state);
+
 class AudioTrack extends Track {
+  // it's possible to set custom function here to customize audio session configuration
+  static ConfigureNativeAudioFunc nativeAudioConfigurationForAudioTrackState =
+      defaultNativeAudioConfigurationFunc;
+
+  static final _trackCounterLock = sync.Lock();
+  static AudioTrackState audioTrackState = AudioTrackState.none;
+  static int _localTrackCount = 0;
+  static int _remoteTrackCount = 0;
+
   rtc.MediaStream? mediaStream;
 
-  AudioTrack(String name, rtc.MediaStreamTrack track, this.mediaStream)
-      : super(lk_models.TrackType.AUDIO, name, track);
+  AudioTrack(
+    String name,
+    rtc.MediaStreamTrack track,
+    this.mediaStream,
+  ) : super(
+          lk_models.TrackType.AUDIO,
+          name,
+          track,
+        );
 
   /// Start playing audio track. On web platform, create an audio element and
   /// start playback
-  void start() {
-    if (this is! LocalAudioTrack) {
-      audio.startAudio(getCid(), mediaStreamTrack);
+  @override
+  Future<bool> start() async {
+    final didStart = await super.start();
+    if (didStart) {
+      if (this is! LocalAudioTrack) {
+        audio.startAudio(getCid(), mediaStreamTrack);
+      }
+
+      // update counter
+      await _trackCounterLock.synchronized(() async {
+        if (this is LocalAudioTrack) {
+          _localTrackCount++;
+        } else if (this is! LocalAudioTrack) {
+          _remoteTrackCount++;
+        }
+        await _onAudioTrackCountDidChange();
+      });
     }
+
+    return didStart;
   }
 
   @override
-  Future<void> stop() async {
-    await mediaStream?.dispose();
-    mediaStream = null;
-    audio.stopAudio(getCid());
-    await super.stop();
+  Future<bool> stop() async {
+    final didStop = await super.stop();
+    if (didStop) {
+      await mediaStream?.dispose();
+      mediaStream = null;
+      audio.stopAudio(getCid());
+
+      // update counter
+      await _trackCounterLock.synchronized(() async {
+        if (this is LocalAudioTrack) {
+          _localTrackCount--;
+        } else if (this is! LocalAudioTrack) {
+          _remoteTrackCount--;
+        }
+        await _onAudioTrackCountDidChange();
+      });
+    }
+
+    return didStop;
+  }
+
+  Future<void> _onAudioTrackCountDidChange() async {
+    logger.fine('[$runtimeType] onAudioTrackCountDidChange: '
+        'local: $_localTrackCount, remote: $_remoteTrackCount');
+
+    final newState = _computeAudioTrackState();
+
+    if (audioTrackState != newState) {
+      audioTrackState = newState;
+      logger.fine('[$runtimeType] didUpdateSate: $audioTrackState');
+
+      NativeAudioConfiguration? config;
+      if (!kIsWeb && Platform.isIOS) {
+        // Only iOS for now...
+        config = await nativeAudioConfigurationForAudioTrackState.call(audioTrackState);
+      }
+
+      if (config != null) {
+        logger.fine('[$runtimeType] configuring for ${audioTrackState} using ${config}...');
+        try {
+          await configureNativeAudio(config);
+        } catch (error) {
+          logger.warning('[$runtimeType] Failed to configure ${error}');
+        }
+      }
+    }
+  }
+
+  static AudioTrackState _computeAudioTrackState() {
+    if (_localTrackCount > 0 && _remoteTrackCount == 0) {
+      return AudioTrackState.localOnly;
+    } else if (_localTrackCount == 0 && _remoteTrackCount > 0) {
+      return AudioTrackState.remoteOnly;
+    } else if (_localTrackCount > 0 && _remoteTrackCount > 0) {
+      return AudioTrackState.localAndRemote;
+    }
+    // Default
+    return AudioTrackState.none;
   }
 }
+
+Future<NativeAudioConfiguration> defaultNativeAudioConfigurationFunc(AudioTrackState state) async {
+  //
+  if (state == AudioTrackState.remoteOnly) {
+    return NativeAudioConfiguration(
+      appleAudioCategory: AppleAudioCategory.playback,
+      appleAudioCategoryOptions: {
+        AppleAudioCategoryOption.mixWithOthers,
+        // IosAudioCategoryOption.duckOthers,
+      },
+      appleAudioMode: AppleAudioMode.spokenAudio,
+    );
+  } else if ([
+    AudioTrackState.localOnly,
+    AudioTrackState.localAndRemote,
+  ].contains(state)) {
+    return NativeAudioConfiguration(
+      appleAudioCategory: AppleAudioCategory.playAndRecord,
+      appleAudioCategoryOptions: {
+        AppleAudioCategoryOption.allowBluetooth,
+        AppleAudioCategoryOption.mixWithOthers,
+        // IosAudioCategoryOption.duckOthers,
+      },
+      appleAudioMode: AppleAudioMode.voiceChat,
+    );
+  }
+
+  // TODO: .record category causes exception in WebRTC lib for unknown reason
+  // if (this == AudioTrackState.localOnly) {
+  //   return NativeAudioConfiguration(
+  //     iosCategory: IosAudioCategory.record,
+  //     iosCategoryOptions: {
+  //       // IosAudioCategoryOption.allowBluetooth,
+  //     },
+  //     iosMode: IosAudioMode.spokenAudio,
+  //   );
+  // }
+
+  return NativeAudioConfiguration(
+    appleAudioCategory: AppleAudioCategory.soloAmbient,
+    appleAudioCategoryOptions: {},
+    appleAudioMode: AppleAudioMode.default_,
+  );
+}

lib/src/track/local_audio_track.dart

@@ -2,7 +2,7 @@ import 'dart:async';
 
 import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
 
-import '../errors.dart';
+import '../exceptions.dart';
 import 'audio_track.dart';
 import 'options.dart';
 
@@ -15,8 +15,12 @@ class LocalAudioTrack extends AudioTrack {
 
   /// Creates a new audio track from the default audio input device.
   static Future<LocalAudioTrack> create([LocalAudioTrackOptions? options]) async {
-    // try {
+    // TODO: have back up incase the options fail
     final stream = await rtc.navigator.mediaDevices.getUserMedia(<String, dynamic>{
+      // 'audio': <String, dynamic>{
+      // 'echoCancellation': true,
+      // 'noiseSuppression': true,
+      // },
       'audio': true,
       'video': false,
     });

lib/src/track/local_video_track.dart

@@ -1,6 +1,6 @@
 import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
 
-import '../errors.dart';
+import '../exceptions.dart';
 import '../logger.dart';
 import 'options.dart';
 import 'track.dart';

lib/src/track/remote_track_publication.dart

@@ -1,11 +1,9 @@
-import 'package:livekit_client/livekit_client.dart';
-
+import '../events.dart';
+import '../extensions.dart';
 import '../participant/remote_participant.dart';
 import '../proto/livekit_models.pb.dart' as lk_models;
 import '../proto/livekit_rtc.pb.dart' as lk_rtc;
 import 'track.dart';
-import '../extensions.dart';
-
 import 'track_publication.dart';
 
 /// Represents a track publication from a RemoteParticipant. Provides methods to

lib/src/track/track.dart

@@ -1,13 +1,17 @@
+import 'package:flutter/material.dart';
 import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
-import 'package:livekit_client/src/classes/change_notifier.dart';
+import 'package:meta/meta.dart';
 import 'package:uuid/uuid.dart';
 
+import '../extensions.dart';
+import '../logger.dart';
 import '../proto/livekit_models.pb.dart' as lk_models;
+import '../support/disposable.dart';
 
 /// Wrapper around a MediaStreamTrack with additional metadata.
 /// Base for [AudioTrack] and [VideoTrack],
 /// can not be instantiated directly.
-abstract class Track extends LKChangeNotifier {
+abstract class Track extends DisposableChangeNotifier {
   static const cameraName = 'camera';
   static const screenShareName = 'screen';
 
@@ -19,6 +23,10 @@ abstract class Track extends LKChangeNotifier {
   rtc.RTCRtpTransceiver? transceiver;
   String? _cid;
 
+  // started / stopped
+  bool _active = false;
+  bool get isActive => _active;
+
   Track(
     this.kind,
     this.name,
@@ -50,7 +58,33 @@ abstract class Track extends LKChangeNotifier {
     return cid;
   }
 
-  Future<void> stop() async {
-    await mediaStreamTrack.stop();
+  // returns true if started, false if already started
+  @mustCallSuper
+  Future<bool> start() async {
+    if (_active) {
+      // already started
+      return false;
+    }
+
+    _active = true;
+    return true;
+  }
+
+  // returns true if stopped, false if already stopped
+  @mustCallSuper
+  Future<bool> stop() async {
+    if (!_active) {
+      // already stopped
+      return false;
+    }
+
+    try {
+      await mediaStreamTrack.stop();
+    } catch (_) {
+      logger.warning('[$objectId] rtc.mediaStreamTrack.stop() did throw ${_}');
+    }
+
+    _active = false;
+    return true;
   }
 }

lib/src/track/track_publication.dart

@@ -1,3 +1,4 @@
+import '../support/disposable.dart';
 import '../proto/livekit_models.pb.dart' as lk_models;
 import '../types.dart';
 import 'track.dart';
@@ -8,9 +9,9 @@ import 'track.dart';
 /// Base for [RemoteTrackPublication] and [LocalTrackPublication],
 /// can not be instantiated directly.
 
-abstract class TrackPublication {
-  final String name;
+abstract class TrackPublication extends Disposable {
   final String sid;
+  final String name;
   final lk_models.TrackType kind;
 
   Track? track;

lib/src/track/video_track.dart

@@ -5,6 +5,7 @@ import 'track.dart';
 
 /// A video track will notify when its mediaTrack has changed.
 class VideoTrack extends Track {
+  //
   rtc.MediaStream _mediaStream;
 
   VideoTrack(
@@ -27,9 +28,12 @@ class VideoTrack extends Track {
   }
 
   @override
-  Future<void> stop() async {
-    await super.stop();
-    await _mediaStream.dispose();
+  Future<bool> stop() async {
+    final didStop = await super.stop();
+    if (didStop) {
+      await _mediaStream.dispose();
+    }
     // _mediaStream = null;
+    return didStop;
   }
 }