fix: audio play bug for ios safari. (#285)

* fix: audio play bug for ios safari. (WIP).

* chore: Repair and can pop up the playback prompt dialog box correctly.

* chore: add AudioContext to reduce playback delay.

example/lib/exts.dart

@@ -20,6 +20,25 @@ extension LKExampleExt on BuildContext {
         ),
       );
 
+  Future<bool?> showPlayAudioManuallyDialog() => showDialog<bool>(
+        context: this,
+        builder: (ctx) => AlertDialog(
+          title: const Text('Play Audio'),
+          content: const Text(
+              'You need to manually activate audio PlayBack for iOS Safari !'),
+          actions: [
+            TextButton(
+              onPressed: () => Navigator.pop(ctx, false),
+              child: const Text('Ignore'),
+            ),
+            TextButton(
+              onPressed: () => Navigator.pop(ctx, true),
+              child: const Text('Play Audio'),
+            ),
+          ],
+        ),
+      );
+
   Future<bool?> showUnPublishDialog() => showDialog<bool>(
         context: this,
         builder: (ctx) => AlertDialog(

example/lib/pages/room.dart

@@ -79,6 +79,15 @@ class _RoomPageState extends State<RoomPage> {
         print('Failed to decode: $_');
       }
       context.showDataReceivedDialog(decoded);
+    })
+    ..on<AudioPlaybackStatusChanged>((event) async {
+      if (!widget.room.canPlaybackAudio) {
+        print('Audio playback failed for iOS Safari ..........');
+        bool? yesno = await context.showPlayAudioManuallyDialog();
+        if (yesno == true) {
+          await widget.room.startAudio();
+        }
+      }
     });
 
   void _askPublish() async {

lib/src/core/room.dart

@@ -28,6 +28,9 @@ import '../track/track.dart';
 import '../types/other.dart';
 import 'engine.dart';
 
+import '../track/web/_audio_api.dart'
+    if (dart.library.html) '../track/web/_audio_html.dart' as audio;
+
 /// Room is the primary construct for LiveKit conferences. It contains a
 /// group of [Participant]s, each publishing and subscribing to [Track]s.
 /// Notifies changes to its state via two ways, by assigning a delegate, or using
@@ -78,6 +81,7 @@ class Room extends DisposableChangeNotifier with EventsEmittable<RoomEvent> {
   E2EEManager? _e2eeManager;
   bool get isRecording => _isRecording;
   bool _isRecording = false;
+  bool _audioEnabled = true;
 
   /// a list of participants that are actively speaking, including local participant.
   UnmodifiableListView<Participant> get activeSpeakers =>
@@ -352,6 +356,12 @@ class Room extends DisposableChangeNotifier with EventsEmittable<RoomEvent> {
     ..on<EngineActiveSpeakersUpdateEvent>(
         (event) => _onEngineActiveSpeakersUpdateEvent(event.speakers))
     ..on<EngineDataPacketReceivedEvent>(_onDataMessageEvent)
+    ..on<AudioPlaybackStarted>((event) {
+      _handleAudioPlaybackStarted();
+    })
+    ..on<AudioPlaybackFailed>((event) {
+      _handleAudioPlaybackFailed();
+    })
     ..on<EngineTrackAddedEvent>((event) async {
       logger.fine('EngineTrackAddedEvent trackSid:${event.track.id}');
 
@@ -775,4 +785,38 @@ extension RoomHardwareManagementMethods on Room {
       }
     }
   }
+
+  Future<void> startAudio() async {
+    try {
+      var audioContextRunning = await audio.startAllAudioElement();
+      if (audioContextRunning) {
+        _handleAudioPlaybackStarted();
+      } else {
+        _handleAudioPlaybackFailed();
+      }
+    } catch (err) {
+      logger.warning('could not playback audio $err');
+      _handleAudioPlaybackFailed();
+    }
+  }
+
+  bool get canPlaybackAudio {
+    return _audioEnabled;
+  }
+
+  void _handleAudioPlaybackStarted() {
+    if (canPlaybackAudio) {
+      return;
+    }
+    _audioEnabled = true;
+    events.emit(const AudioPlaybackStatusChanged(isPlaying: true));
+  }
+
+  void _handleAudioPlaybackFailed() {
+    if (!canPlaybackAudio) {
+      return;
+    }
+    _audioEnabled = false;
+    events.emit(const AudioPlaybackStatusChanged(isPlaying: false));
+  }
 }

lib/src/events.dart

@@ -424,3 +424,14 @@ class ParticipantNameUpdatedEvent with RoomEvent, ParticipantEvent {
   String toString() => '${runtimeType}'
       '(participant: ${participant}, name: ${name})';
 }
+
+class AudioPlaybackStatusChanged with RoomEvent {
+  final bool isPlaying;
+  const AudioPlaybackStatusChanged({
+    required this.isPlaying,
+  });
+
+  @override
+  String toString() => '${runtimeType}'
+      'Audio Playback Status Changed, isPlaying: ${isPlaying})';
+}

lib/src/internal/events.dart

@@ -61,6 +61,22 @@ class TrackStreamUpdatedEvent with TrackEvent, InternalEvent {
   });
 }
 
+@internal
+class AudioPlaybackStarted with TrackEvent, EngineEvent, InternalEvent {
+  final Track track;
+  const AudioPlaybackStarted({
+    required this.track,
+  });
+}
+
+@internal
+class AudioPlaybackFailed with TrackEvent, EngineEvent, InternalEvent {
+  final Track track;
+  const AudioPlaybackFailed({
+    required this.track,
+  });
+}
+
 @internal
 class LocalTrackOptionsUpdatedEvent with TrackEvent, InternalEvent {
   final LocalTrack track;

lib/src/participant/remote.dart

@@ -1,4 +1,5 @@
 import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
+import 'package:livekit_client/src/internal/events.dart';
 import 'package:meta/meta.dart';
 
 import '../core/room.dart';
@@ -121,6 +122,17 @@ class RemoteParticipant extends Participant<RemoteTrackPublication> {
       // audio track
       track = RemoteAudioTrack(pub.name, pub.source, stream, mediaTrack,
           receiver: receiver);
+
+      var listener = track.createListener();
+      listener.on<AudioPlaybackStarted>((event) {
+        logger.fine('AudioPlaybackStarted');
+        room.engine.events.emit(event);
+      });
+
+      listener.on<AudioPlaybackFailed>((event) {
+        logger.fine('AudioPlaybackFailed');
+        room.engine.events.emit(event);
+      });
     } else {
       throw UnexpectedStateException('Unknown track type');
     }

lib/src/track/remote/audio.dart

@@ -1,4 +1,5 @@
 import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
+import 'package:livekit_client/src/internal/events.dart';
 
 import '../../proto/livekit_models.pb.dart' as lk_models;
 import '../../types/other.dart';
@@ -28,10 +29,16 @@ class RemoteAudioTrack extends RemoteTrack
   Future<bool> start() async {
     final didStart = await super.start();
     if (didStart) {
-      // web support
+      try {
-      audio.startAudio(getCid(), mediaStreamTrack);
+        // web support
-      if (_deviceId != null) {
+        await audio.startAudio(getCid(), mediaStreamTrack);
-        audio.setSinkId(getCid(), _deviceId!);
+        if (_deviceId != null) {
+          audio.setSinkId(getCid(), _deviceId!);
+        }
+      } catch (e) {
+        if (e.toString().startsWith('NotAllowedError')) {
+          events.emit(AudioPlaybackFailed(track: this));
+        }
       }
     }
     return didStart;

lib/src/track/web/_audio_api.dart

@@ -1,6 +1,6 @@
 import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
 
-void startAudio(String id, rtc.MediaStreamTrack stream) {
+Future<dynamic> startAudio(String id, rtc.MediaStreamTrack stream) async {
   // do nothing
 }
 
@@ -8,6 +8,10 @@ void stopAudio(String id) {
   // do nothing
 }
 
+Future<bool> startAllAudioElement() async {
+  return true;
+}
+
 void setSinkId(String id, String deviceId) {
   // do nothing
 }

lib/src/track/web/_audio_html.dart

@@ -1,6 +1,7 @@
 import 'dart:html' as html;
 import 'dart:js_util' as jsutil;
 import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
+import 'package:js_bindings/js_bindings.dart' as js_bindings;
 
 // ignore: implementation_imports
 import 'package:dart_webrtc/src/media_stream_track_impl.dart'; // import_sorter: keep
@@ -8,10 +9,14 @@ import 'package:dart_webrtc/src/media_stream_track_impl.dart'; // import_sorter:
 const audioContainerId = 'livekit_audio_container';
 const audioPrefix = 'livekit_audio_';
 
-void startAudio(String id, rtc.MediaStreamTrack track) {
+js_bindings.AudioContext _audioContext = js_bindings.AudioContext();
+Map<String, html.Element> _audioElements = {};
+
+Future<dynamic> startAudio(String id, rtc.MediaStreamTrack track) async {
   if (track is! MediaStreamTrackWeb) {
     return;
   }
+
   final elementId = audioPrefix + id;
   var audioElement = html.document.getElementById(elementId);
   if (audioElement == null) {
@@ -19,6 +24,7 @@ void startAudio(String id, rtc.MediaStreamTrack track) {
       ..id = elementId
       ..autoplay = true;
     findOrCreateAudioContainer().append(audioElement);
+    _audioElements[id] = audioElement;
   }
 
   if (audioElement is! html.AudioElement) {
@@ -27,6 +33,16 @@ void startAudio(String id, rtc.MediaStreamTrack track) {
   final audioStream = html.MediaStream();
   audioStream.addTrack(track.jsTrack);
   audioElement.srcObject = audioStream;
+  return audioElement.play();
+}
+
+Future<bool> startAllAudioElement() async {
+  for (final element in _audioElements.values) {
+    if (element is html.AudioElement) {
+      await element.play();
+    }
+  }
+  return _audioContext.state == js_bindings.AudioContextState.running;
 }
 
 void stopAudio(String id) {
@@ -35,6 +51,7 @@ void stopAudio(String id) {
     if (audioElement is html.AudioElement) {
       audioElement.srcObject = null;
     }
+    _audioElements.remove(id);
     audioElement.remove();
   }
 }

pubspec.yaml

@@ -31,6 +31,7 @@ dependencies:
   dart_webrtc: 1.0.16
   js: ^0.6.4
   platform_detect: ^2.0.7
+  js_bindings: ^0.1.2+1
 
 dev_dependencies:
   flutter_test: