From 0b8aeeeba117907b9ca7497d168722704367d58e Mon Sep 17 00:00:00 2001 From: CloudWebRTC Date: Tue, 16 May 2023 10:28:09 +0800 Subject: [PATCH] fix: audio play bug for ios safari. (#285) * fix: audio play bug for ios safari. (WIP). * chore: Repair and can pop up the playback prompt dialog box correctly. * chore: add AudioContext to reduce playback delay. --- example/lib/exts.dart | 19 +++++++++++++ example/lib/pages/room.dart | 9 ++++++ lib/src/core/room.dart | 44 ++++++++++++++++++++++++++++++ lib/src/events.dart | 11 ++++++++ lib/src/internal/events.dart | 16 +++++++++++ lib/src/participant/remote.dart | 12 ++++++++ lib/src/track/remote/audio.dart | 15 +++++++--- lib/src/track/web/_audio_api.dart | 6 +++- lib/src/track/web/_audio_html.dart | 19 ++++++++++++- pubspec.yaml | 1 + 10 files changed, 146 insertions(+), 6 deletions(-) diff --git a/example/lib/exts.dart b/example/lib/exts.dart index 74d68b2..8712a86 100644 --- a/example/lib/exts.dart +++ b/example/lib/exts.dart @@ -20,6 +20,25 @@ extension LKExampleExt on BuildContext { ), ); + Future showPlayAudioManuallyDialog() => showDialog( + context: this, + builder: (ctx) => AlertDialog( + title: const Text('Play Audio'), + content: const Text( + 'You need to manually activate audio PlayBack for iOS Safari !'), + actions: [ + TextButton( + onPressed: () => Navigator.pop(ctx, false), + child: const Text('Ignore'), + ), + TextButton( + onPressed: () => Navigator.pop(ctx, true), + child: const Text('Play Audio'), + ), + ], + ), + ); + Future showUnPublishDialog() => showDialog( context: this, builder: (ctx) => AlertDialog( diff --git a/example/lib/pages/room.dart b/example/lib/pages/room.dart index 7a7540d..3defac2 100644 --- a/example/lib/pages/room.dart +++ b/example/lib/pages/room.dart @@ -79,6 +79,15 @@ class _RoomPageState extends State { print('Failed to decode: $_'); } context.showDataReceivedDialog(decoded); + }) + ..on((event) async { + if (!widget.room.canPlaybackAudio) { + print('Audio playback failed for iOS Safari ..........'); + bool? yesno = await context.showPlayAudioManuallyDialog(); + if (yesno == true) { + await widget.room.startAudio(); + } + } }); void _askPublish() async { diff --git a/lib/src/core/room.dart b/lib/src/core/room.dart index bdd080b..171428e 100644 --- a/lib/src/core/room.dart +++ b/lib/src/core/room.dart @@ -28,6 +28,9 @@ import '../track/track.dart'; import '../types/other.dart'; import 'engine.dart'; +import '../track/web/_audio_api.dart' + if (dart.library.html) '../track/web/_audio_html.dart' as audio; + /// Room is the primary construct for LiveKit conferences. It contains a /// group of [Participant]s, each publishing and subscribing to [Track]s. /// Notifies changes to its state via two ways, by assigning a delegate, or using @@ -78,6 +81,7 @@ class Room extends DisposableChangeNotifier with EventsEmittable { E2EEManager? _e2eeManager; bool get isRecording => _isRecording; bool _isRecording = false; + bool _audioEnabled = true; /// a list of participants that are actively speaking, including local participant. UnmodifiableListView get activeSpeakers => @@ -352,6 +356,12 @@ class Room extends DisposableChangeNotifier with EventsEmittable { ..on( (event) => _onEngineActiveSpeakersUpdateEvent(event.speakers)) ..on(_onDataMessageEvent) + ..on((event) { + _handleAudioPlaybackStarted(); + }) + ..on((event) { + _handleAudioPlaybackFailed(); + }) ..on((event) async { logger.fine('EngineTrackAddedEvent trackSid:${event.track.id}'); @@ -775,4 +785,38 @@ extension RoomHardwareManagementMethods on Room { } } } + + Future startAudio() async { + try { + var audioContextRunning = await audio.startAllAudioElement(); + if (audioContextRunning) { + _handleAudioPlaybackStarted(); + } else { + _handleAudioPlaybackFailed(); + } + } catch (err) { + logger.warning('could not playback audio $err'); + _handleAudioPlaybackFailed(); + } + } + + bool get canPlaybackAudio { + return _audioEnabled; + } + + void _handleAudioPlaybackStarted() { + if (canPlaybackAudio) { + return; + } + _audioEnabled = true; + events.emit(const AudioPlaybackStatusChanged(isPlaying: true)); + } + + void _handleAudioPlaybackFailed() { + if (!canPlaybackAudio) { + return; + } + _audioEnabled = false; + events.emit(const AudioPlaybackStatusChanged(isPlaying: false)); + } } diff --git a/lib/src/events.dart b/lib/src/events.dart index db2fea1..ac2e4c7 100644 --- a/lib/src/events.dart +++ b/lib/src/events.dart @@ -424,3 +424,14 @@ class ParticipantNameUpdatedEvent with RoomEvent, ParticipantEvent { String toString() => '${runtimeType}' '(participant: ${participant}, name: ${name})'; } + +class AudioPlaybackStatusChanged with RoomEvent { + final bool isPlaying; + const AudioPlaybackStatusChanged({ + required this.isPlaying, + }); + + @override + String toString() => '${runtimeType}' + 'Audio Playback Status Changed, isPlaying: ${isPlaying})'; +} diff --git a/lib/src/internal/events.dart b/lib/src/internal/events.dart index 042849e..f36fc3a 100644 --- a/lib/src/internal/events.dart +++ b/lib/src/internal/events.dart @@ -61,6 +61,22 @@ class TrackStreamUpdatedEvent with TrackEvent, InternalEvent { }); } +@internal +class AudioPlaybackStarted with TrackEvent, EngineEvent, InternalEvent { + final Track track; + const AudioPlaybackStarted({ + required this.track, + }); +} + +@internal +class AudioPlaybackFailed with TrackEvent, EngineEvent, InternalEvent { + final Track track; + const AudioPlaybackFailed({ + required this.track, + }); +} + @internal class LocalTrackOptionsUpdatedEvent with TrackEvent, InternalEvent { final LocalTrack track; diff --git a/lib/src/participant/remote.dart b/lib/src/participant/remote.dart index 4d177b4..107ee5a 100644 --- a/lib/src/participant/remote.dart +++ b/lib/src/participant/remote.dart @@ -1,4 +1,5 @@ import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; +import 'package:livekit_client/src/internal/events.dart'; import 'package:meta/meta.dart'; import '../core/room.dart'; @@ -121,6 +122,17 @@ class RemoteParticipant extends Participant { // audio track track = RemoteAudioTrack(pub.name, pub.source, stream, mediaTrack, receiver: receiver); + + var listener = track.createListener(); + listener.on((event) { + logger.fine('AudioPlaybackStarted'); + room.engine.events.emit(event); + }); + + listener.on((event) { + logger.fine('AudioPlaybackFailed'); + room.engine.events.emit(event); + }); } else { throw UnexpectedStateException('Unknown track type'); } diff --git a/lib/src/track/remote/audio.dart b/lib/src/track/remote/audio.dart index ff67940..b286308 100644 --- a/lib/src/track/remote/audio.dart +++ b/lib/src/track/remote/audio.dart @@ -1,4 +1,5 @@ import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; +import 'package:livekit_client/src/internal/events.dart'; import '../../proto/livekit_models.pb.dart' as lk_models; import '../../types/other.dart'; @@ -28,10 +29,16 @@ class RemoteAudioTrack extends RemoteTrack Future start() async { final didStart = await super.start(); if (didStart) { - // web support - audio.startAudio(getCid(), mediaStreamTrack); - if (_deviceId != null) { - audio.setSinkId(getCid(), _deviceId!); + try { + // web support + await audio.startAudio(getCid(), mediaStreamTrack); + if (_deviceId != null) { + audio.setSinkId(getCid(), _deviceId!); + } + } catch (e) { + if (e.toString().startsWith('NotAllowedError')) { + events.emit(AudioPlaybackFailed(track: this)); + } } } return didStart; diff --git a/lib/src/track/web/_audio_api.dart b/lib/src/track/web/_audio_api.dart index b9c215a..c175387 100644 --- a/lib/src/track/web/_audio_api.dart +++ b/lib/src/track/web/_audio_api.dart @@ -1,6 +1,6 @@ import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; -void startAudio(String id, rtc.MediaStreamTrack stream) { +Future startAudio(String id, rtc.MediaStreamTrack stream) async { // do nothing } @@ -8,6 +8,10 @@ void stopAudio(String id) { // do nothing } +Future startAllAudioElement() async { + return true; +} + void setSinkId(String id, String deviceId) { // do nothing } diff --git a/lib/src/track/web/_audio_html.dart b/lib/src/track/web/_audio_html.dart index 09a73ad..021a7d1 100644 --- a/lib/src/track/web/_audio_html.dart +++ b/lib/src/track/web/_audio_html.dart @@ -1,6 +1,7 @@ import 'dart:html' as html; import 'dart:js_util' as jsutil; import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; +import 'package:js_bindings/js_bindings.dart' as js_bindings; // ignore: implementation_imports import 'package:dart_webrtc/src/media_stream_track_impl.dart'; // import_sorter: keep @@ -8,10 +9,14 @@ import 'package:dart_webrtc/src/media_stream_track_impl.dart'; // import_sorter: const audioContainerId = 'livekit_audio_container'; const audioPrefix = 'livekit_audio_'; -void startAudio(String id, rtc.MediaStreamTrack track) { +js_bindings.AudioContext _audioContext = js_bindings.AudioContext(); +Map _audioElements = {}; + +Future startAudio(String id, rtc.MediaStreamTrack track) async { if (track is! MediaStreamTrackWeb) { return; } + final elementId = audioPrefix + id; var audioElement = html.document.getElementById(elementId); if (audioElement == null) { @@ -19,6 +24,7 @@ void startAudio(String id, rtc.MediaStreamTrack track) { ..id = elementId ..autoplay = true; findOrCreateAudioContainer().append(audioElement); + _audioElements[id] = audioElement; } if (audioElement is! html.AudioElement) { @@ -27,6 +33,16 @@ void startAudio(String id, rtc.MediaStreamTrack track) { final audioStream = html.MediaStream(); audioStream.addTrack(track.jsTrack); audioElement.srcObject = audioStream; + return audioElement.play(); +} + +Future startAllAudioElement() async { + for (final element in _audioElements.values) { + if (element is html.AudioElement) { + await element.play(); + } + } + return _audioContext.state == js_bindings.AudioContextState.running; } void stopAudio(String id) { @@ -35,6 +51,7 @@ void stopAudio(String id) { if (audioElement is html.AudioElement) { audioElement.srcObject = null; } + _audioElements.remove(id); audioElement.remove(); } } diff --git a/pubspec.yaml b/pubspec.yaml index 94d7cb5..3567fde 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -31,6 +31,7 @@ dependencies: dart_webrtc: 1.0.16 js: ^0.6.4 platform_detect: ^2.0.7 + js_bindings: ^0.1.2+1 dev_dependencies: flutter_test: