From 3a260443999d5642a25ec37d976c32e668ca727f Mon Sep 17 00:00:00 2001 From: David Zhao Date: Thu, 26 Aug 2021 22:58:04 -0700 Subject: [PATCH 1/2] manually handle audio tracks for web --- lib/src/livekit.dart | 7 +-- lib/src/participant/remote_participant.dart | 5 +- lib/src/signal_client.dart | 6 +-- lib/src/track/_audio_api.dart | 9 ++++ lib/src/track/_audio_html.dart | 53 +++++++++++++++++++++ lib/src/track/audio_track.dart | 27 +++++++++++ lib/src/track/local_audio_track.dart | 18 ++----- 7 files changed, 104 insertions(+), 21 deletions(-) create mode 100644 lib/src/track/_audio_api.dart create mode 100644 lib/src/track/_audio_html.dart create mode 100644 lib/src/track/audio_track.dart diff --git a/lib/src/livekit.dart b/lib/src/livekit.dart index 5d19f8d..40d46b2 100644 --- a/lib/src/livekit.dart +++ b/lib/src/livekit.dart @@ -1,9 +1,10 @@ import 'room.dart'; +import 'signal_client.dart'; class LiveKitClient { - // TODO: take in connect options - static Future connect(String url, String token) { + static Future connect(String url, String token, + [JoinOptions? options]) { var room = Room(); - return room.connect(url, token); + return room.connect(url, token, options); } } diff --git a/lib/src/participant/remote_participant.dart b/lib/src/participant/remote_participant.dart index 4a891bd..1e44a93 100644 --- a/lib/src/participant/remote_participant.dart +++ b/lib/src/participant/remote_participant.dart @@ -1,4 +1,5 @@ import 'package:flutter_webrtc/flutter_webrtc.dart'; +import 'package:livekit_client/src/track/audio_track.dart'; import '../proto/livekit_models.pb.dart'; import '../signal_client.dart'; import '../track/remote_track_publication.dart'; @@ -49,7 +50,9 @@ class RemoteParticipant extends Participant { Track? track; if (pub.kind == TrackType.AUDIO) { - track = new Track(pub.kind, pub.name, mediaTrack); + var audioTrack = new AudioTrack(pub.name, mediaTrack, stream); + audioTrack.start(); + track = audioTrack; } else if (pub.kind == TrackType.VIDEO) { track = new VideoTrack(pub.name, mediaTrack, stream); } else { diff --git a/lib/src/signal_client.dart b/lib/src/signal_client.dart index 7f959ea..89f383d 100644 --- a/lib/src/signal_client.dart +++ b/lib/src/signal_client.dart @@ -55,7 +55,7 @@ class SignalClient { Future join(String url, String token, JoinOptions? options) async { var rtcUrl = '$url/rtc'; - var params = _paramsForToken(token); + var params = _joinParams(token); if (options != null && options.autoSubscribe != null) { params += '&auto_subscribe=${options.autoSubscribe! ? '1' : '0'}'; } @@ -88,7 +88,7 @@ class SignalClient { _ws = null; url += '/rtc'; - var params = _paramsForToken(token); + var params = _joinParams(token); params += '&reconnect=1'; var uri = Uri.parse(url + params); @@ -239,7 +239,7 @@ class SignalClient { } } -String _paramsForToken(String token) { +String _joinParams(String token) { return '?access_token=$token&protocol=$protocolVersion'; } diff --git a/lib/src/track/_audio_api.dart b/lib/src/track/_audio_api.dart new file mode 100644 index 0000000..c6a2923 --- /dev/null +++ b/lib/src/track/_audio_api.dart @@ -0,0 +1,9 @@ +import 'package:flutter_webrtc/flutter_webrtc.dart'; + +void startAudio(String id, MediaStreamTrack stream) { + // do nothing +} + +void stopAudio(String id) { + // do nothing +} diff --git a/lib/src/track/_audio_html.dart b/lib/src/track/_audio_html.dart new file mode 100644 index 0000000..9d94de5 --- /dev/null +++ b/lib/src/track/_audio_html.dart @@ -0,0 +1,53 @@ +// ignore: avoid_web_libraries_in_flutter +import 'dart:html' as html; + +import 'package:flutter_webrtc/flutter_webrtc.dart'; +// ignore: implementation_imports +import 'package:flutter_webrtc/src/web/media_stream_track_impl.dart'; + +const audioContainerId = 'livekit_audio_container'; +const audioPrefix = 'livekit_audio_'; + +void startAudio(String id, MediaStreamTrack track) { + if (!(track is MediaStreamTrackWeb)) { + return; + } + var elementId = audioPrefix + id; + var audioElement = html.document.getElementById(elementId); + if (audioElement == null) { + audioElement = html.AudioElement() + ..id = elementId + ..autoplay = true; + findOrCreateAudioContainer().append(audioElement); + } + + if (!(audioElement is html.AudioElement)) { + return; + } + var audioStream = html.MediaStream(); + audioStream.addTrack(track.jsTrack); + audioElement.srcObject = audioStream; +} + +void stopAudio(String id) { + var audioElement = html.document.getElementById(audioPrefix + id); + if (audioElement != null) { + if (audioElement is html.AudioElement) { + audioElement.srcObject = null; + } + audioElement.remove(); + } +} + +html.DivElement findOrCreateAudioContainer() { + var div = html.document.getElementById(audioContainerId); + if (div != null) { + return div as html.DivElement; + } + + div = html.DivElement(); + div.id = audioContainerId; + div.style.display = 'none'; + html.document.body?.append(div); + return div as html.DivElement; +} diff --git a/lib/src/track/audio_track.dart b/lib/src/track/audio_track.dart new file mode 100644 index 0000000..80490c1 --- /dev/null +++ b/lib/src/track/audio_track.dart @@ -0,0 +1,27 @@ +import 'package:flutter_webrtc/flutter_webrtc.dart'; + +import '../proto/livekit_models.pbenum.dart'; +import 'local_audio_track.dart'; +import 'track.dart'; +import '_audio_api.dart' if (dart.library.html) '_audio_html.dart' as audio; + +class AudioTrack extends Track { + MediaStream? mediaStream; + + AudioTrack(String name, MediaStreamTrack track, this.mediaStream) + : super(TrackType.AUDIO, name, track); + + start() { + if (!(this is LocalAudioTrack)) { + audio.startAudio(getCid(), mediaTrack); + } + } + + @override + stop() { + mediaStream?.dispose(); + mediaStream = null; + audio.stopAudio(getCid()); + super.stop(); + } +} diff --git a/lib/src/track/local_audio_track.dart b/lib/src/track/local_audio_track.dart index d633160..5382c4d 100644 --- a/lib/src/track/local_audio_track.dart +++ b/lib/src/track/local_audio_track.dart @@ -1,15 +1,12 @@ import 'package:flutter_webrtc/flutter_webrtc.dart'; +import 'package:livekit_client/src/track/audio_track.dart'; import '../errors.dart'; -import '../proto/livekit_models.pb.dart'; import 'options.dart'; -import 'track.dart'; -class LocalAudioTrack extends Track { - MediaStream? mediaStream; - - LocalAudioTrack(String name, MediaStreamTrack track, this.mediaStream) - : super(TrackType.AUDIO, name, track); +class LocalAudioTrack extends AudioTrack { + LocalAudioTrack(String name, MediaStreamTrack track, MediaStream stream) + : super(name, track, stream); static Future createTrack( [LocalAudioTrackOptions? options]) async { @@ -28,11 +25,4 @@ class LocalAudioTrack extends Track { return Future.error(e); } } - - @override - stop() { - super.stop(); - mediaStream?.dispose(); - mediaStream = null; - } } From d40d1bb547bd66a331e25cde8bcddbe25988139e Mon Sep 17 00:00:00 2001 From: David Zhao Date: Fri, 27 Aug 2021 15:37:44 -0700 Subject: [PATCH 2/2] avoid disposing attached media stream --- example/lib/room.dart | 1 + lib/src/widget/video_track_renderer.dart | 3 +++ 2 files changed, 4 insertions(+) diff --git a/example/lib/room.dart b/example/lib/room.dart index f33fc36..d7b9667 100644 --- a/example/lib/room.dart +++ b/example/lib/room.dart @@ -155,6 +155,7 @@ class _RoomState extends State with RoomDelegate { } } +// displays a participant in view class VideoView extends StatefulWidget { final Participant participant; final VideoQuality quality; diff --git a/lib/src/widget/video_track_renderer.dart b/lib/src/widget/video_track_renderer.dart index 325f753..ceebccd 100644 --- a/lib/src/widget/video_track_renderer.dart +++ b/lib/src/widget/video_track_renderer.dart @@ -1,5 +1,6 @@ import 'package:flutter/material.dart'; import 'package:flutter_webrtc/flutter_webrtc.dart'; + import '../track/video_track.dart'; import '../track/local_video_track.dart'; @@ -30,6 +31,7 @@ class _VideoTrackRendererState extends State { @override void dispose() { widget.track.removeListener(_trackChanged); + _renderer.srcObject = null; _renderer.dispose(); super.dispose(); } @@ -38,6 +40,7 @@ class _VideoTrackRendererState extends State { void didUpdateWidget(covariant VideoTrackRenderer oldWidget) { oldWidget.track.removeListener(_trackChanged); widget.track.addListener(_trackChanged); + _trackChanged(); super.didUpdateWidget(oldWidget); }