diff --git a/example/lib/widgets/participant.dart b/example/lib/widgets/participant.dart index 3b3abd9..9234972 100644 --- a/example/lib/widgets/participant.dart +++ b/example/lib/widgets/participant.dart @@ -27,6 +27,7 @@ class _ParticipantWidgetState extends State { // TrackPublication? firstVideoPub; TrackPublication? firstAudioPub; + bool _visible = true; @override void initState() { @@ -79,14 +80,17 @@ class _ParticipantWidgetState extends State { child: Stack( children: [ // Video - if (firstVideoPub?.subscribed == true && - firstVideoPub?.muted == false) - VideoTrackRenderer( - firstVideoPub!.track as VideoTrack, - fit: RTCVideoViewObjectFit.RTCVideoViewObjectFitCover, - ) - else - const NoVideoWidget(), + InkWell( + onTap: () => setState(() => _visible = !_visible), + child: (firstVideoPub?.subscribed == true && + firstVideoPub?.muted == false && + _visible) + ? VideoTrackRenderer( + firstVideoPub!.track as VideoTrack, + fit: RTCVideoViewObjectFit.RTCVideoViewObjectFitCover, + ) + : const NoVideoWidget(), + ), Align( alignment: Alignment.bottomCenter, diff --git a/example/pubspec.lock b/example/pubspec.lock index 9bcba69..14b2b03 100644 --- a/example/pubspec.lock +++ b/example/pubspec.lock @@ -406,6 +406,13 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "2.1.0" + visibility_detector: + dependency: transitive + description: + name: visibility_detector + url: "https://pub.dartlang.org" + source: hosted + version: "0.2.2" win32: dependency: transitive description: diff --git a/example/pubspec.yaml b/example/pubspec.yaml index e03b2b1..3419096 100644 --- a/example/pubspec.yaml +++ b/example/pubspec.yaml @@ -21,6 +21,9 @@ dependencies: livekit_client: path: ../ + # git: + # url: https://github.com/livekit/client-sdk-flutter + # ref: main dev_dependencies: flutter_test: diff --git a/lib/src/internal/events.dart b/lib/src/internal/events.dart index c7b51b2..7e6b9c6 100644 --- a/lib/src/internal/events.dart +++ b/lib/src/internal/events.dart @@ -1,11 +1,15 @@ // added import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; +import 'package:visibility_detector/visibility_detector.dart'; import 'package:meta/meta.dart'; import '../events.dart'; +import '../track/track.dart'; + +abstract class InternalEvent implements LiveKitEvent {} @internal -abstract class EngineIceStateUpdatedEvent with EngineEvent { +abstract class EngineIceStateUpdatedEvent with EngineEvent, InternalEvent { final rtc.RTCIceConnectionState iceState; final bool isPrimary; const EngineIceStateUpdatedEvent({ @@ -35,3 +39,25 @@ class EnginePublisherIceStateUpdatedEvent extends EngineIceStateUpdatedEvent { isPrimary: isPrimary, ); } + +@internal +class TrackStreamUpdatedEvent with TrackEvent, InternalEvent { + final Track track; + final rtc.MediaStream stream; + const TrackStreamUpdatedEvent({ + required this.track, + required this.stream, + }); +} + +@internal +class TrackVisibilityUpdatedEvent with TrackEvent, InternalEvent { + final String rendererId; + final Track track; + final VisibilityInfo? info; // null means disposed + const TrackVisibilityUpdatedEvent({ + required this.rendererId, + required this.track, + required this.info, + }); +} diff --git a/lib/src/managers/event.dart b/lib/src/managers/event.dart index 7bc0c7e..5199a46 100644 --- a/lib/src/managers/event.dart +++ b/lib/src/managers/event.dart @@ -1,6 +1,7 @@ import 'dart:async'; import 'package:synchronized/synchronized.dart' as sync; +import 'package:meta/meta.dart'; import '../exceptions.dart'; import '../extensions.dart'; @@ -30,9 +31,13 @@ class EventsEmitter extends EventsListenable { @override EventsEmitter get emitter => this; + @internal void emit(T event) { // do nothing if already closed - if (streamCtrl.isClosed) return; + if (streamCtrl.isClosed) { + logger.warning('failed to emit event ${event} on a disposed emitter'); + return; + } // emit the event streamCtrl.add(event); } @@ -56,25 +61,31 @@ abstract class EventsListenable extends Disposable { // the emitter to listen to EventsEmitter get emitter; - bool synchronized; + final bool synchronized; // keep track of listeners to cancel later final _listeners = >[]; final _syncLock = sync.Lock(); + List> get listeners => _listeners; + EventsListenable({ required this.synchronized, }) { onDispose(() async { - if (_listeners.isNotEmpty) { - // Stop listening to all events - logger.fine('${objectId} cancelling ${_listeners.length} listeners(s)'); - for (final listener in _listeners) { - await listener.cancel(); - } - } + await cancelAll(); }); } + Future cancelAll() async { + if (_listeners.isNotEmpty) { + // Stop listening to all events + logger.fine('${objectId} cancelling ${_listeners.length} listeners(s)'); + for (final listener in _listeners) { + await listener.cancel(); + } + } + } + // @override // @mustCallSuper // Future dispose() async { diff --git a/lib/src/participant/local_participant.dart b/lib/src/participant/local_participant.dart index eeba840..9bc14c6 100644 --- a/lib/src/participant/local_participant.dart +++ b/lib/src/participant/local_participant.dart @@ -45,37 +45,29 @@ class LocalParticipant extends Participant { throw TrackPublishException('track already exists'); } - // await AudioManager().incrementPublishCounter(); + final trackInfo = await engine.addTrack( + cid: track.getCid(), + name: track.name, + kind: track.kind, + ); - try { - final trackInfo = await engine.addTrack( - cid: track.getCid(), - name: track.name, - kind: track.kind, - ); + await track.start(); - await track.start(); + final transceiverInit = rtc.RTCRtpTransceiverInit( + direction: rtc.TransceiverDirection.SendOnly, + ); + // addTransceiver cannot pass in a kind parameter due to a bug in flutter-webrtc (web) + track.transceiver = await engine.publisher?.pc.addTransceiver( + track: track.mediaStreamTrack, + kind: rtc.RTCRtpMediaType.RTCRtpMediaTypeAudio, + init: transceiverInit, + ); + await engine.negotiate(); - final transceiverInit = rtc.RTCRtpTransceiverInit( - direction: rtc.TransceiverDirection.SendOnly, - ); - // addTransceiver cannot pass in a kind parameter due to a bug in flutter-webrtc (web) - track.transceiver = await engine.publisher?.pc.addTransceiver( - track: track.mediaStreamTrack, - kind: rtc.RTCRtpMediaType.RTCRtpMediaTypeAudio, - init: transceiverInit, - ); - await engine.negotiate(); - - final pub = LocalTrackPublication(trackInfo, track, this); - addTrackPublication(pub); - notifyListeners(); - return pub; - } catch (e) { - // In any case there was an exception, revert the count. - // await AudioManager().decrementPublishCounter(); - rethrow; - } + final pub = LocalTrackPublication(trackInfo, track, this); + addTrackPublication(pub); + notifyListeners(); + return pub; } /// Publish a video track to the room diff --git a/lib/src/participant/remote_participant.dart b/lib/src/participant/remote_participant.dart index 87abda7..418572d 100644 --- a/lib/src/participant/remote_participant.dart +++ b/lib/src/participant/remote_participant.dart @@ -18,7 +18,6 @@ import 'participant.dart'; /// Represents other participant in the [Room]. class RemoteParticipant extends Participant { final SignalClient _client; - SignalClient get client => _client; RemoteParticipant( @@ -92,8 +91,6 @@ class RemoteParticipant extends Participant { final Track track; if (pub.kind == lk_models.TrackType.AUDIO) { // audio track - // await AudioManager().incrementSubscriptionCounter(); - final audioTrack = AudioTrack(pub.name, mediaTrack, stream); await audioTrack.start(); track = audioTrack; @@ -162,7 +159,7 @@ class RemoteParticipant extends Participant { if (pub is! RemoteTrackPublication) { // no publication exists for trackSid // or publication is not RemoteTrackPublication - // logger.warning('pub is not RemoteTrackPublication'); + await pub?.dispose(); return; } diff --git a/lib/src/rtc_engine.dart b/lib/src/rtc_engine.dart index 060e45d..29b073c 100644 --- a/lib/src/rtc_engine.dart +++ b/lib/src/rtc_engine.dart @@ -159,7 +159,7 @@ class RTCEngine extends Disposable with EventsEmittable { } _hasPublished = true; - publisher!.negotiate(); + publisher!.negotiate(null); } /* @internal */ diff --git a/lib/src/support/disposable.dart b/lib/src/support/disposable.dart index 1aaefe8..2b8497a 100644 --- a/lib/src/support/disposable.dart +++ b/lib/src/support/disposable.dart @@ -13,6 +13,7 @@ mixin _Disposer { final _disposeFuncs = []; bool _isDisposed = false; bool get isDisposed => _isDisposed; + int get disposeFuncCount => _disposeFuncs.length; // last added func will be called first when disposing void onDispose(OnDisposeFunc func) => _disposeFuncs.add(func); diff --git a/lib/src/track/local_track_publication.dart b/lib/src/track/local_track_publication.dart index 3ba6696..3f5bbfe 100644 --- a/lib/src/track/local_track_publication.dart +++ b/lib/src/track/local_track_publication.dart @@ -18,7 +18,7 @@ class LocalTrackPublication extends TrackPublication { // register dispose func onDispose(() async { // this object is responsible for disposing track - await track.dispose(); + await this.track?.dispose(); }); } diff --git a/lib/src/track/remote_track_publication.dart b/lib/src/track/remote_track_publication.dart index c098af8..785cfd8 100644 --- a/lib/src/track/remote_track_publication.dart +++ b/lib/src/track/remote_track_publication.dart @@ -1,3 +1,8 @@ +import 'dart:math'; +import 'dart:ui'; + +import 'package:collection/collection.dart'; +import 'package:livekit_client/src/internal/events.dart'; import 'package:livekit_client/src/logger.dart'; import 'package:meta/meta.dart'; @@ -6,9 +11,23 @@ import '../extensions.dart'; import '../participant/remote_participant.dart'; import '../proto/livekit_models.pb.dart' as lk_models; import '../proto/livekit_rtc.pb.dart' as lk_rtc; +import '../utils.dart'; import 'track.dart'; import 'track_publication.dart'; +class RendererVisibility { + final String rendererId; + final String trackId; + final bool visible; + final Size size; + RendererVisibility({ + required this.rendererId, + required this.trackId, + required this.visible, + required this.size, + }); +} + /// Represents a track publication from a RemoteParticipant. Provides methods to /// control if we should subscribe to the track, and its quality (for video). class RemoteTrackPublication extends TrackPublication { @@ -17,17 +36,124 @@ class RemoteTrackPublication extends TrackPublication { lk_rtc.VideoQuality _videoQuality = lk_rtc.VideoQuality.HIGH; lk_rtc.VideoQuality get videoQuality => _videoQuality; + // used to report renderer visibility to the server + // and optimize + final _visibilities = {}; + Function(void)? _visibilityDidUpdate; + Function? _cancelVisibilityDebounceFunc; + RemoteTrackPublication( lk_models.TrackInfo info, this._participant, [ Track? track, ]) : super.fromInfo(info) { - this.track = track; // register dispose func onDispose(() async { + _cancelVisibilityDebounceFunc?.call(); // this object is responsible for disposing track - await track?.dispose(); + await this.track?.dispose(); }); + + _visibilityDidUpdate = Utils.createDebounceFunc( + _shouldComputeVisibilityUpdate, + cancelFunc: (func) => _cancelVisibilityDebounceFunc = func, + wait: const Duration(seconds: 2), + ); + + this.track = track; + } + + // called any time visibility info updates + // from one of the renderers + void _onVideoRendererVisibilityUpdateEvent( + TrackVisibilityUpdatedEvent event) { + final info = event.info; + final trackSid = event.track.sid; + if (trackSid != null && info != null) { + logger.fine('[Visibility] ${event.rendererId} did update ' + 'track: ${event.track.sid} ' + 'visibleFraction: ${info.visibleFraction} ' + 'size: ${info.size}'); + _visibilities[event.rendererId] = RendererVisibility( + rendererId: event.rendererId, + trackId: trackSid, + visible: info.visibleFraction > 0, + size: info.size, + ); + + // quickly enable if currently disabled + if (!enabled && _hasVisibleRenderers()) { + logger.fine('[Visibility] Trying to re-enable quickly'); + _cancelVisibilityDebounceFunc?.call(); + _shouldComputeVisibilityUpdate(null); + } else { + _visibilityDidUpdate?.call(null); + } + } else { + // widget as been disposed, but track still exists + logger.fine('[Visibility] ${event.rendererId} was removed'); + _visibilities.remove(event.rendererId); + _visibilityDidUpdate?.call(null); + } + + logger.fine( + '[Visibility] Ids ${_visibilities.values.map((e) => e.rendererId)}'); + } + + bool _hasVisibleRenderers() => + _visibilities.values.firstWhereOrNull((e) => e.visible) != null; + + void _shouldComputeVisibilityUpdate(void _) { + // + Size maxSize(Size s1, Size s2) => Size( + max(s1.width, s2.width), + max(s1.height, s2.height), + ); + + _disabled = !_hasVisibleRenderers(); + + final settings = lk_rtc.UpdateTrackSettings( + trackSids: [sid], + disabled: _disabled, + ); + + if (!_disabled) { + final largest = _visibilities.values + .map((e) => e.size) + .reduce((value, element) => maxSize(value, element)); + settings.width = largest.width.floor(); + settings.height = largest.height.floor(); + } + + logger.fine('[Visibility] Sending to server ${settings.toProto3Json()}'); + _participant.client.sendUpdateTrackSettings(settings); + } + + @override + set track(Track? newValue) { + if (super.track != newValue) { + logger.fine('setTrack ${newValue} $sid ${objectId}'); + // dispose previous track (if exists) + super.track?.dispose(); + super.track = newValue; + + if (newValue != null) { + // + // Attach visibility event listener + // + final listener = newValue.createListener(); + listener.on( + _onVideoRendererVisibilityUpdateEvent); + newValue.onDispose(() async { + await listener.dispose(); + // consider all views are disposed when track is null + _visibilities.clear(); + // _visibilityDidUpdate?.call(null); + }); + } + + _visibilityDidUpdate?.call(null); + } } set videoQuality(lk_rtc.VideoQuality val) { diff --git a/lib/src/track/track.dart b/lib/src/track/track.dart index 0c0e3c6..689e16f 100644 --- a/lib/src/track/track.dart +++ b/lib/src/track/track.dart @@ -3,15 +3,19 @@ import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; import 'package:meta/meta.dart'; import 'package:uuid/uuid.dart'; +import '../events.dart'; import '../extensions.dart'; import '../logger.dart'; +import '../managers/event.dart'; import '../proto/livekit_models.pb.dart' as lk_models; import '../support/disposable.dart'; /// Wrapper around a MediaStreamTrack with additional metadata. /// Base for [AudioTrack] and [VideoTrack], /// can not be instantiated directly. -abstract class Track extends DisposableChangeNotifier { +abstract class Track extends DisposableChangeNotifier + with EventsEmittable { + static const uuid = Uuid(); static const cameraName = 'camera'; static const screenShareName = 'screen'; @@ -31,7 +35,13 @@ abstract class Track extends DisposableChangeNotifier { this.kind, this.name, this.mediaStreamTrack, - ); + ) { + onDispose(() async { + logger.fine('${objectId} onDispose()'); + // dispose events + await events.dispose(); + }); + } bool get muted => mediaStreamTrack.muted == null ? false : mediaStreamTrack.muted!; @@ -52,7 +62,6 @@ abstract class Track extends DisposableChangeNotifier { var cid = _cid ?? mediaStreamTrack.id; if (cid == null) { - const uuid = Uuid(); cid = uuid.v4(); _cid = cid; } diff --git a/lib/src/track/video_track.dart b/lib/src/track/video_track.dart index 12fe003..0677b9d 100644 --- a/lib/src/track/video_track.dart +++ b/lib/src/track/video_track.dart @@ -1,17 +1,18 @@ import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; +import 'package:meta/meta.dart'; import '../proto/livekit_models.pb.dart' as lk_models; import 'track.dart'; /// A video track will notify when its mediaTrack has changed. class VideoTrack extends Track { - // rtc.MediaStream _mediaStream; VideoTrack( String name, rtc.MediaStreamTrack mediaTrack, this._mediaStream, + // this._client, ) : super( lk_models.TrackType.VIDEO, name, @@ -22,6 +23,7 @@ class VideoTrack extends Track { /// internal use /// {@nodoc} + @internal void setMediaStream(rtc.MediaStream stream) { _mediaStream = stream; notifyListeners(); diff --git a/lib/src/transport.dart b/lib/src/transport.dart index 6eeb7ae..76c02b2 100644 --- a/lib/src/transport.dart +++ b/lib/src/transport.dart @@ -62,7 +62,7 @@ class PCTransport extends Disposable { } late final negotiate = Utils.createDebounceFunc( - () => createAndSendOffer(), + (void _) => createAndSendOffer(), cancelFunc: (f) => _cancelDebounce = f, wait: Timeouts.debounce, ); diff --git a/lib/src/utils.dart b/lib/src/utils.dart index 4dfcc15..d4d65f3 100644 --- a/lib/src/utils.dart +++ b/lib/src/utils.dart @@ -138,18 +138,18 @@ class Utils { ]; } - // makes a debounce func - static Function createDebounceFunc( - Function f, { + // makes a debounce func, with 1 param + static Function(T) createDebounceFunc( + Function(T) f, { Function(Function)? cancelFunc, required Duration wait, }) { Timer? t; - return () { + return (p) { t?.cancel(); t = Timer(wait, () { t = null; - f(); + f(p); }); // pass back the cancel method so we can cancel it when no longer needed cancelFunc?.call(t!.cancel); diff --git a/lib/src/widget/video_track_renderer.dart b/lib/src/widget/video_track_renderer.dart index 987bcfd..370ecac 100644 --- a/lib/src/widget/video_track_renderer.dart +++ b/lib/src/widget/video_track_renderer.dart @@ -1,21 +1,25 @@ import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; +import 'package:livekit_client/livekit_client.dart'; +import 'package:visibility_detector/visibility_detector.dart'; +import '../extensions.dart'; +import '../internal/events.dart'; +import '../logger.dart'; import '../track/local_video_track.dart'; import '../track/video_track.dart'; /// Widget that renders a [VideoTrack]. class VideoTrackRenderer extends StatefulWidget { final VideoTrack track; - final rtc.RTCVideoRenderer renderer; final rtc.RTCVideoViewObjectFit fit; - VideoTrackRenderer( + const VideoTrackRenderer( this.track, { this.fit = rtc.RTCVideoViewObjectFit.RTCVideoViewObjectFitContain, - }) : renderer = rtc.RTCVideoRenderer(), - super(key: ValueKey(track.sid)); + Key? key, + }) : super(key: key); @override State createState() => _VideoTrackRendererState(); @@ -23,49 +27,79 @@ class VideoTrackRenderer extends StatefulWidget { class _VideoTrackRendererState extends State { final _renderer = rtc.RTCVideoRenderer(); + bool _rendererReady = false; + EventsListener? _listener; + + Key get _keyForVisibilityDetector => + ValueKey('${objectId}-VisibilityDetector'); @override void initState() { super.initState(); - widget.track.addListener(_trackChanged); - _initRenderer(); + logger.fine('$objectId initState()'); + + (() async { + await _renderer.initialize(); + await _attach(); + setState(() => _rendererReady = true); + })(); } @override void dispose() { - widget.track.removeListener(_trackChanged); + logger.fine('$objectId dispose()'); + VisibilityDetectorController.instance.forget(_keyForVisibilityDetector); + // report that instance is disposing + // if the track is disposed first we can't emit event + widget.track.events.emit(TrackVisibilityUpdatedEvent( + rendererId: objectId, + track: widget.track, + info: null, + )); + _listener?.dispose(); _renderer.srcObject = null; _renderer.dispose(); super.dispose(); } + Future _attach() async { + logger.fine('[VideoTrackRenderer] attached to ${widget.track.objectId}'); + _renderer.srcObject = widget.track.mediaStream; + await _listener?.dispose(); + _listener = widget.track.createListener() + ..on((event) { + _renderer.srcObject = event.stream; + }); + } + @override void didUpdateWidget(covariant VideoTrackRenderer oldWidget) { - oldWidget.track.removeListener(_trackChanged); - widget.track.addListener(_trackChanged); - _trackChanged(); super.didUpdateWidget(oldWidget); - } - - void _trackChanged() { - setState(() { - _renderer.srcObject = widget.track.mediaStream; - }); - } - - void _initRenderer() async { - await _renderer.initialize(); - _trackChanged(); + if (widget.track != oldWidget.track) { + // TODO: re-attach only if needed + (() async { + await _attach(); + })(); + } } @override - Widget build(BuildContext context) { - final isLocal = widget.track is LocalVideoTrack; - return rtc.RTCVideoView( - _renderer, - mirror: isLocal, - filterQuality: FilterQuality.medium, - objectFit: widget.fit, - ); - } + Widget build(BuildContext context) => !_rendererReady + ? Container() + : VisibilityDetector( + key: _keyForVisibilityDetector, + // emit event when visibility updates + onVisibilityChanged: (VisibilityInfo info) => + widget.track.events.emit(TrackVisibilityUpdatedEvent( + rendererId: objectId, + track: widget.track, + info: info, + )), + child: rtc.RTCVideoView( + _renderer, + mirror: widget.track is LocalVideoTrack, + filterQuality: FilterQuality.medium, + objectFit: widget.fit, + ), + ); } diff --git a/pubspec.lock b/pubspec.lock index 8093964..6579a29 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -301,6 +301,13 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "2.1.0" + visibility_detector: + dependency: "direct main" + description: + name: visibility_detector + url: "https://pub.dartlang.org" + source: hosted + version: "0.2.2" win32: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index f3c2a6c..062d2e6 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -17,12 +17,12 @@ dependencies: collection: ^1.15.0 fixnum: ^1.0.0 meta: ^1.7.0 - http: ^0.13.3 logging: ^1.0.2 uuid: ^3.0.4 synchronized: ^3.0.0 protobuf: ^2.0.0 + visibility_detector: ^0.2.2 flutter_webrtc: ^0.7.0+hotfix.2 dev_dependencies: