Video optimizations (#18)

* use events for `VideoTrackRenderer`

* logger

* implemented visibility detection

* optimize code with 1 param debounce func

* stop pending visibility event when disposed

* clean up

* mark internal

* report to track

* call dispose on pub and track

* use internal events to receive visibility events

* clean up visibility for disposed track

* listen to track visibility events on pub

* re-design

* quickly re-enable

* clean up

* clean up

example/lib/widgets/participant.dart

@@ -27,6 +27,7 @@ class _ParticipantWidgetState extends State<ParticipantWidget> {
   //
   TrackPublication? firstVideoPub;
   TrackPublication? firstAudioPub;
+  bool _visible = true;
 
   @override
   void initState() {
@@ -79,14 +80,17 @@ class _ParticipantWidgetState extends State<ParticipantWidget> {
         child: Stack(
           children: [
             // Video
-            if (firstVideoPub?.subscribed == true &&
+            InkWell(
-                firstVideoPub?.muted == false)
+              onTap: () => setState(() => _visible = !_visible),
-              VideoTrackRenderer(
+              child: (firstVideoPub?.subscribed == true &&
-                firstVideoPub!.track as VideoTrack,
+                      firstVideoPub?.muted == false &&
-                fit: RTCVideoViewObjectFit.RTCVideoViewObjectFitCover,
+                      _visible)
-              )
+                  ? VideoTrackRenderer(
-            else
+                      firstVideoPub!.track as VideoTrack,
-              const NoVideoWidget(),
+                      fit: RTCVideoViewObjectFit.RTCVideoViewObjectFitCover,
+                    )
+                  : const NoVideoWidget(),
+            ),
 
             Align(
               alignment: Alignment.bottomCenter,

example/pubspec.lock

@@ -406,6 +406,13 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "2.1.0"
+  visibility_detector:
+    dependency: transitive
+    description:
+      name: visibility_detector
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "0.2.2"
   win32:
     dependency: transitive
     description:

example/pubspec.yaml

@@ -21,6 +21,9 @@ dependencies:
 
   livekit_client:
     path: ../
+    # git:
+    #   url: https://github.com/livekit/client-sdk-flutter
+    #   ref: main
 
 dev_dependencies:
   flutter_test:

lib/src/internal/events.dart

@@ -1,11 +1,15 @@
 // added
 import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
+import 'package:visibility_detector/visibility_detector.dart';
 import 'package:meta/meta.dart';
 
 import '../events.dart';
+import '../track/track.dart';
+
+abstract class InternalEvent implements LiveKitEvent {}
 
 @internal
-abstract class EngineIceStateUpdatedEvent with EngineEvent {
+abstract class EngineIceStateUpdatedEvent with EngineEvent, InternalEvent {
   final rtc.RTCIceConnectionState iceState;
   final bool isPrimary;
   const EngineIceStateUpdatedEvent({
@@ -35,3 +39,25 @@ class EnginePublisherIceStateUpdatedEvent extends EngineIceStateUpdatedEvent {
           isPrimary: isPrimary,
         );
 }
+
+@internal
+class TrackStreamUpdatedEvent with TrackEvent, InternalEvent {
+  final Track track;
+  final rtc.MediaStream stream;
+  const TrackStreamUpdatedEvent({
+    required this.track,
+    required this.stream,
+  });
+}
+
+@internal
+class TrackVisibilityUpdatedEvent with TrackEvent, InternalEvent {
+  final String rendererId;
+  final Track track;
+  final VisibilityInfo? info; // null means disposed
+  const TrackVisibilityUpdatedEvent({
+    required this.rendererId,
+    required this.track,
+    required this.info,
+  });
+}

lib/src/managers/event.dart

@@ -1,6 +1,7 @@
 import 'dart:async';
 
 import 'package:synchronized/synchronized.dart' as sync;
+import 'package:meta/meta.dart';
 
 import '../exceptions.dart';
 import '../extensions.dart';
@@ -30,9 +31,13 @@ class EventsEmitter<T> extends EventsListenable<T> {
   @override
   EventsEmitter<T> get emitter => this;
 
+  @internal
   void emit(T event) {
     // do nothing if already closed
-    if (streamCtrl.isClosed) return;
+    if (streamCtrl.isClosed) {
+      logger.warning('failed to emit event ${event} on a disposed emitter');
+      return;
+    }
     // emit the event
     streamCtrl.add(event);
   }
@@ -56,25 +61,31 @@ abstract class EventsListenable<T> extends Disposable {
   // the emitter to listen to
   EventsEmitter<T> get emitter;
 
-  bool synchronized;
+  final bool synchronized;
   // keep track of listeners to cancel later
   final _listeners = <StreamSubscription<T>>[];
   final _syncLock = sync.Lock();
 
+  List<StreamSubscription<T>> get listeners => _listeners;
+
   EventsListenable({
     required this.synchronized,
   }) {
     onDispose(() async {
-      if (_listeners.isNotEmpty) {
+      await cancelAll();
-        // Stop listening to all events
-        logger.fine('${objectId} cancelling ${_listeners.length} listeners(s)');
-        for (final listener in _listeners) {
-          await listener.cancel();
-        }
-      }
     });
   }
 
+  Future<void> cancelAll() async {
+    if (_listeners.isNotEmpty) {
+      // Stop listening to all events
+      logger.fine('${objectId} cancelling ${_listeners.length} listeners(s)');
+      for (final listener in _listeners) {
+        await listener.cancel();
+      }
+    }
+  }
+
   // @override
   // @mustCallSuper
   // Future<bool> dispose() async {

lib/src/participant/local_participant.dart

@@ -45,37 +45,29 @@ class LocalParticipant extends Participant {
       throw TrackPublishException('track already exists');
     }
 
-    // await AudioManager().incrementPublishCounter();
+    final trackInfo = await engine.addTrack(
+      cid: track.getCid(),
+      name: track.name,
+      kind: track.kind,
+    );
 
-    try {
+    await track.start();
-      final trackInfo = await engine.addTrack(
-        cid: track.getCid(),
-        name: track.name,
-        kind: track.kind,
-      );
 
-      await track.start();
+    final transceiverInit = rtc.RTCRtpTransceiverInit(
+      direction: rtc.TransceiverDirection.SendOnly,
+    );
+    // addTransceiver cannot pass in a kind parameter due to a bug in flutter-webrtc (web)
+    track.transceiver = await engine.publisher?.pc.addTransceiver(
+      track: track.mediaStreamTrack,
+      kind: rtc.RTCRtpMediaType.RTCRtpMediaTypeAudio,
+      init: transceiverInit,
+    );
+    await engine.negotiate();
 
-      final transceiverInit = rtc.RTCRtpTransceiverInit(
+    final pub = LocalTrackPublication(trackInfo, track, this);
-        direction: rtc.TransceiverDirection.SendOnly,
+    addTrackPublication(pub);
-      );
+    notifyListeners();
-      // addTransceiver cannot pass in a kind parameter due to a bug in flutter-webrtc (web)
+    return pub;
-      track.transceiver = await engine.publisher?.pc.addTransceiver(
-        track: track.mediaStreamTrack,
-        kind: rtc.RTCRtpMediaType.RTCRtpMediaTypeAudio,
-        init: transceiverInit,
-      );
-      await engine.negotiate();
-
-      final pub = LocalTrackPublication(trackInfo, track, this);
-      addTrackPublication(pub);
-      notifyListeners();
-      return pub;
-    } catch (e) {
-      // In any case there was an exception, revert the count.
-      // await AudioManager().decrementPublishCounter();
-      rethrow;
-    }
   }
 
   /// Publish a video track to the room

lib/src/participant/remote_participant.dart

@@ -18,7 +18,6 @@ import 'participant.dart';
 /// Represents other participant in the [Room].
 class RemoteParticipant extends Participant {
   final SignalClient _client;
-
   SignalClient get client => _client;
 
   RemoteParticipant(
@@ -92,8 +91,6 @@ class RemoteParticipant extends Participant {
     final Track track;
     if (pub.kind == lk_models.TrackType.AUDIO) {
       // audio track
-      // await AudioManager().incrementSubscriptionCounter();
-
       final audioTrack = AudioTrack(pub.name, mediaTrack, stream);
       await audioTrack.start();
       track = audioTrack;
@@ -162,7 +159,7 @@ class RemoteParticipant extends Participant {
     if (pub is! RemoteTrackPublication) {
       // no publication exists for trackSid
       // or publication is not RemoteTrackPublication
-      // logger.warning('pub is not RemoteTrackPublication');
+
       await pub?.dispose();
       return;
     }

lib/src/rtc_engine.dart

@@ -159,7 +159,7 @@ class RTCEngine extends Disposable with EventsEmittable<EngineEvent> {
     }
 
     _hasPublished = true;
-    publisher!.negotiate();
+    publisher!.negotiate(null);
   }
 
   /* @internal */

lib/src/support/disposable.dart

@@ -13,6 +13,7 @@ mixin _Disposer {
   final _disposeFuncs = <OnDisposeFunc>[];
   bool _isDisposed = false;
   bool get isDisposed => _isDisposed;
+  int get disposeFuncCount => _disposeFuncs.length;
 
   // last added func will be called first when disposing
   void onDispose(OnDisposeFunc func) => _disposeFuncs.add(func);

lib/src/track/local_track_publication.dart

@@ -18,7 +18,7 @@ class LocalTrackPublication extends TrackPublication {
     // register dispose func
     onDispose(() async {
       // this object is responsible for disposing track
-      await track.dispose();
+      await this.track?.dispose();
     });
   }
 

lib/src/track/remote_track_publication.dart

@@ -1,3 +1,8 @@
+import 'dart:math';
+import 'dart:ui';
+
+import 'package:collection/collection.dart';
+import 'package:livekit_client/src/internal/events.dart';
 import 'package:livekit_client/src/logger.dart';
 import 'package:meta/meta.dart';
 
@@ -6,9 +11,23 @@ import '../extensions.dart';
 import '../participant/remote_participant.dart';
 import '../proto/livekit_models.pb.dart' as lk_models;
 import '../proto/livekit_rtc.pb.dart' as lk_rtc;
+import '../utils.dart';
 import 'track.dart';
 import 'track_publication.dart';
 
+class RendererVisibility {
+  final String rendererId;
+  final String trackId;
+  final bool visible;
+  final Size size;
+  RendererVisibility({
+    required this.rendererId,
+    required this.trackId,
+    required this.visible,
+    required this.size,
+  });
+}
+
 /// Represents a track publication from a RemoteParticipant. Provides methods to
 /// control if we should subscribe to the track, and its quality (for video).
 class RemoteTrackPublication extends TrackPublication {
@@ -17,17 +36,124 @@ class RemoteTrackPublication extends TrackPublication {
   lk_rtc.VideoQuality _videoQuality = lk_rtc.VideoQuality.HIGH;
   lk_rtc.VideoQuality get videoQuality => _videoQuality;
 
+  // used to report renderer visibility to the server
+  // and optimize
+  final _visibilities = <String, RendererVisibility>{};
+  Function(void)? _visibilityDidUpdate;
+  Function? _cancelVisibilityDebounceFunc;
+
   RemoteTrackPublication(
     lk_models.TrackInfo info,
     this._participant, [
     Track? track,
   ]) : super.fromInfo(info) {
-    this.track = track;
     // register dispose func
     onDispose(() async {
+      _cancelVisibilityDebounceFunc?.call();
       // this object is responsible for disposing track
-      await track?.dispose();
+      await this.track?.dispose();
     });
+
+    _visibilityDidUpdate = Utils.createDebounceFunc(
+      _shouldComputeVisibilityUpdate,
+      cancelFunc: (func) => _cancelVisibilityDebounceFunc = func,
+      wait: const Duration(seconds: 2),
+    );
+
+    this.track = track;
+  }
+
+  // called any time visibility info updates
+  // from one of the renderers
+  void _onVideoRendererVisibilityUpdateEvent(
+      TrackVisibilityUpdatedEvent event) {
+    final info = event.info;
+    final trackSid = event.track.sid;
+    if (trackSid != null && info != null) {
+      logger.fine('[Visibility] ${event.rendererId} did update '
+          'track: ${event.track.sid} '
+          'visibleFraction: ${info.visibleFraction} '
+          'size: ${info.size}');
+      _visibilities[event.rendererId] = RendererVisibility(
+        rendererId: event.rendererId,
+        trackId: trackSid,
+        visible: info.visibleFraction > 0,
+        size: info.size,
+      );
+
+      // quickly enable if currently disabled
+      if (!enabled && _hasVisibleRenderers()) {
+        logger.fine('[Visibility] Trying to re-enable quickly');
+        _cancelVisibilityDebounceFunc?.call();
+        _shouldComputeVisibilityUpdate(null);
+      } else {
+        _visibilityDidUpdate?.call(null);
+      }
+    } else {
+      // widget as been disposed, but track still exists
+      logger.fine('[Visibility] ${event.rendererId} was removed');
+      _visibilities.remove(event.rendererId);
+      _visibilityDidUpdate?.call(null);
+    }
+
+    logger.fine(
+        '[Visibility] Ids ${_visibilities.values.map((e) => e.rendererId)}');
+  }
+
+  bool _hasVisibleRenderers() =>
+      _visibilities.values.firstWhereOrNull((e) => e.visible) != null;
+
+  void _shouldComputeVisibilityUpdate(void _) {
+    //
+    Size maxSize(Size s1, Size s2) => Size(
+          max(s1.width, s2.width),
+          max(s1.height, s2.height),
+        );
+
+    _disabled = !_hasVisibleRenderers();
+
+    final settings = lk_rtc.UpdateTrackSettings(
+      trackSids: [sid],
+      disabled: _disabled,
+    );
+
+    if (!_disabled) {
+      final largest = _visibilities.values
+          .map((e) => e.size)
+          .reduce((value, element) => maxSize(value, element));
+      settings.width = largest.width.floor();
+      settings.height = largest.height.floor();
+    }
+
+    logger.fine('[Visibility] Sending to server ${settings.toProto3Json()}');
+    _participant.client.sendUpdateTrackSettings(settings);
+  }
+
+  @override
+  set track(Track? newValue) {
+    if (super.track != newValue) {
+      logger.fine('setTrack ${newValue} $sid ${objectId}');
+      // dispose previous track (if exists)
+      super.track?.dispose();
+      super.track = newValue;
+
+      if (newValue != null) {
+        //
+        // Attach visibility event listener
+        //
+        final listener = newValue.createListener();
+        listener.on<TrackVisibilityUpdatedEvent>(
+            _onVideoRendererVisibilityUpdateEvent);
+        newValue.onDispose(() async {
+          await listener.dispose();
+          // consider all views are disposed when track is null
+          _visibilities.clear();
+          // _visibilityDidUpdate?.call(null);
+        });
+      }
+
+      _visibilityDidUpdate?.call(null);
+    }
   }
 
   set videoQuality(lk_rtc.VideoQuality val) {

lib/src/track/track.dart

@@ -3,15 +3,19 @@ import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
 import 'package:meta/meta.dart';
 import 'package:uuid/uuid.dart';
 
+import '../events.dart';
 import '../extensions.dart';
 import '../logger.dart';
+import '../managers/event.dart';
 import '../proto/livekit_models.pb.dart' as lk_models;
 import '../support/disposable.dart';
 
 /// Wrapper around a MediaStreamTrack with additional metadata.
 /// Base for [AudioTrack] and [VideoTrack],
 /// can not be instantiated directly.
-abstract class Track extends DisposableChangeNotifier {
+abstract class Track extends DisposableChangeNotifier
+    with EventsEmittable<TrackEvent> {
+  static const uuid = Uuid();
   static const cameraName = 'camera';
   static const screenShareName = 'screen';
 
@@ -31,7 +35,13 @@ abstract class Track extends DisposableChangeNotifier {
     this.kind,
     this.name,
     this.mediaStreamTrack,
-  );
+  ) {
+    onDispose(() async {
+      logger.fine('${objectId} onDispose()');
+      // dispose events
+      await events.dispose();
+    });
+  }
 
   bool get muted =>
       mediaStreamTrack.muted == null ? false : mediaStreamTrack.muted!;
@@ -52,7 +62,6 @@ abstract class Track extends DisposableChangeNotifier {
     var cid = _cid ?? mediaStreamTrack.id;
 
     if (cid == null) {
-      const uuid = Uuid();
       cid = uuid.v4();
       _cid = cid;
     }

lib/src/track/video_track.dart

@@ -1,17 +1,18 @@
 import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
+import 'package:meta/meta.dart';
 
 import '../proto/livekit_models.pb.dart' as lk_models;
 import 'track.dart';
 
 /// A video track will notify when its mediaTrack has changed.
 class VideoTrack extends Track {
-  //
   rtc.MediaStream _mediaStream;
 
   VideoTrack(
     String name,
     rtc.MediaStreamTrack mediaTrack,
     this._mediaStream,
+    // this._client,
   ) : super(
           lk_models.TrackType.VIDEO,
           name,
@@ -22,6 +23,7 @@ class VideoTrack extends Track {
 
   /// internal use
   /// {@nodoc}
+  @internal
   void setMediaStream(rtc.MediaStream stream) {
     _mediaStream = stream;
     notifyListeners();

lib/src/transport.dart

@@ -62,7 +62,7 @@ class PCTransport extends Disposable {
   }
 
   late final negotiate = Utils.createDebounceFunc(
-    () => createAndSendOffer(),
+    (void _) => createAndSendOffer(),
     cancelFunc: (f) => _cancelDebounce = f,
     wait: Timeouts.debounce,
   );

lib/src/utils.dart

@@ -138,18 +138,18 @@ class Utils {
     ];
   }
 
-  // makes a debounce func
+  // makes a debounce func, with 1 param
-  static Function createDebounceFunc(
+  static Function(T) createDebounceFunc<T>(
-    Function f, {
+    Function(T) f, {
     Function(Function)? cancelFunc,
     required Duration wait,
   }) {
     Timer? t;
-    return () {
+    return (p) {
       t?.cancel();
       t = Timer(wait, () {
         t = null;
-        f();
+        f(p);
       });
       // pass back the cancel method so we can cancel it when no longer needed
       cancelFunc?.call(t!.cancel);

lib/src/widget/video_track_renderer.dart

@@ -1,21 +1,25 @@
 import 'package:flutter/foundation.dart';
 import 'package:flutter/material.dart';
 import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
+import 'package:livekit_client/livekit_client.dart';
+import 'package:visibility_detector/visibility_detector.dart';
 
+import '../extensions.dart';
+import '../internal/events.dart';
+import '../logger.dart';
 import '../track/local_video_track.dart';
 import '../track/video_track.dart';
 
 /// Widget that renders a [VideoTrack].
 class VideoTrackRenderer extends StatefulWidget {
   final VideoTrack track;
-  final rtc.RTCVideoRenderer renderer;
   final rtc.RTCVideoViewObjectFit fit;
 
-  VideoTrackRenderer(
+  const VideoTrackRenderer(
     this.track, {
     this.fit = rtc.RTCVideoViewObjectFit.RTCVideoViewObjectFitContain,
-  })  : renderer = rtc.RTCVideoRenderer(),
+    Key? key,
-        super(key: ValueKey(track.sid));
+  }) : super(key: key);
 
   @override
   State<StatefulWidget> createState() => _VideoTrackRendererState();
@@ -23,49 +27,79 @@ class VideoTrackRenderer extends StatefulWidget {
 
 class _VideoTrackRendererState extends State<VideoTrackRenderer> {
   final _renderer = rtc.RTCVideoRenderer();
+  bool _rendererReady = false;
+  EventsListener<TrackEvent>? _listener;
+
+  Key get _keyForVisibilityDetector =>
+      ValueKey('${objectId}-VisibilityDetector');
 
   @override
   void initState() {
     super.initState();
-    widget.track.addListener(_trackChanged);
+    logger.fine('$objectId initState()');
-    _initRenderer();
+
+    (() async {
+      await _renderer.initialize();
+      await _attach();
+      setState(() => _rendererReady = true);
+    })();
   }
 
   @override
   void dispose() {
-    widget.track.removeListener(_trackChanged);
+    logger.fine('$objectId dispose()');
+    VisibilityDetectorController.instance.forget(_keyForVisibilityDetector);
+    // report that instance is disposing
+    // if the track is disposed first we can't emit event
+    widget.track.events.emit(TrackVisibilityUpdatedEvent(
+      rendererId: objectId,
+      track: widget.track,
+      info: null,
+    ));
+    _listener?.dispose();
     _renderer.srcObject = null;
     _renderer.dispose();
     super.dispose();
   }
 
+  Future<void> _attach() async {
+    logger.fine('[VideoTrackRenderer] attached to ${widget.track.objectId}');
+    _renderer.srcObject = widget.track.mediaStream;
+    await _listener?.dispose();
+    _listener = widget.track.createListener()
+      ..on<TrackStreamUpdatedEvent>((event) {
+        _renderer.srcObject = event.stream;
+      });
+  }
+
   @override
   void didUpdateWidget(covariant VideoTrackRenderer oldWidget) {
-    oldWidget.track.removeListener(_trackChanged);
-    widget.track.addListener(_trackChanged);
-    _trackChanged();
     super.didUpdateWidget(oldWidget);
-  }
+    if (widget.track != oldWidget.track) {
-
+      // TODO: re-attach only if needed
-  void _trackChanged() {
+      (() async {
-    setState(() {
+        await _attach();
-      _renderer.srcObject = widget.track.mediaStream;
+      })();
-    });
+    }
-  }
-
-  void _initRenderer() async {
-    await _renderer.initialize();
-    _trackChanged();
   }
 
   @override
-  Widget build(BuildContext context) {
+  Widget build(BuildContext context) => !_rendererReady
-    final isLocal = widget.track is LocalVideoTrack;
+      ? Container()
-    return rtc.RTCVideoView(
+      : VisibilityDetector(
-      _renderer,
+          key: _keyForVisibilityDetector,
-      mirror: isLocal,
+          // emit event when visibility updates
-      filterQuality: FilterQuality.medium,
+          onVisibilityChanged: (VisibilityInfo info) =>
-      objectFit: widget.fit,
+              widget.track.events.emit(TrackVisibilityUpdatedEvent(
-    );
+            rendererId: objectId,
-  }
+            track: widget.track,
+            info: info,
+          )),
+          child: rtc.RTCVideoView(
+            _renderer,
+            mirror: widget.track is LocalVideoTrack,
+            filterQuality: FilterQuality.medium,
+            objectFit: widget.fit,
+          ),
+        );
 }

pubspec.lock

@@ -301,6 +301,13 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "2.1.0"
+  visibility_detector:
+    dependency: "direct main"
+    description:
+      name: visibility_detector
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "0.2.2"
   win32:
     dependency: transitive
     description:

pubspec.yaml

@@ -17,12 +17,12 @@ dependencies:
   collection: ^1.15.0
   fixnum: ^1.0.0
   meta: ^1.7.0
-
   http: ^0.13.3
   logging: ^1.0.2
   uuid: ^3.0.4
   synchronized: ^3.0.0
   protobuf: ^2.0.0
+  visibility_detector: ^0.2.2
   flutter_webrtc: ^0.7.0+hotfix.2
 
 dev_dependencies: