Fix: Visibility Optimization randomly not working (#65)

* use widget id instead of state id

* ignore local video view events

* switch remote view visibility also

* use Key for a reliable id

* remove local check

* clean up

* redesign

* check mounted

* remove mount check

* more optimization

* quick wake up

* mark internal

* rename to adaptiveStream

example/ios/Podfile.lock

@@ -5,7 +5,7 @@ PODS:
     - Libyuv (= 1703)
     - WebRTC-SDK (= 92.4515.11)
   - Libyuv (1703)
-  - livekit_client (0.5.3):
+  - livekit_client (0.5.5):
     - Flutter
     - WebRTC-SDK (~> 92.4515)
   - path_provider_ios (0.0.1):
@@ -42,7 +42,7 @@ SPEC CHECKSUMS:
   Flutter: 50d75fe2f02b26cc09d224853bb45737f8b3214a
   flutter_webrtc: a5a79904f0bca0ea23aff49c51ca765f70a0f703
   Libyuv: 5f79ced0ee66e60a612ca97de1e6ccacd187a437
-  livekit_client: 2b15f8f6f37bc447184af227bd66d30ed1c5fd38
+  livekit_client: 92b5a241beced828bff42ec8ed559d1ec00c16e4
   path_provider_ios: 7d7ce634493af4477d156294792024ec3485acd5
   shared_preferences_ios: aef470a42dc4675a1cdd50e3158b42e3d1232b32
   WebRTC-SDK: 21dbc6028a68f3a89718057a2caa970a4da8ebb7

example/lib/widgets/participant.dart

@@ -174,7 +174,7 @@ class _RemoteParticipantWidgetState
     for (final trackPublication in widget.participant.videoTracks) {
       print(
           'video track ${trackPublication.sid} subscribed ${trackPublication.subscribed} muted ${trackPublication.muted}');
-      if (trackPublication.subscribed && !trackPublication.muted) {
+      if (trackPublication.subscribed && !trackPublication.muted && _visible) {
         return trackPublication.track;
       }
     }

example/pubspec.lock

@@ -448,13 +448,6 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "2.1.1"
-  visibility_detector:
-    dependency: transitive
-    description:
-      name: visibility_detector
-      url: "https://pub.dartlang.org"
-    source: hosted
-    version: "0.2.2"
   webrtc_interface:
     dependency: transitive
     description:

lib/src/internal/events.dart

@@ -1,6 +1,5 @@
 import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
 import 'package:meta/meta.dart';
-import 'package:visibility_detector/visibility_detector.dart';
 
 import '../events.dart';
 import '../proto/livekit_models.pb.dart' as lk_models;
@@ -59,18 +58,6 @@ class TrackStreamUpdatedEvent with TrackEvent, InternalEvent {
   });
 }
 
-@internal
-class TrackVisibilityUpdatedEvent with TrackEvent, InternalEvent {
-  final String rendererId;
-  final Track track;
-  final VisibilityInfo? info; // null means disposed
-  const TrackVisibilityUpdatedEvent({
-    required this.rendererId,
-    required this.track,
-    required this.info,
-  });
-}
-
 // Used to notify muted state from Track to TrackPublication.
 @internal
 class InternalTrackMuteUpdatedEvent with TrackEvent, InternalEvent {

lib/src/internal/types.dart

@@ -1,22 +1,5 @@
-import 'dart:ui';
-
 import 'package:meta/meta.dart';
 
-@internal
-@immutable
-class RendererVisibility {
-  final String rendererId;
-  final String trackId;
-  final bool visible;
-  final Size size;
-  const RendererVisibility({
-    required this.rendererId,
-    required this.trackId,
-    required this.visible,
-    required this.size,
-  });
-}
-
 @internal
 @immutable
 class RTCOfferOptions {

lib/src/options.dart

@@ -49,8 +49,8 @@ class RoomOptions {
   /// [VideoTrackRenderer]'s visibility on screen.
   /// - Re-sizing a [VideoTrackRenderer] will signal the server to send down
   /// a relavant quality layer (if simulcast is enabled on the publisher)
-  /// Defaults to true.
+  /// Defaults to false.
-  final bool optimizeVideo;
+  final bool adaptiveStream;
 
   /// Set this to false in case you would like to stop the track yourself.
   /// If you set this to false, make sure you call [Track.stop].
@@ -63,7 +63,7 @@ class RoomOptions {
     this.defaultAudioCaptureOptions = const AudioCaptureOptions(),
     this.defaultVideoPublishOptions = const VideoPublishOptions(),
     this.defaultAudioPublishOptions = const AudioPublishOptions(),
-    this.optimizeVideo = true,
+    this.adaptiveStream = false,
     this.stopLocalTrackOnUnpublish = true,
   });
 }

lib/src/publication/remote.dart

@@ -1,20 +1,21 @@
+import 'dart:async';
 import 'dart:math';
-import 'dart:ui';
 
 import 'package:collection/collection.dart';
+import 'package:flutter/widgets.dart';
 import 'package:meta/meta.dart';
 
 import '../core/signal_client.dart';
 import '../events.dart';
 import '../extensions.dart';
-import '../internal/events.dart';
-import '../internal/types.dart';
 import '../logger.dart';
 import '../options.dart';
 import '../participant/remote.dart';
 import '../proto/livekit_models.pb.dart' as lk_models;
 import '../proto/livekit_rtc.pb.dart' as lk_rtc;
+import '../track/local/local.dart';
 import '../track/remote/remote.dart';
+import '../track/remote/video.dart';
 import '../types.dart';
 import '../utils.dart';
 import 'track_publication.dart';
@@ -56,9 +57,11 @@ class RemoteTrackPublication<T extends RemoteTrack>
 
   // used to report renderer visibility to the server
   // and optimize
-  final _visibilities = <String, RendererVisibility>{};
+  lk_rtc.UpdateTrackSettings? _lastSentTrackSettings;
-  Function(void)? _visibilityDidUpdate;
+  Timer? _visibilityTimer;
-  Function? _cancelVisibilityDebounceFunc;
+
+  Function(lk_rtc.UpdateTrackSettings)? _setPendingTrackSettingsUpdateRequest;
+  Function? _cancelPendingTrackSettingsUpdateRequest;
 
   RemoteTrackPublication({
     required this.participant,
@@ -69,15 +72,16 @@ class RemoteTrackPublication<T extends RemoteTrack>
 
     // register dispose func
     onDispose(() async {
-      _cancelVisibilityDebounceFunc?.call();
+      _cancelPendingTrackSettingsUpdateRequest?.call();
+      _visibilityTimer?.cancel();
       // this object is responsible for disposing track
       await this.track?.dispose();
     });
 
-    _visibilityDidUpdate = Utils.createDebounceFunc(
+    _setPendingTrackSettingsUpdateRequest = Utils.createDebounceFunc(
-      _shouldComputeVisibilityUpdate,
+      _sendPendingTrackSettingsUpdateRequest,
-      cancelFunc: (func) => _cancelVisibilityDebounceFunc = func,
+      cancelFunc: (func) => _cancelPendingTrackSettingsUpdateRequest = func,
-      wait: const Duration(seconds: 2),
+      wait: const Duration(milliseconds: 1500),
     );
 
     updateTrack(track);
@@ -93,73 +97,59 @@ class RemoteTrackPublication<T extends RemoteTrack>
     _metadataMuted = info.muted;
   }
 
-  // called any time visibility info updates
+  void _computeVideoViewVisibility({
-  // from one of the renderers
+    bool quick = false,
-  void _onVideoRendererVisibilityUpdateEvent(
+  }) {
-      TrackVisibilityUpdatedEvent event) {
+    //
-    final info = event.info;
+    Size maxOfSizes(Size s1, Size s2) => Size(
-    final trackSid = event.track.sid;
-    if (trackSid != null && info != null) {
-      logger.fine('[Visibility] ${event.rendererId} did update '
-          'track: ${event.track.sid} '
-          'visibleFraction: ${info.visibleFraction} '
-          'size: ${info.size}');
-      _visibilities[event.rendererId] = RendererVisibility(
-        rendererId: event.rendererId,
-        trackId: trackSid,
-        visible: info.visibleFraction > 0,
-        size: info.size,
-      );
-
-      // quickly enable if currently disabled
-      if (!enabled && _hasVisibleRenderers()) {
-        logger.fine('[Visibility] Trying to re-enable quickly');
-        _cancelVisibilityDebounceFunc?.call();
-        _shouldComputeVisibilityUpdate(null);
-      } else {
-        _visibilityDidUpdate?.call(null);
-      }
-    } else {
-      // widget as been disposed, but track still exists
-      logger.fine('[Visibility] ${event.rendererId} was removed');
-      _visibilities.remove(event.rendererId);
-      _visibilityDidUpdate?.call(null);
-    }
-
-    logger.fine(
-        '[Visibility] Ids ${_visibilities.values.map((e) => e.rendererId)}');
-  }
-
-  bool _hasVisibleRenderers() =>
-      _visibilities.values.firstWhereOrNull((e) => e.visible) != null;
-
-  void _shouldComputeVisibilityUpdate(void _) {
-    if (isDisposed) {
-      logger.warning('_shouldComputeVisibilityUpdate already disposed');
-      return;
-    }
-
-    Size maxSize(Size s1, Size s2) => Size(
           max(s1.width, s2.width),
           max(s1.height, s2.height),
         );
 
-    _enabled = _hasVisibleRenderers();
+    final videoTrack = track as VideoTrack;
 
     final settings = lk_rtc.UpdateTrackSettings(
       trackSids: [sid],
-      disabled: !_enabled,
+      disabled: true,
     );
 
-    if (_enabled) {
+    // filter visible build contexts
-      final largest = _visibilities.values
+    final viewSizes = videoTrack.viewKeys
-          .map((e) => e.size)
+        .map((e) => e.currentContext)
-          .reduce((value, element) => maxSize(value, element));
+        .whereNotNull()
-      settings.width = largest.width.floor();
+        .map((e) => e.findRenderObject() as RenderBox?)
-      settings.height = largest.height.floor();
+        .whereNotNull()
+        .map((e) => e.size);
+
+    logger.finer(
+        '[Visibility] ${track?.sid} watching ${viewSizes.length} views...');
+
+    if (viewSizes.isNotEmpty) {
+      // compute largest size
+      final largestSize =
+          viewSizes.reduce((value, element) => maxOfSizes(value, element));
+
+      settings
+        ..disabled = false
+        ..width = largestSize.width.ceil()
+        ..height = largestSize.height.ceil();
     }
 
-    logger.fine('[Visibility] Sending to server ${settings.toProto3Json()}');
+    // Only send new settings to server if it changed
+    if (settings != _lastSentTrackSettings) {
+      _lastSentTrackSettings = settings;
+      logger.fine('[Visibility] Change detected, quick: $quick');
+      if (quick) {
+        _sendPendingTrackSettingsUpdateRequest(settings);
+      } else {
+        _setPendingTrackSettingsUpdateRequest?.call(settings);
+      }
+    }
+  }
+
+  void _sendPendingTrackSettingsUpdateRequest(
+      lk_rtc.UpdateTrackSettings settings) {
+    logger.fine('[Visibility] Sending... ${settings.toProto3Json()}');
     participant.room.engine.signalClient.sendUpdateTrackSettings(settings);
   }
 
@@ -169,31 +159,36 @@ class RemoteTrackPublication<T extends RemoteTrack>
     logger.fine('RemoteTrackPublication.updateTrack track: $newValue');
     final didUpdate = await super.updateTrack(newValue);
 
-    final roomOptions = participant.room.roomOptions ?? const RoomOptions();
+    if (didUpdate) {
+      // Stop current visibility timer (if exists)
+      _cancelPendingTrackSettingsUpdateRequest?.call();
+      _visibilityTimer?.cancel();
 
-    if (didUpdate && newValue != null) {
+      final roomOptions = participant.room.roomOptions ?? const RoomOptions();
-      // if new Track has been set to this RemoteTrackPublication,
+      if (roomOptions.adaptiveStream && newValue is RemoteVideoTrack) {
-      // update the Track's muted state from the latest info.
+        // Start monitoring visibility
-      newValue.updateMuted(
+        _visibilityTimer = Timer.periodic(
-        _metadataMuted,
+          const Duration(milliseconds: 300),
-        shouldNotify: false, // don't emit event since this is initial state
+          (_) => _computeVideoViewVisibility(),
-      );
+        );
 
-      // Only listen for visibility updates if video optimization is on
+        newValue.onVideoViewBuild = (_) {
-      // and the attached track is a video track
+          logger.fine('[Visibility] VideoView did build');
-      if (roomOptions.optimizeVideo &&
+          if (_lastSentTrackSettings?.disabled == true) {
-          newValue.kind == lk_models.TrackType.VIDEO) {
+            // quick enable
-        // Attach visibility event listener
+            _cancelPendingTrackSettingsUpdateRequest?.call();
-        final listener = newValue.createListener();
+            _computeVideoViewVisibility(quick: true);
-        listener.on<TrackVisibilityUpdatedEvent>(
+          }
-            _onVideoRendererVisibilityUpdateEvent);
+        };
+      }
 
-        newValue.onDispose(() async {
+      if (newValue != null) {
-          await listener.dispose();
+        // if new Track has been set to this RemoteTrackPublication,
-          // consider all views are disposed when track is null
+        // update the Track's muted state from the latest info.
-          _visibilities.clear();
+        newValue.updateMuted(
-          if (!isDisposed) _visibilityDidUpdate?.call(null);
+          _metadataMuted,
-        });
+          shouldNotify: false, // don't emit event since this is initial state
+        );
       }
     }
 

lib/src/track/local/local.dart

@@ -1,3 +1,4 @@
+import 'package:flutter/material.dart';
 import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
 import 'package:meta/meta.dart';
 
@@ -16,7 +17,25 @@ import 'audio.dart';
 import 'video.dart';
 
 /// Used to group [LocalVideoTrack] and [RemoteVideoTrack].
-mixin VideoTrack on Track {}
+mixin VideoTrack on Track {
+  @internal
+  final List<GlobalKey> viewKeys = [];
+
+  @internal
+  Function(Key)? onVideoViewBuild;
+
+  @internal
+  GlobalKey addViewKey() {
+    final key = GlobalKey();
+    viewKeys.add(key);
+    return key;
+  }
+
+  @internal
+  void removeViewKey(GlobalKey key) {
+    viewKeys.remove(key);
+  }
+}
 
 /// Used to group [LocalAudioTrack] and [RemoteAudioTrack].
 mixin AudioTrack on Track {}

lib/src/widget/video_track_renderer.dart

@@ -1,12 +1,8 @@
 import 'package:flutter/material.dart';
-
 import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
-import 'package:visibility_detector/visibility_detector.dart';
 
 import '../events.dart';
-import '../extensions.dart';
 import '../internal/events.dart';
-import '../logger.dart';
 import '../managers/event.dart';
 import '../track/local/local.dart';
 import '../track/local/video.dart';
@@ -30,14 +26,13 @@ class _VideoTrackRendererState extends State<VideoTrackRenderer> {
   final _renderer = rtc.RTCVideoRenderer();
   bool _rendererReady = false;
   EventsListener<TrackEvent>? _listener;
-
+  // Used to compute visibility information
-  Key get _keyForVisibilityDetector =>
+  late GlobalKey _internalKey;
-      ValueKey('${objectId}-VisibilityDetector');
 
   @override
   void initState() {
     super.initState();
-    logger.fine('$objectId initState()');
+    _internalKey = widget.track.addViewKey();
 
     (() async {
       await _renderer.initialize();
@@ -48,15 +43,7 @@ class _VideoTrackRendererState extends State<VideoTrackRenderer> {
 
   @override
   void dispose() {
-    logger.fine('$objectId dispose()');
+    widget.track.removeViewKey(_internalKey);
-    VisibilityDetectorController.instance.forget(_keyForVisibilityDetector);
-    // report that instance is disposing
-    // if the track is disposed first we can't emit event
-    widget.track.events.emit(TrackVisibilityUpdatedEvent(
-      rendererId: objectId,
-      track: widget.track,
-      info: null,
-    ));
     _listener?.dispose();
     _renderer.srcObject = null;
     _renderer.dispose();
@@ -64,19 +51,23 @@ class _VideoTrackRendererState extends State<VideoTrackRenderer> {
   }
 
   Future<void> _attach() async {
-    logger.fine('[VideoTrackRenderer] attached to ${widget.track.objectId}');
     _renderer.srcObject = widget.track.mediaStream;
     await _listener?.dispose();
     _listener = widget.track.createListener()
       ..on<TrackStreamUpdatedEvent>((event) {
-        _renderer.srcObject = event.stream;
+        if (mounted) {
+          _renderer.srcObject = event.stream;
+        }
       });
   }
 
   @override
   void didUpdateWidget(covariant VideoTrackRenderer oldWidget) {
     super.didUpdateWidget(oldWidget);
+    //
     if (widget.track != oldWidget.track) {
+      oldWidget.track.removeViewKey(_internalKey);
+      _internalKey = widget.track.addViewKey();
       // TODO: re-attach only if needed
       (() async {
         await _attach();
@@ -87,20 +78,19 @@ class _VideoTrackRendererState extends State<VideoTrackRenderer> {
   @override
   Widget build(BuildContext context) => !_rendererReady
       ? Container()
-      : VisibilityDetector(
+      : Builder(
-          key: _keyForVisibilityDetector,
+          key: _internalKey,
-          // emit event when visibility updates
+          builder: (ctx) {
-          onVisibilityChanged: (VisibilityInfo info) =>
+            // let it render before notifying build
-              widget.track.events.emit(TrackVisibilityUpdatedEvent(
+            WidgetsBinding.instance?.addPostFrameCallback((timeStamp) {
-            rendererId: objectId,
+              widget.track.onVideoViewBuild?.call(_internalKey);
-            track: widget.track,
+            });
-            info: info,
+            return rtc.RTCVideoView(
-          )),
+              _renderer,
-          child: rtc.RTCVideoView(
+              mirror: widget.track is LocalVideoTrack,
-            _renderer,
+              filterQuality: FilterQuality.medium,
-            mirror: widget.track is LocalVideoTrack,
+              objectFit: widget.fit,
-            filterQuality: FilterQuality.medium,
+            );
-            objectFit: widget.fit,
+          },
-          ),
         );
 }

pubspec.lock

@@ -343,13 +343,6 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "2.1.1"
-  visibility_detector:
-    dependency: "direct main"
-    description:
-      name: visibility_detector
-      url: "https://pub.dartlang.org"
-    source: hosted
-    version: "0.2.2"
   webrtc_interface:
     dependency: transitive
     description:

pubspec.yaml

@@ -22,7 +22,6 @@ dependencies:
   uuid: ^3.0.4
   synchronized: ^3.0.0
   protobuf: ^2.0.1
-  visibility_detector: ^0.2.2
   flutter_webrtc: ^0.8.0
   dart_webrtc: ^1.0.2