diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index 389e50e..dfd4708 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -5,7 +5,7 @@ PODS: - Libyuv (= 1703) - WebRTC-SDK (= 92.4515.11) - Libyuv (1703) - - livekit_client (0.5.3): + - livekit_client (0.5.5): - Flutter - WebRTC-SDK (~> 92.4515) - path_provider_ios (0.0.1): @@ -42,7 +42,7 @@ SPEC CHECKSUMS: Flutter: 50d75fe2f02b26cc09d224853bb45737f8b3214a flutter_webrtc: a5a79904f0bca0ea23aff49c51ca765f70a0f703 Libyuv: 5f79ced0ee66e60a612ca97de1e6ccacd187a437 - livekit_client: 2b15f8f6f37bc447184af227bd66d30ed1c5fd38 + livekit_client: 92b5a241beced828bff42ec8ed559d1ec00c16e4 path_provider_ios: 7d7ce634493af4477d156294792024ec3485acd5 shared_preferences_ios: aef470a42dc4675a1cdd50e3158b42e3d1232b32 WebRTC-SDK: 21dbc6028a68f3a89718057a2caa970a4da8ebb7 diff --git a/example/lib/widgets/participant.dart b/example/lib/widgets/participant.dart index 1f7e568..caeaa02 100644 --- a/example/lib/widgets/participant.dart +++ b/example/lib/widgets/participant.dart @@ -174,7 +174,7 @@ class _RemoteParticipantWidgetState for (final trackPublication in widget.participant.videoTracks) { print( 'video track ${trackPublication.sid} subscribed ${trackPublication.subscribed} muted ${trackPublication.muted}'); - if (trackPublication.subscribed && !trackPublication.muted) { + if (trackPublication.subscribed && !trackPublication.muted && _visible) { return trackPublication.track; } } diff --git a/example/pubspec.lock b/example/pubspec.lock index 20034a2..df901bc 100644 --- a/example/pubspec.lock +++ b/example/pubspec.lock @@ -448,13 +448,6 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "2.1.1" - visibility_detector: - dependency: transitive - description: - name: visibility_detector - url: "https://pub.dartlang.org" - source: hosted - version: "0.2.2" webrtc_interface: dependency: transitive description: diff --git a/lib/src/internal/events.dart b/lib/src/internal/events.dart index 7852f98..f90e79b 100644 --- a/lib/src/internal/events.dart +++ b/lib/src/internal/events.dart @@ -1,6 +1,5 @@ import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; import 'package:meta/meta.dart'; -import 'package:visibility_detector/visibility_detector.dart'; import '../events.dart'; import '../proto/livekit_models.pb.dart' as lk_models; @@ -59,18 +58,6 @@ class TrackStreamUpdatedEvent with TrackEvent, InternalEvent { }); } -@internal -class TrackVisibilityUpdatedEvent with TrackEvent, InternalEvent { - final String rendererId; - final Track track; - final VisibilityInfo? info; // null means disposed - const TrackVisibilityUpdatedEvent({ - required this.rendererId, - required this.track, - required this.info, - }); -} - // Used to notify muted state from Track to TrackPublication. @internal class InternalTrackMuteUpdatedEvent with TrackEvent, InternalEvent { diff --git a/lib/src/internal/types.dart b/lib/src/internal/types.dart index 747a2df..21e6e3f 100644 --- a/lib/src/internal/types.dart +++ b/lib/src/internal/types.dart @@ -1,22 +1,5 @@ -import 'dart:ui'; - import 'package:meta/meta.dart'; -@internal -@immutable -class RendererVisibility { - final String rendererId; - final String trackId; - final bool visible; - final Size size; - const RendererVisibility({ - required this.rendererId, - required this.trackId, - required this.visible, - required this.size, - }); -} - @internal @immutable class RTCOfferOptions { diff --git a/lib/src/options.dart b/lib/src/options.dart index b717229..a6d9375 100644 --- a/lib/src/options.dart +++ b/lib/src/options.dart @@ -49,8 +49,8 @@ class RoomOptions { /// [VideoTrackRenderer]'s visibility on screen. /// - Re-sizing a [VideoTrackRenderer] will signal the server to send down /// a relavant quality layer (if simulcast is enabled on the publisher) - /// Defaults to true. - final bool optimizeVideo; + /// Defaults to false. + final bool adaptiveStream; /// Set this to false in case you would like to stop the track yourself. /// If you set this to false, make sure you call [Track.stop]. @@ -63,7 +63,7 @@ class RoomOptions { this.defaultAudioCaptureOptions = const AudioCaptureOptions(), this.defaultVideoPublishOptions = const VideoPublishOptions(), this.defaultAudioPublishOptions = const AudioPublishOptions(), - this.optimizeVideo = true, + this.adaptiveStream = false, this.stopLocalTrackOnUnpublish = true, }); } diff --git a/lib/src/publication/remote.dart b/lib/src/publication/remote.dart index 8117f6a..05f0c06 100644 --- a/lib/src/publication/remote.dart +++ b/lib/src/publication/remote.dart @@ -1,20 +1,21 @@ +import 'dart:async'; import 'dart:math'; -import 'dart:ui'; import 'package:collection/collection.dart'; +import 'package:flutter/widgets.dart'; import 'package:meta/meta.dart'; import '../core/signal_client.dart'; import '../events.dart'; import '../extensions.dart'; -import '../internal/events.dart'; -import '../internal/types.dart'; import '../logger.dart'; import '../options.dart'; import '../participant/remote.dart'; import '../proto/livekit_models.pb.dart' as lk_models; import '../proto/livekit_rtc.pb.dart' as lk_rtc; +import '../track/local/local.dart'; import '../track/remote/remote.dart'; +import '../track/remote/video.dart'; import '../types.dart'; import '../utils.dart'; import 'track_publication.dart'; @@ -56,9 +57,11 @@ class RemoteTrackPublication // used to report renderer visibility to the server // and optimize - final _visibilities = {}; - Function(void)? _visibilityDidUpdate; - Function? _cancelVisibilityDebounceFunc; + lk_rtc.UpdateTrackSettings? _lastSentTrackSettings; + Timer? _visibilityTimer; + + Function(lk_rtc.UpdateTrackSettings)? _setPendingTrackSettingsUpdateRequest; + Function? _cancelPendingTrackSettingsUpdateRequest; RemoteTrackPublication({ required this.participant, @@ -69,15 +72,16 @@ class RemoteTrackPublication // register dispose func onDispose(() async { - _cancelVisibilityDebounceFunc?.call(); + _cancelPendingTrackSettingsUpdateRequest?.call(); + _visibilityTimer?.cancel(); // this object is responsible for disposing track await this.track?.dispose(); }); - _visibilityDidUpdate = Utils.createDebounceFunc( - _shouldComputeVisibilityUpdate, - cancelFunc: (func) => _cancelVisibilityDebounceFunc = func, - wait: const Duration(seconds: 2), + _setPendingTrackSettingsUpdateRequest = Utils.createDebounceFunc( + _sendPendingTrackSettingsUpdateRequest, + cancelFunc: (func) => _cancelPendingTrackSettingsUpdateRequest = func, + wait: const Duration(milliseconds: 1500), ); updateTrack(track); @@ -93,73 +97,59 @@ class RemoteTrackPublication _metadataMuted = info.muted; } - // called any time visibility info updates - // from one of the renderers - void _onVideoRendererVisibilityUpdateEvent( - TrackVisibilityUpdatedEvent event) { - final info = event.info; - final trackSid = event.track.sid; - if (trackSid != null && info != null) { - logger.fine('[Visibility] ${event.rendererId} did update ' - 'track: ${event.track.sid} ' - 'visibleFraction: ${info.visibleFraction} ' - 'size: ${info.size}'); - _visibilities[event.rendererId] = RendererVisibility( - rendererId: event.rendererId, - trackId: trackSid, - visible: info.visibleFraction > 0, - size: info.size, - ); - - // quickly enable if currently disabled - if (!enabled && _hasVisibleRenderers()) { - logger.fine('[Visibility] Trying to re-enable quickly'); - _cancelVisibilityDebounceFunc?.call(); - _shouldComputeVisibilityUpdate(null); - } else { - _visibilityDidUpdate?.call(null); - } - } else { - // widget as been disposed, but track still exists - logger.fine('[Visibility] ${event.rendererId} was removed'); - _visibilities.remove(event.rendererId); - _visibilityDidUpdate?.call(null); - } - - logger.fine( - '[Visibility] Ids ${_visibilities.values.map((e) => e.rendererId)}'); - } - - bool _hasVisibleRenderers() => - _visibilities.values.firstWhereOrNull((e) => e.visible) != null; - - void _shouldComputeVisibilityUpdate(void _) { - if (isDisposed) { - logger.warning('_shouldComputeVisibilityUpdate already disposed'); - return; - } - - Size maxSize(Size s1, Size s2) => Size( + void _computeVideoViewVisibility({ + bool quick = false, + }) { + // + Size maxOfSizes(Size s1, Size s2) => Size( max(s1.width, s2.width), max(s1.height, s2.height), ); - _enabled = _hasVisibleRenderers(); + final videoTrack = track as VideoTrack; final settings = lk_rtc.UpdateTrackSettings( trackSids: [sid], - disabled: !_enabled, + disabled: true, ); - if (_enabled) { - final largest = _visibilities.values - .map((e) => e.size) - .reduce((value, element) => maxSize(value, element)); - settings.width = largest.width.floor(); - settings.height = largest.height.floor(); + // filter visible build contexts + final viewSizes = videoTrack.viewKeys + .map((e) => e.currentContext) + .whereNotNull() + .map((e) => e.findRenderObject() as RenderBox?) + .whereNotNull() + .map((e) => e.size); + + logger.finer( + '[Visibility] ${track?.sid} watching ${viewSizes.length} views...'); + + if (viewSizes.isNotEmpty) { + // compute largest size + final largestSize = + viewSizes.reduce((value, element) => maxOfSizes(value, element)); + + settings + ..disabled = false + ..width = largestSize.width.ceil() + ..height = largestSize.height.ceil(); } - logger.fine('[Visibility] Sending to server ${settings.toProto3Json()}'); + // Only send new settings to server if it changed + if (settings != _lastSentTrackSettings) { + _lastSentTrackSettings = settings; + logger.fine('[Visibility] Change detected, quick: $quick'); + if (quick) { + _sendPendingTrackSettingsUpdateRequest(settings); + } else { + _setPendingTrackSettingsUpdateRequest?.call(settings); + } + } + } + + void _sendPendingTrackSettingsUpdateRequest( + lk_rtc.UpdateTrackSettings settings) { + logger.fine('[Visibility] Sending... ${settings.toProto3Json()}'); participant.room.engine.signalClient.sendUpdateTrackSettings(settings); } @@ -169,31 +159,36 @@ class RemoteTrackPublication logger.fine('RemoteTrackPublication.updateTrack track: $newValue'); final didUpdate = await super.updateTrack(newValue); - final roomOptions = participant.room.roomOptions ?? const RoomOptions(); + if (didUpdate) { + // Stop current visibility timer (if exists) + _cancelPendingTrackSettingsUpdateRequest?.call(); + _visibilityTimer?.cancel(); - if (didUpdate && newValue != null) { - // if new Track has been set to this RemoteTrackPublication, - // update the Track's muted state from the latest info. - newValue.updateMuted( - _metadataMuted, - shouldNotify: false, // don't emit event since this is initial state - ); + final roomOptions = participant.room.roomOptions ?? const RoomOptions(); + if (roomOptions.adaptiveStream && newValue is RemoteVideoTrack) { + // Start monitoring visibility + _visibilityTimer = Timer.periodic( + const Duration(milliseconds: 300), + (_) => _computeVideoViewVisibility(), + ); - // Only listen for visibility updates if video optimization is on - // and the attached track is a video track - if (roomOptions.optimizeVideo && - newValue.kind == lk_models.TrackType.VIDEO) { - // Attach visibility event listener - final listener = newValue.createListener(); - listener.on( - _onVideoRendererVisibilityUpdateEvent); + newValue.onVideoViewBuild = (_) { + logger.fine('[Visibility] VideoView did build'); + if (_lastSentTrackSettings?.disabled == true) { + // quick enable + _cancelPendingTrackSettingsUpdateRequest?.call(); + _computeVideoViewVisibility(quick: true); + } + }; + } - newValue.onDispose(() async { - await listener.dispose(); - // consider all views are disposed when track is null - _visibilities.clear(); - if (!isDisposed) _visibilityDidUpdate?.call(null); - }); + if (newValue != null) { + // if new Track has been set to this RemoteTrackPublication, + // update the Track's muted state from the latest info. + newValue.updateMuted( + _metadataMuted, + shouldNotify: false, // don't emit event since this is initial state + ); } } diff --git a/lib/src/track/local/local.dart b/lib/src/track/local/local.dart index 5a479a1..827568a 100644 --- a/lib/src/track/local/local.dart +++ b/lib/src/track/local/local.dart @@ -1,3 +1,4 @@ +import 'package:flutter/material.dart'; import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; import 'package:meta/meta.dart'; @@ -16,7 +17,25 @@ import 'audio.dart'; import 'video.dart'; /// Used to group [LocalVideoTrack] and [RemoteVideoTrack]. -mixin VideoTrack on Track {} +mixin VideoTrack on Track { + @internal + final List viewKeys = []; + + @internal + Function(Key)? onVideoViewBuild; + + @internal + GlobalKey addViewKey() { + final key = GlobalKey(); + viewKeys.add(key); + return key; + } + + @internal + void removeViewKey(GlobalKey key) { + viewKeys.remove(key); + } +} /// Used to group [LocalAudioTrack] and [RemoteAudioTrack]. mixin AudioTrack on Track {} diff --git a/lib/src/widget/video_track_renderer.dart b/lib/src/widget/video_track_renderer.dart index e8061a7..2e6f76c 100644 --- a/lib/src/widget/video_track_renderer.dart +++ b/lib/src/widget/video_track_renderer.dart @@ -1,12 +1,8 @@ import 'package:flutter/material.dart'; - import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; -import 'package:visibility_detector/visibility_detector.dart'; import '../events.dart'; -import '../extensions.dart'; import '../internal/events.dart'; -import '../logger.dart'; import '../managers/event.dart'; import '../track/local/local.dart'; import '../track/local/video.dart'; @@ -30,14 +26,13 @@ class _VideoTrackRendererState extends State { final _renderer = rtc.RTCVideoRenderer(); bool _rendererReady = false; EventsListener? _listener; - - Key get _keyForVisibilityDetector => - ValueKey('${objectId}-VisibilityDetector'); + // Used to compute visibility information + late GlobalKey _internalKey; @override void initState() { super.initState(); - logger.fine('$objectId initState()'); + _internalKey = widget.track.addViewKey(); (() async { await _renderer.initialize(); @@ -48,15 +43,7 @@ class _VideoTrackRendererState extends State { @override void dispose() { - logger.fine('$objectId dispose()'); - VisibilityDetectorController.instance.forget(_keyForVisibilityDetector); - // report that instance is disposing - // if the track is disposed first we can't emit event - widget.track.events.emit(TrackVisibilityUpdatedEvent( - rendererId: objectId, - track: widget.track, - info: null, - )); + widget.track.removeViewKey(_internalKey); _listener?.dispose(); _renderer.srcObject = null; _renderer.dispose(); @@ -64,19 +51,23 @@ class _VideoTrackRendererState extends State { } Future _attach() async { - logger.fine('[VideoTrackRenderer] attached to ${widget.track.objectId}'); _renderer.srcObject = widget.track.mediaStream; await _listener?.dispose(); _listener = widget.track.createListener() ..on((event) { - _renderer.srcObject = event.stream; + if (mounted) { + _renderer.srcObject = event.stream; + } }); } @override void didUpdateWidget(covariant VideoTrackRenderer oldWidget) { super.didUpdateWidget(oldWidget); + // if (widget.track != oldWidget.track) { + oldWidget.track.removeViewKey(_internalKey); + _internalKey = widget.track.addViewKey(); // TODO: re-attach only if needed (() async { await _attach(); @@ -87,20 +78,19 @@ class _VideoTrackRendererState extends State { @override Widget build(BuildContext context) => !_rendererReady ? Container() - : VisibilityDetector( - key: _keyForVisibilityDetector, - // emit event when visibility updates - onVisibilityChanged: (VisibilityInfo info) => - widget.track.events.emit(TrackVisibilityUpdatedEvent( - rendererId: objectId, - track: widget.track, - info: info, - )), - child: rtc.RTCVideoView( - _renderer, - mirror: widget.track is LocalVideoTrack, - filterQuality: FilterQuality.medium, - objectFit: widget.fit, - ), + : Builder( + key: _internalKey, + builder: (ctx) { + // let it render before notifying build + WidgetsBinding.instance?.addPostFrameCallback((timeStamp) { + widget.track.onVideoViewBuild?.call(_internalKey); + }); + return rtc.RTCVideoView( + _renderer, + mirror: widget.track is LocalVideoTrack, + filterQuality: FilterQuality.medium, + objectFit: widget.fit, + ); + }, ); } diff --git a/pubspec.lock b/pubspec.lock index f8ab714..9cc1397 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -343,13 +343,6 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "2.1.1" - visibility_detector: - dependency: "direct main" - description: - name: visibility_detector - url: "https://pub.dartlang.org" - source: hosted - version: "0.2.2" webrtc_interface: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index 2e809cb..87724fd 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -22,7 +22,6 @@ dependencies: uuid: ^3.0.4 synchronized: ^3.0.0 protobuf: ^2.0.1 - visibility_detector: ^0.2.2 flutter_webrtc: ^0.8.0 dart_webrtc: ^1.0.2