talksik/client-sdk-flutter
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix: Visibility Optimization randomly not working (#65)

Browse Source 
* use widget id instead of state id

* ignore local video view events

* switch remote view visibility also

* use Key for a reliable id

* remove local check

* clean up

* redesign

* check mounted

* remove mount check

* more optimization

* quick wake up

* mark internal

* rename to adaptiveStream
This commit is contained in:
Hiroshi Horie
2022-01-01 12:59:24 +07:00
committed by GitHub
parent 2b375033e9
commit 38328c5e0d
11 changed files with 134 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files
example/ios/Podfile.lock
+2 -2
View File
@@ -5,7 +5,7 @@ PODS:
- Libyuv (= 1703)
- WebRTC-SDK (= 92.4515.11)
- Libyuv (1703)
- livekit_client (0.5.3):
- livekit_client (0.5.5):
- Flutter
- WebRTC-SDK (~> 92.4515)
- path_provider_ios (0.0.1):
@@ -42,7 +42,7 @@ SPEC CHECKSUMS:
Flutter: 50d75fe2f02b26cc09d224853bb45737f8b3214a
flutter_webrtc: a5a79904f0bca0ea23aff49c51ca765f70a0f703
Libyuv: 5f79ced0ee66e60a612ca97de1e6ccacd187a437
livekit_client: 2b15f8f6f37bc447184af227bd66d30ed1c5fd38
livekit_client: 92b5a241beced828bff42ec8ed559d1ec00c16e4
path_provider_ios: 7d7ce634493af4477d156294792024ec3485acd5
shared_preferences_ios: aef470a42dc4675a1cdd50e3158b42e3d1232b32
WebRTC-SDK: 21dbc6028a68f3a89718057a2caa970a4da8ebb7
example/lib/widgets/participant.dart
+1 -1
View File
@@ -174,7 +174,7 @@ class _RemoteParticipantWidgetState
for (final trackPublication in widget.participant.videoTracks) {
print(
'video track ${trackPublication.sid} subscribed ${trackPublication.subscribed} muted ${trackPublication.muted}');
if (trackPublication.subscribed && !trackPublication.muted) {
if (trackPublication.subscribed && !trackPublication.muted && _visible) {
return trackPublication.track;
}
}
example/pubspec.lock
-7
View File
@@ -448,13 +448,6 @@ packages:
url: "https://pub.dartlang.org"
source: hosted
version: "2.1.1"
visibility_detector:
dependency: transitive
description:
name: visibility_detector
url: "https://pub.dartlang.org"
source: hosted
version: "0.2.2"
webrtc_interface:
dependency: transitive
description:
lib/src/internal/events.dart
-13
View File
@@ -1,6 +1,5 @@
import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
import 'package:meta/meta.dart';
import 'package:visibility_detector/visibility_detector.dart';
import '../events.dart';
import '../proto/livekit_models.pb.dart' as lk_models;
@@ -59,18 +58,6 @@ class TrackStreamUpdatedEvent with TrackEvent, InternalEvent {
});
}
@internal
class TrackVisibilityUpdatedEvent with TrackEvent, InternalEvent {
final String rendererId;
final Track track;
final VisibilityInfo? info; // null means disposed
const TrackVisibilityUpdatedEvent({
required this.rendererId,
required this.track,
required this.info,
});
}
// Used to notify muted state from Track to TrackPublication.
@internal
class InternalTrackMuteUpdatedEvent with TrackEvent, InternalEvent {
lib/src/internal/types.dart
-17
View File
@@ -1,22 +1,5 @@
import 'dart:ui';
import 'package:meta/meta.dart';
@internal
@immutable
class RendererVisibility {
final String rendererId;
final String trackId;
final bool visible;
final Size size;
const RendererVisibility({
required this.rendererId,
required this.trackId,
required this.visible,
required this.size,
});
}
@internal
@immutable
class RTCOfferOptions {
lib/src/options.dart
+3 -3
View File
@@ -49,8 +49,8 @@ class RoomOptions {
/// [VideoTrackRenderer]'s visibility on screen.
/// - Re-sizing a [VideoTrackRenderer] will signal the server to send down
/// a relavant quality layer (if simulcast is enabled on the publisher)
/// Defaults to true.
final bool optimizeVideo;
/// Defaults to false.
final bool adaptiveStream;
/// Set this to false in case you would like to stop the track yourself.
/// If you set this to false, make sure you call [Track.stop].
@@ -63,7 +63,7 @@ class RoomOptions {
this.defaultAudioCaptureOptions = const AudioCaptureOptions(),
this.defaultVideoPublishOptions = const VideoPublishOptions(),
this.defaultAudioPublishOptions = const AudioPublishOptions(),
this.optimizeVideo = true,
this.adaptiveStream = false,
this.stopLocalTrackOnUnpublish = true,
});
}
lib/src/publication/remote.dart
+84 -89
View File
@@ -1,20 +1,21 @@
import 'dart:async';
import 'dart:math';
import 'dart:ui';
import 'package:collection/collection.dart';
import 'package:flutter/widgets.dart';
import 'package:meta/meta.dart';
import '../core/signal_client.dart';
import '../events.dart';
import '../extensions.dart';
import '../internal/events.dart';
import '../internal/types.dart';
import '../logger.dart';
import '../options.dart';
import '../participant/remote.dart';
import '../proto/livekit_models.pb.dart' as lk_models;
import '../proto/livekit_rtc.pb.dart' as lk_rtc;
import '../track/local/local.dart';
import '../track/remote/remote.dart';
import '../track/remote/video.dart';
import '../types.dart';
import '../utils.dart';
import 'track_publication.dart';
@@ -56,9 +57,11 @@ class RemoteTrackPublication<T extends RemoteTrack>
// used to report renderer visibility to the server
// and optimize
final _visibilities = <String, RendererVisibility>{};
Function(void)? _visibilityDidUpdate;
Function? _cancelVisibilityDebounceFunc;
lk_rtc.UpdateTrackSettings? _lastSentTrackSettings;
Timer? _visibilityTimer;
Function(lk_rtc.UpdateTrackSettings)? _setPendingTrackSettingsUpdateRequest;
Function? _cancelPendingTrackSettingsUpdateRequest;
RemoteTrackPublication({
required this.participant,
@@ -69,15 +72,16 @@ class RemoteTrackPublication<T extends RemoteTrack>
// register dispose func
onDispose(() async {
_cancelVisibilityDebounceFunc?.call();
_cancelPendingTrackSettingsUpdateRequest?.call();
_visibilityTimer?.cancel();
// this object is responsible for disposing track
await this.track?.dispose();
});
_visibilityDidUpdate = Utils.createDebounceFunc(
_shouldComputeVisibilityUpdate,
cancelFunc: (func) => _cancelVisibilityDebounceFunc = func,
wait: const Duration(seconds: 2),
_setPendingTrackSettingsUpdateRequest = Utils.createDebounceFunc(
_sendPendingTrackSettingsUpdateRequest,
cancelFunc: (func) => _cancelPendingTrackSettingsUpdateRequest = func,
wait: const Duration(milliseconds: 1500),
);
updateTrack(track);
@@ -93,73 +97,59 @@ class RemoteTrackPublication<T extends RemoteTrack>
_metadataMuted = info.muted;
}
// called any time visibility info updates
// from one of the renderers
void _onVideoRendererVisibilityUpdateEvent(
TrackVisibilityUpdatedEvent event) {
final info = event.info;
final trackSid = event.track.sid;
if (trackSid != null && info != null) {
logger.fine('[Visibility] ${event.rendererId} did update '
'track: ${event.track.sid} '
'visibleFraction: ${info.visibleFraction} '
'size: ${info.size}');
_visibilities[event.rendererId] = RendererVisibility(
rendererId: event.rendererId,
trackId: trackSid,
visible: info.visibleFraction > 0,
size: info.size,
);
// quickly enable if currently disabled
if (!enabled && _hasVisibleRenderers()) {
logger.fine('[Visibility] Trying to re-enable quickly');
_cancelVisibilityDebounceFunc?.call();
_shouldComputeVisibilityUpdate(null);
} else {
_visibilityDidUpdate?.call(null);
}
} else {
// widget as been disposed, but track still exists
logger.fine('[Visibility] ${event.rendererId} was removed');
_visibilities.remove(event.rendererId);
_visibilityDidUpdate?.call(null);
}
logger.fine(
'[Visibility] Ids ${_visibilities.values.map((e) => e.rendererId)}');
}
bool _hasVisibleRenderers() =>
_visibilities.values.firstWhereOrNull((e) => e.visible) != null;
void _shouldComputeVisibilityUpdate(void _) {
if (isDisposed) {
logger.warning('_shouldComputeVisibilityUpdate already disposed');
return;
}
Size maxSize(Size s1, Size s2) => Size(
void _computeVideoViewVisibility({
bool quick = false,
}) {
//
Size maxOfSizes(Size s1, Size s2) => Size(
max(s1.width, s2.width),
max(s1.height, s2.height),
);
_enabled = _hasVisibleRenderers();
final videoTrack = track as VideoTrack;
final settings = lk_rtc.UpdateTrackSettings(
trackSids: [sid],
disabled: !_enabled,
disabled: true,
);
if (_enabled) {
final largest = _visibilities.values
.map((e) => e.size)
.reduce((value, element) => maxSize(value, element));
settings.width = largest.width.floor();
settings.height = largest.height.floor();
// filter visible build contexts
final viewSizes = videoTrack.viewKeys
.map((e) => e.currentContext)
.whereNotNull()
.map((e) => e.findRenderObject() as RenderBox?)
.whereNotNull()
.map((e) => e.size);
logger.finer(
'[Visibility] ${track?.sid} watching ${viewSizes.length} views...');
if (viewSizes.isNotEmpty) {
// compute largest size
final largestSize =
viewSizes.reduce((value, element) => maxOfSizes(value, element));
settings
..disabled = false
..width = largestSize.width.ceil()
..height = largestSize.height.ceil();
}
logger.fine('[Visibility] Sending to server ${settings.toProto3Json()}');
// Only send new settings to server if it changed
if (settings != _lastSentTrackSettings) {
_lastSentTrackSettings = settings;
logger.fine('[Visibility] Change detected, quick: $quick');
if (quick) {
_sendPendingTrackSettingsUpdateRequest(settings);
} else {
_setPendingTrackSettingsUpdateRequest?.call(settings);
}
}
}
void _sendPendingTrackSettingsUpdateRequest(
lk_rtc.UpdateTrackSettings settings) {
logger.fine('[Visibility] Sending... ${settings.toProto3Json()}');
participant.room.engine.signalClient.sendUpdateTrackSettings(settings);
}
@@ -169,31 +159,36 @@ class RemoteTrackPublication<T extends RemoteTrack>
logger.fine('RemoteTrackPublication.updateTrack track: $newValue');
final didUpdate = await super.updateTrack(newValue);
final roomOptions = participant.room.roomOptions ?? const RoomOptions();
if (didUpdate) {
// Stop current visibility timer (if exists)
_cancelPendingTrackSettingsUpdateRequest?.call();
_visibilityTimer?.cancel();
if (didUpdate && newValue != null) {
// if new Track has been set to this RemoteTrackPublication,
// update the Track's muted state from the latest info.
newValue.updateMuted(
_metadataMuted,
shouldNotify: false, // don't emit event since this is initial state
);
final roomOptions = participant.room.roomOptions ?? const RoomOptions();
if (roomOptions.adaptiveStream && newValue is RemoteVideoTrack) {
// Start monitoring visibility
_visibilityTimer = Timer.periodic(
const Duration(milliseconds: 300),
(_) => _computeVideoViewVisibility(),
);
// Only listen for visibility updates if video optimization is on
// and the attached track is a video track
if (roomOptions.optimizeVideo &&
newValue.kind == lk_models.TrackType.VIDEO) {
// Attach visibility event listener
final listener = newValue.createListener();
listener.on<TrackVisibilityUpdatedEvent>(
_onVideoRendererVisibilityUpdateEvent);
newValue.onVideoViewBuild = (_) {
logger.fine('[Visibility] VideoView did build');
if (_lastSentTrackSettings?.disabled == true) {
// quick enable
_cancelPendingTrackSettingsUpdateRequest?.call();
_computeVideoViewVisibility(quick: true);
}
};
}
newValue.onDispose(() async {
await listener.dispose();
// consider all views are disposed when track is null
_visibilities.clear();
if (!isDisposed) _visibilityDidUpdate?.call(null);
});
if (newValue != null) {
// if new Track has been set to this RemoteTrackPublication,
// update the Track's muted state from the latest info.
newValue.updateMuted(
_metadataMuted,
shouldNotify: false, // don't emit event since this is initial state
);
}
}
lib/src/track/local/local.dart
+20 -1
View File
@@ -1,3 +1,4 @@
import 'package:flutter/material.dart';
import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
import 'package:meta/meta.dart';
@@ -16,7 +17,25 @@ import 'audio.dart';
import 'video.dart';
/// Used to group [LocalVideoTrack] and [RemoteVideoTrack].
mixin VideoTrack on Track {}
mixin VideoTrack on Track {
@internal
final List<GlobalKey> viewKeys = [];
@internal
Function(Key)? onVideoViewBuild;
@internal
GlobalKey addViewKey() {
final key = GlobalKey();
viewKeys.add(key);
return key;
}
@internal
void removeViewKey(GlobalKey key) {
viewKeys.remove(key);
}
}
/// Used to group [LocalAudioTrack] and [RemoteAudioTrack].
mixin AudioTrack on Track {}
lib/src/widget/video_track_renderer.dart
+24 -34
View File
@@ -1,12 +1,8 @@
import 'package:flutter/material.dart';
import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
import 'package:visibility_detector/visibility_detector.dart';
import '../events.dart';
import '../extensions.dart';
import '../internal/events.dart';
import '../logger.dart';
import '../managers/event.dart';
import '../track/local/local.dart';
import '../track/local/video.dart';
@@ -30,14 +26,13 @@ class _VideoTrackRendererState extends State<VideoTrackRenderer> {
final _renderer = rtc.RTCVideoRenderer();
bool _rendererReady = false;
EventsListener<TrackEvent>? _listener;
Key get _keyForVisibilityDetector =>
ValueKey('${objectId}-VisibilityDetector');
// Used to compute visibility information
late GlobalKey _internalKey;
@override
void initState() {
super.initState();
logger.fine('$objectId initState()');
_internalKey = widget.track.addViewKey();
(() async {
await _renderer.initialize();
@@ -48,15 +43,7 @@ class _VideoTrackRendererState extends State<VideoTrackRenderer> {
@override
void dispose() {
logger.fine('$objectId dispose()');
VisibilityDetectorController.instance.forget(_keyForVisibilityDetector);
// report that instance is disposing
// if the track is disposed first we can't emit event
widget.track.events.emit(TrackVisibilityUpdatedEvent(
rendererId: objectId,
track: widget.track,
info: null,
));
widget.track.removeViewKey(_internalKey);
_listener?.dispose();
_renderer.srcObject = null;
_renderer.dispose();
@@ -64,19 +51,23 @@ class _VideoTrackRendererState extends State<VideoTrackRenderer> {
}
Future<void> _attach() async {
logger.fine('[VideoTrackRenderer] attached to ${widget.track.objectId}');
_renderer.srcObject = widget.track.mediaStream;
await _listener?.dispose();
_listener = widget.track.createListener()
..on<TrackStreamUpdatedEvent>((event) {
_renderer.srcObject = event.stream;
if (mounted) {
_renderer.srcObject = event.stream;
}
});
}
@override
void didUpdateWidget(covariant VideoTrackRenderer oldWidget) {
super.didUpdateWidget(oldWidget);
//
if (widget.track != oldWidget.track) {
oldWidget.track.removeViewKey(_internalKey);
_internalKey = widget.track.addViewKey();
// TODO: re-attach only if needed
(() async {
await _attach();
@@ -87,20 +78,19 @@ class _VideoTrackRendererState extends State<VideoTrackRenderer> {
@override
Widget build(BuildContext context) => !_rendererReady
? Container()
: VisibilityDetector(
key: _keyForVisibilityDetector,
// emit event when visibility updates
onVisibilityChanged: (VisibilityInfo info) =>
widget.track.events.emit(TrackVisibilityUpdatedEvent(
rendererId: objectId,
track: widget.track,
info: info,
)),
child: rtc.RTCVideoView(
_renderer,
mirror: widget.track is LocalVideoTrack,
filterQuality: FilterQuality.medium,
objectFit: widget.fit,
),
: Builder(
key: _internalKey,
builder: (ctx) {
// let it render before notifying build
WidgetsBinding.instance?.addPostFrameCallback((timeStamp) {
widget.track.onVideoViewBuild?.call(_internalKey);
});
return rtc.RTCVideoView(
_renderer,
mirror: widget.track is LocalVideoTrack,
filterQuality: FilterQuality.medium,
objectFit: widget.fit,
);
},
);
}
pubspec.lock
-7
View File
@@ -343,13 +343,6 @@ packages:
url: "https://pub.dartlang.org"
source: hosted
version: "2.1.1"
visibility_detector:
dependency: "direct main"
description:
name: visibility_detector
url: "https://pub.dartlang.org"
source: hosted
version: "0.2.2"
webrtc_interface:
dependency: transitive
description:
pubspec.yaml
-1
View File
@@ -22,7 +22,6 @@ dependencies:
uuid: ^3.0.4
synchronized: ^3.0.0
protobuf: ^2.0.1
visibility_detector: ^0.2.2
flutter_webrtc: ^0.8.0
dart_webrtc: ^1.0.2