From 28e7091325f6a692948037940065d02187a1dbb1 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 15 Dec 2021 19:09:38 +0700 Subject: [PATCH] Send video layers (#55) * signal method * videoLayers param on addTrack request * screen share presets * implemented * cleaner syntax --- lib/src/participant/local_participant.dart | 28 ++-- lib/src/rtc_engine.dart | 2 + lib/src/signal_client.dart | 25 +++- lib/src/utils.dart | 160 +++++++++++++++------ 4 files changed, 159 insertions(+), 56 deletions(-) diff --git a/lib/src/participant/local_participant.dart b/lib/src/participant/local_participant.dart index ddae576..a5a3e2e 100644 --- a/lib/src/participant/local_participant.dart +++ b/lib/src/participant/local_participant.dart @@ -119,29 +119,35 @@ class LocalParticipant extends Participant { } } + logger.fine( + 'Compute encodings with resolution: ${dimensions}, options: ${publishOptions}'); + + // Video encodings and simulcasts + final encodings = Utils.computeVideoEncodings( + isScreenShare: track.source == TrackSource.screenShareVideo, + dimensions: dimensions, + options: publishOptions, + ); + + logger.fine('Using encodings: ${encodings?.map((e) => e.toMap())}'); + + final layers = Utils.computeVideoLayers(dimensions, encodings); + + logger.fine('Video layers: ${layers.map((e) => e)}'); + final trackInfo = await room.engine.addTrack( cid: track.getCid(), name: track.name, kind: track.kind, source: track.source.toPBType(), dimensions: dimensions, + videoLayers: layers, ); logger.fine('publishVideoTrack addTrack response: ${trackInfo}'); await track.start(); - logger.fine( - 'Compute encodings with resolution: ${dimensions}, options: ${publishOptions}'); - - // Video encodings and simulcasts - final encodings = Utils.computeVideoEncodings( - dimensions: dimensions, - options: publishOptions, - ); - - logger.fine('Using encodings: ${encodings?.map((e) => e.toMap())}'); - final transceiverInit = rtc.RTCRtpTransceiverInit( direction: rtc.TransceiverDirection.SendOnly, sendEncodings: encodings, diff --git a/lib/src/rtc_engine.dart b/lib/src/rtc_engine.dart index 936338b..b9ecd0d 100644 --- a/lib/src/rtc_engine.dart +++ b/lib/src/rtc_engine.dart @@ -149,6 +149,7 @@ class RTCEngine extends Disposable with EventsEmittable { required lk_models.TrackSource source, VideoDimensions? dimensions, bool? dtx, + List? videoLayers, }) async { // TODO: Check if cid already published @@ -160,6 +161,7 @@ class RTCEngine extends Disposable with EventsEmittable { source: source, dimensions: dimensions, dtx: dtx, + videoLayers: videoLayers, ); // wait for response, or timeout diff --git a/lib/src/signal_client.dart b/lib/src/signal_client.dart index 007b506..6b1b591 100644 --- a/lib/src/signal_client.dart +++ b/lib/src/signal_client.dart @@ -150,6 +150,7 @@ class SignalClient extends Disposable with EventsEmittable { required lk_models.TrackSource source, VideoDimensions? dimensions, bool? dtx, + List? videoLayers, }) { final req = lk_rtc.AddTrackRequest( cid: cid, @@ -158,10 +159,17 @@ class SignalClient extends Disposable with EventsEmittable { source: source, ); - if (type == lk_models.TrackType.VIDEO && dimensions != null) { + if (type == lk_models.TrackType.VIDEO) { // video specific - req.width = dimensions.width; - req.height = dimensions.height; + if (dimensions != null) { + req.width = dimensions.width; + req.height = dimensions.height; + } + if (videoLayers != null && videoLayers.isNotEmpty) { + req.layers + ..clear() + ..addAll(videoLayers); + } } if (type == lk_models.TrackType.AUDIO && dtx != null) { @@ -184,6 +192,17 @@ class SignalClient extends Disposable with EventsEmittable { subscription: subscription, )); + void sendUpdateVideoLayers( + String trackSid, + List layers, + ) => + _sendRequest(lk_rtc.SignalRequest( + updateLayers: lk_rtc.UpdateVideoLayers( + trackSid: trackSid, + layers: layers, + ), + )); + void sendLeave() => _sendRequest(lk_rtc.SignalRequest( leave: lk_rtc.LeaveRequest(), )); diff --git a/lib/src/utils.dart b/lib/src/utils.dart index d36a6a5..f03f71d 100644 --- a/lib/src/utils.dart +++ b/lib/src/utils.dart @@ -1,9 +1,12 @@ import 'dart:async'; import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; +import 'package:collection/collection.dart'; +import './proto/livekit_models.pb.dart' as lk_models; import 'extensions.dart'; import 'livekit.dart'; +import 'logger.dart'; import 'options.dart'; import 'track/options.dart'; import 'types.dart'; @@ -55,31 +58,63 @@ class Utils { ); } - static List _presetsForDimensions( - VideoDimensions dimensions, - ) { - final double aspect = dimensions.width / dimensions.height; + static List _presetsForDimensions({ + required bool isScreenShare, + required VideoDimensions dimensions, + }) { + if (isScreenShare) return VideoParameters.presetsScreenShare; + + final double aspect = dimensions.width > dimensions.height + ? dimensions.width / dimensions.height + : dimensions.height / dimensions.width; if ((aspect - 16.0 / 9.0).abs() < (aspect - 4.0 / 3.0).abs()) { return VideoParameters.presets169; } return VideoParameters.presets43; } - static VideoParameters _findPresetForDimensions( - VideoDimensions dimensions, { + static VideoEncoding _findAppropriateEncoding({ + required bool isScreenShare, + required VideoDimensions dimensions, required List presets, }) { assert(presets.isNotEmpty, 'presets should not be empty'); - VideoParameters result = presets.first; + VideoEncoding result = presets.first.encoding; + + // handle portrait by swapping dimensions + final size = dimensions.max(); + for (final preset in presets) { - if (dimensions.width >= preset.dimensions.width && - dimensions.height >= preset.dimensions.height) result = preset; + result = preset.encoding; + if (preset.dimensions.width >= size) break; } return result; } + static final videoRids = ['q', 'h', 'f']; + + static List encodingsFromPresets( + VideoDimensions dimensions, { + required List presets, + }) { + List result = []; + presets.forEachIndexed((i, e) { + if (i >= videoRids.length) { + return; + } + final size = dimensions.max(); + final rid = videoRids[i]; + result.add(e.encoding.toRTCRtpEncoding( + rid: rid, + scaleResolutionDownBy: size / e.dimensions.height, + )); + }); + return result; + } + static List? computeVideoEncodings({ + required bool isScreenShare, VideoDimensions? dimensions, VideoPublishOptions? options, }) { @@ -87,53 +122,94 @@ class Utils { VideoEncoding? videoEncoding = options.videoEncoding; - if ((videoEncoding == null && !options.simulcast) || dimensions == null) { + final useSimulcast = !isScreenShare && options.simulcast; + + if ((videoEncoding == null && !useSimulcast) || dimensions == null) { // don't set encoding when we are not simulcasting and user isn't restricting // encoding parameters return null; } - final presets = _presetsForDimensions(dimensions); + final presets = _presetsForDimensions( + isScreenShare: isScreenShare, + dimensions: dimensions, + ); if (videoEncoding == null) { // find the right encoding based on width/height - final preset = _findPresetForDimensions(dimensions, presets: presets); - // print('Using preset: ${preset.id}'); - videoEncoding = preset.encoding; - // log.debug('using video encoding', videoEncoding); + videoEncoding = _findAppropriateEncoding( + isScreenShare: isScreenShare, + dimensions: dimensions, + presets: presets, + ); + logger.fine('using video encoding', videoEncoding); } // Not simulcast - if (!options.simulcast) return [videoEncoding.toRTCRtpEncoding()]; + if (!useSimulcast) return [videoEncoding.toRTCRtpEncoding()]; - // Compute for simulcast - final midPreset = presets[1]; - final lowPreset = presets[0]; - return [ - videoEncoding.toRTCRtpEncoding( - rid: 'f', - ), - // if resolution is high enough, we would send both h and q res.. - // otherwise only send h - if (dimensions.max() >= 960) ...[ - midPreset.encoding.toRTCRtpEncoding( - rid: 'h', - // passing decimals to hardware encoder of android devices - // often causes issues so we better use integers - scaleResolutionDownBy: 2, - ), - lowPreset.encoding.toRTCRtpEncoding( - rid: 'q', - scaleResolutionDownBy: 4, - ), - ] else - lowPreset.encoding.toRTCRtpEncoding( - rid: 'h', - scaleResolutionDownBy: 2, - ), - ]; + final VideoParameters lowPreset = presets.first; + VideoParameters? midPreset; + if (presets.length > 1) { + midPreset = presets[1]; + } + final original = VideoParameters( + dimensions: dimensions, + encoding: videoEncoding, + ); + + final size = dimensions.max(); + List computedPresets = [original]; + + if (size >= 960 && midPreset != null) { + computedPresets = [lowPreset, midPreset, original]; + } else if (size >= 500) { + computedPresets = [lowPreset, original]; + } + + return encodingsFromPresets( + dimensions, + presets: computedPresets, + ); } + static List computeVideoLayers( + VideoDimensions dimensions, + List? encodings, + ) { + // default to a single layer, HQ + if (encodings == null) { + return [ + lk_models.VideoLayer( + quality: lk_models.VideoQuality.HIGH, + width: dimensions.width, + height: dimensions.height, + bitrate: 0, + ) + ]; + } + + return encodings.map((e) { + final scale = e.scaleResolutionDownBy ?? 1; + var quality = videoQualityForRid(e.rid); + if (quality == null && encodings.length == 1) { + quality = lk_models.VideoQuality.HIGH; + } + return lk_models.VideoLayer( + quality: quality, + width: (dimensions.width.toDouble() / scale).floor(), + height: (dimensions.height.toDouble() / scale).floor(), + bitrate: e.maxBitrate ?? 0, + ); + }).toList(); + } + + static lk_models.VideoQuality? videoQualityForRid(String? rid) => { + 'f': lk_models.VideoQuality.HIGH, + 'h': lk_models.VideoQuality.MEDIUM, + 'q': lk_models.VideoQuality.LOW, + }[rid]; + // makes a debounce func, with 1 param static Function(T) createDebounceFunc( Function(T) f, {