talksik/client-sdk-flutter
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Send video layers (#55)

Browse Source 
* signal method

* videoLayers param on addTrack request

* screen share presets

* implemented

* cleaner syntax
This commit is contained in:
Hiroshi Horie
2021-12-15 19:09:38 +07:00
committed by GitHub
parent b24fad0f3b
commit 28e7091325
4 changed files with 159 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/src/participant/local_participant.dart
+17 -11
View File
@@ -119,29 +119,35 @@ class LocalParticipant extends Participant<LocalTrackPublication> {
}
}
logger.fine(
'Compute encodings with resolution: ${dimensions}, options: ${publishOptions}');
// Video encodings and simulcasts
final encodings = Utils.computeVideoEncodings(
isScreenShare: track.source == TrackSource.screenShareVideo,
dimensions: dimensions,
options: publishOptions,
);
logger.fine('Using encodings: ${encodings?.map((e) => e.toMap())}');
final layers = Utils.computeVideoLayers(dimensions, encodings);
logger.fine('Video layers: ${layers.map((e) => e)}');
final trackInfo = await room.engine.addTrack(
cid: track.getCid(),
name: track.name,
kind: track.kind,
source: track.source.toPBType(),
dimensions: dimensions,
videoLayers: layers,
);
logger.fine('publishVideoTrack addTrack response: ${trackInfo}');
await track.start();
logger.fine(
'Compute encodings with resolution: ${dimensions}, options: ${publishOptions}');
// Video encodings and simulcasts
final encodings = Utils.computeVideoEncodings(
dimensions: dimensions,
options: publishOptions,
);
logger.fine('Using encodings: ${encodings?.map((e) => e.toMap())}');
final transceiverInit = rtc.RTCRtpTransceiverInit(
direction: rtc.TransceiverDirection.SendOnly,
sendEncodings: encodings,
lib/src/rtc_engine.dart
+2
View File
@@ -149,6 +149,7 @@ class RTCEngine extends Disposable with EventsEmittable<EngineEvent> {
required lk_models.TrackSource source,
VideoDimensions? dimensions,
bool? dtx,
List<lk_models.VideoLayer>? videoLayers,
}) async {
// TODO: Check if cid already published
@@ -160,6 +161,7 @@ class RTCEngine extends Disposable with EventsEmittable<EngineEvent> {
source: source,
dimensions: dimensions,
dtx: dtx,
videoLayers: videoLayers,
);
// wait for response, or timeout
lib/src/signal_client.dart
+22 -3
View File
@@ -150,6 +150,7 @@ class SignalClient extends Disposable with EventsEmittable<SignalEvent> {
required lk_models.TrackSource source,
VideoDimensions? dimensions,
bool? dtx,
List<lk_models.VideoLayer>? videoLayers,
}) {
final req = lk_rtc.AddTrackRequest(
cid: cid,
@@ -158,10 +159,17 @@ class SignalClient extends Disposable with EventsEmittable<SignalEvent> {
source: source,
);
if (type == lk_models.TrackType.VIDEO && dimensions != null) {
if (type == lk_models.TrackType.VIDEO) {
// video specific
req.width = dimensions.width;
req.height = dimensions.height;
if (dimensions != null) {
req.width = dimensions.width;
req.height = dimensions.height;
}
if (videoLayers != null && videoLayers.isNotEmpty) {
req.layers
..clear()
..addAll(videoLayers);
}
}
if (type == lk_models.TrackType.AUDIO && dtx != null) {
@@ -184,6 +192,17 @@ class SignalClient extends Disposable with EventsEmittable<SignalEvent> {
subscription: subscription,
));
void sendUpdateVideoLayers(
String trackSid,
List<lk_models.VideoLayer> layers,
) =>
_sendRequest(lk_rtc.SignalRequest(
updateLayers: lk_rtc.UpdateVideoLayers(
trackSid: trackSid,
layers: layers,
),
));
void sendLeave() => _sendRequest(lk_rtc.SignalRequest(
leave: lk_rtc.LeaveRequest(),
));
lib/src/utils.dart
+118 -42
View File
@@ -1,9 +1,12 @@
import 'dart:async';
import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
import 'package:collection/collection.dart';
import './proto/livekit_models.pb.dart' as lk_models;
import 'extensions.dart';
import 'livekit.dart';
import 'logger.dart';
import 'options.dart';
import 'track/options.dart';
import 'types.dart';
@@ -55,31 +58,63 @@ class Utils {
);
}
static List<VideoParameters> _presetsForDimensions(
VideoDimensions dimensions,
) {
final double aspect = dimensions.width / dimensions.height;
static List<VideoParameters> _presetsForDimensions({
required bool isScreenShare,
required VideoDimensions dimensions,
}) {
if (isScreenShare) return VideoParameters.presetsScreenShare;
final double aspect = dimensions.width > dimensions.height
? dimensions.width / dimensions.height
: dimensions.height / dimensions.width;
if ((aspect - 16.0 / 9.0).abs() < (aspect - 4.0 / 3.0).abs()) {
return VideoParameters.presets169;
}
return VideoParameters.presets43;
}
static VideoParameters _findPresetForDimensions(
VideoDimensions dimensions, {
static VideoEncoding _findAppropriateEncoding({
required bool isScreenShare,
required VideoDimensions dimensions,
required List<VideoParameters> presets,
}) {
assert(presets.isNotEmpty, 'presets should not be empty');
VideoParameters result = presets.first;
VideoEncoding result = presets.first.encoding;
// handle portrait by swapping dimensions
final size = dimensions.max();
for (final preset in presets) {
if (dimensions.width >= preset.dimensions.width &&
dimensions.height >= preset.dimensions.height) result = preset;
result = preset.encoding;
if (preset.dimensions.width >= size) break;
}
return result;
}
static final videoRids = ['q', 'h', 'f'];
static List<rtc.RTCRtpEncoding> encodingsFromPresets(
VideoDimensions dimensions, {
required List<VideoParameters> presets,
}) {
List<rtc.RTCRtpEncoding> result = [];
presets.forEachIndexed((i, e) {
if (i >= videoRids.length) {
return;
}
final size = dimensions.max();
final rid = videoRids[i];
result.add(e.encoding.toRTCRtpEncoding(
rid: rid,
scaleResolutionDownBy: size / e.dimensions.height,
));
});
return result;
}
static List<rtc.RTCRtpEncoding>? computeVideoEncodings({
required bool isScreenShare,
VideoDimensions? dimensions,
VideoPublishOptions? options,
}) {
@@ -87,53 +122,94 @@ class Utils {
VideoEncoding? videoEncoding = options.videoEncoding;
if ((videoEncoding == null && !options.simulcast) || dimensions == null) {
final useSimulcast = !isScreenShare && options.simulcast;
if ((videoEncoding == null && !useSimulcast) || dimensions == null) {
// don't set encoding when we are not simulcasting and user isn't restricting
// encoding parameters
return null;
}
final presets = _presetsForDimensions(dimensions);
final presets = _presetsForDimensions(
isScreenShare: isScreenShare,
dimensions: dimensions,
);
if (videoEncoding == null) {
// find the right encoding based on width/height
final preset = _findPresetForDimensions(dimensions, presets: presets);
// print('Using preset: ${preset.id}');
videoEncoding = preset.encoding;
// log.debug('using video encoding', videoEncoding);
videoEncoding = _findAppropriateEncoding(
isScreenShare: isScreenShare,
dimensions: dimensions,
presets: presets,
);
logger.fine('using video encoding', videoEncoding);
}
// Not simulcast
if (!options.simulcast) return [videoEncoding.toRTCRtpEncoding()];
if (!useSimulcast) return [videoEncoding.toRTCRtpEncoding()];
// Compute for simulcast
final midPreset = presets[1];
final lowPreset = presets[0];
return [
videoEncoding.toRTCRtpEncoding(
rid: 'f',
),
// if resolution is high enough, we would send both h and q res..
// otherwise only send h
if (dimensions.max() >= 960) ...[
midPreset.encoding.toRTCRtpEncoding(
rid: 'h',
// passing decimals to hardware encoder of android devices
// often causes issues so we better use integers
scaleResolutionDownBy: 2,
),
lowPreset.encoding.toRTCRtpEncoding(
rid: 'q',
scaleResolutionDownBy: 4,
),
] else
lowPreset.encoding.toRTCRtpEncoding(
rid: 'h',
scaleResolutionDownBy: 2,
),
];
final VideoParameters lowPreset = presets.first;
VideoParameters? midPreset;
if (presets.length > 1) {
midPreset = presets[1];
}
final original = VideoParameters(
dimensions: dimensions,
encoding: videoEncoding,
);
final size = dimensions.max();
List<VideoParameters> computedPresets = [original];
if (size >= 960 && midPreset != null) {
computedPresets = [lowPreset, midPreset, original];
} else if (size >= 500) {
computedPresets = [lowPreset, original];
}
return encodingsFromPresets(
dimensions,
presets: computedPresets,
);
}
static List<lk_models.VideoLayer> computeVideoLayers(
VideoDimensions dimensions,
List<rtc.RTCRtpEncoding>? encodings,
) {
// default to a single layer, HQ
if (encodings == null) {
return [
lk_models.VideoLayer(
quality: lk_models.VideoQuality.HIGH,
width: dimensions.width,
height: dimensions.height,
bitrate: 0,
)
];
}
return encodings.map((e) {
final scale = e.scaleResolutionDownBy ?? 1;
var quality = videoQualityForRid(e.rid);
if (quality == null && encodings.length == 1) {
quality = lk_models.VideoQuality.HIGH;
}
return lk_models.VideoLayer(
quality: quality,
width: (dimensions.width.toDouble() / scale).floor(),
height: (dimensions.height.toDouble() / scale).floor(),
bitrate: e.maxBitrate ?? 0,
);
}).toList();
}
static lk_models.VideoQuality? videoQualityForRid(String? rid) => {
'f': lk_models.VideoQuality.HIGH,
'h': lk_models.VideoQuality.MEDIUM,
'q': lk_models.VideoQuality.LOW,
}[rid];
// makes a debounce func, with 1 param
static Function(T) createDebounceFunc<T>(
Function(T) f, {