Simulcast, Screen sharing & Various improvements (#4)

* Respect `RTCIceTransportPolicy` enum and organize

* Simplify syntax where possible etc.

* Combine `VideoPreset` and `VideoPresets`

* Default values for `ConnectOptions`

* Build URI instead of String manipulation

* Slight modifications to Exception

* `LiveKitTheme` for example

* `VideoEncoding` class

* Organize imports

* First simulcast implementation

* Remove unnecessary try-catches

* Update Android settings

* Remember uri and token

* example improvements

* `fit` parameter for VideoTrackRenderer

* Simulcast option for example

* Pass `defaultPublishOptions`

* Show only `VideoQuality`

* Pass tests

* Better buildUri logic

* Named parameter to positional

* Explicit imports

* `VideoParameter` instead of `VideoPreset`

* Use `mediaTrack.getSettings` when possible

* Safer dispose logic

* Safer `PCTransport`

Update transport.dart

* Synchronized events for `SignalClient`

* Use logger instead of print

* Make example compile for iOS

* First screen share implementation

* Make example work with screen share

* Example improvement

* Code optimization

* Don't depend on web_socket_channel

* Fix: Unpublish track bug

* Show participant mute state & identity

* Update protos

* Remote mute/unmute

* iOS Background mode

* Separate `createCameraTrack` and `createScreenTrack`

* Clean up

* PB fix

* format

* Fix analyzer warning

* Android clean up

* Update README.md

* Clean up
This commit is contained in:
Hiroshi Horie
2021-09-11 03:14:14 +09:00
committed by GitHub
parent 8e66e324f6
commit 50f56c5e1e
69 changed files with 2653 additions and 2351 deletions
+7 -7
View File
@@ -1,29 +1,29 @@
import 'package:flutter_webrtc/flutter_webrtc.dart';
import '../proto/livekit_models.pbenum.dart';
import '../proto/livekit_models.pb.dart' as lk_models;
import '_audio_api.dart' if (dart.library.html) '_audio_html.dart' as audio;
import 'local_audio_track.dart';
import 'track.dart';
import '_audio_api.dart' if (dart.library.html) '_audio_html.dart' as audio;
class AudioTrack extends Track {
MediaStream? mediaStream;
AudioTrack(String name, MediaStreamTrack track, this.mediaStream)
: super(TrackType.AUDIO, name, track);
: super(lk_models.TrackType.AUDIO, name, track);
/// Start playing audio track. On web platform, create an audio element and
/// start playback
void start() {
if (this is! LocalAudioTrack) {
audio.startAudio(getCid(), mediaTrack);
audio.startAudio(getCid(), mediaStreamTrack);
}
}
@override
void stop() {
mediaStream?.dispose();
Future<void> stop() async {
await mediaStream?.dispose();
mediaStream = null;
audio.stopAudio(getCid());
super.stop();
await super.stop();
}
}
+16 -16
View File
@@ -1,28 +1,28 @@
import 'dart:async';
import 'package:flutter_webrtc/flutter_webrtc.dart';
import 'package:livekit_client/src/track/audio_track.dart';
import '../errors.dart';
import 'audio_track.dart';
import 'options.dart';
class LocalAudioTrack extends AudioTrack {
LocalAudioTrack(String name, MediaStreamTrack track, MediaStream stream)
: super(name, track, stream);
LocalAudioTrack(
String name,
MediaStreamTrack track,
MediaStream stream,
) : super(name, track, stream);
/// Creates a new audio track from the default audio input device.
static Future<LocalAudioTrack> createTrack([LocalAudioTrackOptions? options]) async {
try {
final stream = await navigator.mediaDevices.getUserMedia(<String, dynamic>{
'audio': true,
'video': false,
});
static Future<LocalAudioTrack> create([LocalAudioTrackOptions? options]) async {
// try {
final stream = await navigator.mediaDevices.getUserMedia(<String, dynamic>{
'audio': true,
'video': false,
});
if (stream.getAudioTracks().isEmpty) {
return Future.error(TrackCreateError());
}
if (stream.getAudioTracks().isEmpty) throw TrackCreateError();
return LocalAudioTrack('', stream.getAudioTracks().first, stream);
} catch (e) {
return Future.error(e);
}
return LocalAudioTrack('', stream.getAudioTracks().first, stream);
}
}
+12 -6
View File
@@ -1,23 +1,29 @@
import 'package:livekit_client/src/logger.dart';
import '../participant/local_participant.dart';
import '../proto/livekit_models.pb.dart';
import '../proto/livekit_models.pb.dart' as lk_models;
import 'track.dart';
import 'track_publication.dart';
class LocalTrackPublication extends TrackPublication {
final LocalParticipant _participant;
LocalTrackPublication(TrackInfo info, Track track, this._participant) : super.fromInfo(info) {
LocalTrackPublication(
lk_models.TrackInfo info,
Track track,
this._participant,
) : super.fromInfo(info) {
this.track = track;
}
/// Mute or unmute the current track. When muted, track will stop sending data
@override
set muted(bool val) {
if (val == muted) {
return;
}
if (val == muted) return;
logger.finer('setMute: ${val}');
super.muted = val;
track?.mediaTrack.enabled = !val;
track?.mediaStreamTrack.enabled = !val;
_participant.engine.client.sendMuteTrack(sid, val);
if (val) {
+89 -42
View File
@@ -1,66 +1,113 @@
import 'package:flutter_webrtc/flutter_webrtc.dart';
import '../errors.dart';
import '../logger.dart';
import 'options.dart';
import 'track.dart';
import 'video_track.dart';
/// A video track from the local device. Use static methods in this class to create
/// video tracks.
class LocalVideoTrack extends VideoTrack {
//
// Options used for this track
//
LocalVideoTrackOptions currentOptions;
//
// Private constructor
//
LocalVideoTrack._(
String name,
MediaStreamTrack mediaTrack,
MediaStream stream,
this.currentOptions,
) : super(name, mediaTrack, stream);
RTCRtpSender? get sender => transceiver?.sender;
LocalVideoTrack(String name, MediaStreamTrack mediaTrack, MediaStream stream)
: super(name, mediaTrack, stream);
/// Creates a LocalVideoTrack from camera input.
static Future<LocalVideoTrack> createCameraTrack([LocalVideoTrackOptions? options]) async {
options ??= LocalVideoTrackOptions(params: VideoPresets.qhd);
try {
final stream = await _createCameraStream(options);
return LocalVideoTrack('camera', stream.getVideoTracks().first, stream);
} catch (e) {
return Future.error(e);
}
}
/// Restarts the track with new options. This is useful when switching between
/// front and back cameras.
Future<void> restartTrack([LocalVideoTrackOptions? options]) async {
if (sender == null) {
return Future.error(TrackCreateError('could not restart track'));
Future<void> restartTrack([
LocalVideoTrackOptions? options,
]) async {
if (sender == null) throw TrackCreateError('could not restart track');
if (options != null && currentOptions.runtimeType != options.runtimeType) {
throw Exception('options must be a ${currentOptions.runtimeType}');
}
options ??= LocalVideoTrackOptions(params: VideoPresets.qhd);
currentOptions = options ?? currentOptions;
try {
final stream = await _createCameraStream(options);
final track = stream.getVideoTracks().first;
mediaStream = stream;
await mediaTrack.stop();
mediaTrack = track;
await sender?.replaceTrack(track);
} catch (e) {
return Future.error(e);
}
final stream = await _createStream(currentOptions);
final track = stream.getVideoTracks().first;
setMediaStream(stream);
await mediaStreamTrack.stop();
mediaStreamTrack = track;
await sender?.replaceTrack(track);
}
static Future<MediaStream> _createCameraStream(LocalVideoTrackOptions? options) async {
options ??= LocalVideoTrackOptions(params: VideoPresets.qhd);
/// Creates a LocalVideoTrack from camera input.
static Future<LocalVideoTrack> createCameraTrack([
CameraTrackOptions? options,
]) async {
options ??= const CameraTrackOptions();
final stream = await _createStream(options);
return LocalVideoTrack._(
Track.cameraName,
stream.getVideoTracks().first,
stream,
options,
);
}
try {
final stream = await navigator.mediaDevices.getUserMedia(<String, dynamic>{
'audio': false,
'video': options.mediaConstraints,
});
static Future<LocalVideoTrack> createScreenTrack([
ScreenTrackOptions? options,
]) async {
options ??= const ScreenTrackOptions();
final stream = await _createStream(options);
return LocalVideoTrack._(
Track.screenShareName,
stream.getVideoTracks().first,
stream,
options,
);
}
if (stream.getVideoTracks().isEmpty) {
return Future.error(TrackCreateError());
}
static Future<MediaStream> _createStream(
LocalVideoTrackOptions options,
) async {
final constraints = <String, dynamic>{
'audio': false,
'video': options.toMediaConstraintsMap(),
};
return stream;
} catch (e) {
return Future.error(e);
final MediaStream stream;
if (options is ScreenTrackOptions) {
stream = await navigator.mediaDevices.getDisplayMedia(constraints);
} else {
// options is CameraVideoTrackOptions
stream = await navigator.mediaDevices.getUserMedia(constraints);
}
if (stream.getVideoTracks().isEmpty) throw TrackCreateError();
return stream;
}
}
//
// Convenience extensions
//
extension LKLocalVideoTrackExt on LocalVideoTrack {
// Calls restartTrack under the hood
Future<void> setCameraPosition(CameraPosition position) async {
final options = currentOptions;
if (options is! CameraTrackOptions) {
logger.warning('Not a camera track');
return;
}
await restartTrack(
options.copyWith(cameraPosition: position),
);
}
}
+223 -55
View File
@@ -1,26 +1,8 @@
/// Options when creating a LocalVideoTrack.
class LocalVideoTrackOptions {
CameraPosition position = CameraPosition.front;
VideoParameter params;
import 'package:flutter_webrtc/flutter_webrtc.dart';
LocalVideoTrackOptions({
VideoParameter? params,
CameraPosition? position,
}) : params = VideoPresets.qhd {
if (params != null) {
this.params = params;
}
if (position != null) {
this.position = position;
}
}
Map<String, dynamic> get mediaConstraints {
return <String, dynamic>{
'mandatory': params.mediaConstraints,
'facingMode': position == CameraPosition.front ? 'user' : 'environment',
};
}
enum LocalVideoTrackType {
camera,
display,
}
enum CameraPosition {
@@ -28,43 +10,229 @@ enum CameraPosition {
back,
}
class VideoParameter {
int width;
int height;
int fps;
int? bitrate;
VideoParameter(
this.width,
this.height,
this.fps, {
this.bitrate,
});
Map<String, dynamic> get mediaConstraints {
return <String, dynamic>{
'minWidth': width,
'minHeight': height,
'minFrameRate': fps,
};
}
extension LKCameraPositionExt on CameraPosition {
CameraPosition swap() => {
CameraPosition.front: CameraPosition.back,
CameraPosition.back: CameraPosition.front,
}[this]!;
}
class VideoPresets {
static final qvga = VideoParameter(320, 180, 15);
static final vga = VideoParameter(640, 360, 30);
static final qhd = VideoParameter(960, 540, 30);
static final hd = VideoParameter(1280, 720, 30);
static final fhd = VideoParameter(1920, 1080, 30);
class CameraTrackOptions extends LocalVideoTrackOptions {
final CameraPosition cameraPosition;
static final List<VideoParameter> all = [
qvga,
vga,
qhd,
hd,
fhd,
const CameraTrackOptions({
this.cameraPosition = CameraPosition.front,
VideoParameters params = VideoParameters.presetQHD169,
}) : super(params: params);
@override
Map<String, dynamic> toMediaConstraintsMap() => <String, dynamic>{
...super.toMediaConstraintsMap(),
'facingMode': cameraPosition == CameraPosition.front ? 'user' : 'environment',
};
// Returns new options with updated properties
CameraTrackOptions copyWith({
VideoParameters? params,
CameraPosition? cameraPosition,
}) =>
CameraTrackOptions(
params: params ?? this.params,
cameraPosition: cameraPosition ?? this.cameraPosition,
);
}
class ScreenTrackOptions extends LocalVideoTrackOptions {
const ScreenTrackOptions();
}
/// Options when creating a LocalVideoTrack.
abstract class LocalVideoTrackOptions {
// final LocalVideoTrackType type;
final VideoParameters params;
const LocalVideoTrackOptions({
this.params = VideoParameters.presetQHD169,
});
Map<String, dynamic> toMediaConstraintsMap() => <String, dynamic>{
'mandatory': params.toMediaConstraintsMap(),
};
}
class VideoEncoding {
final int maxFramerate;
final int? maxBitrate;
const VideoEncoding({
required this.maxFramerate,
this.maxBitrate,
});
@override
String toString() => '${runtimeType}(maxFramerate: ${maxFramerate}, maxBitrate: ${maxBitrate})';
}
extension VideoEncodingExt on VideoEncoding {
RTCRtpEncoding toRTCRtpEncoding({
String? rid,
double? scaleResolutionDownBy = 1.0,
int? numTemporalLayers,
}) =>
RTCRtpEncoding(
rid: rid,
scaleResolutionDownBy: scaleResolutionDownBy,
maxFramerate: maxFramerate,
maxBitrate: maxBitrate,
numTemporalLayers: numTemporalLayers,
);
}
class VideoParameters {
final String description;
final int width;
final int height;
final VideoEncoding encoding;
const VideoParameters({
required this.description,
required this.width,
required this.height,
required this.encoding,
});
//
// TODO: Make sure the resolutions are correct
//
static const presetQVGA169 = VideoParameters(
description: 'QVGA(320x180) 16:9',
width: 320,
height: 180,
encoding: VideoEncoding(
maxBitrate: 125000,
maxFramerate: 15,
),
);
static const presetVGA169 = VideoParameters(
description: 'VGA(640x360) 16:9',
width: 640,
height: 360,
encoding: VideoEncoding(
maxBitrate: 400000,
maxFramerate: 30,
),
);
static const presetQHD169 = VideoParameters(
description: 'QHD(960x540) 16:9',
width: 960,
height: 540,
encoding: VideoEncoding(
maxBitrate: 800000,
maxFramerate: 30,
),
);
static const presetHD169 = VideoParameters(
description: 'HD(1280x720) 16:9',
width: 1280,
height: 720,
encoding: VideoEncoding(
maxBitrate: 2500000,
maxFramerate: 30,
),
);
static const presetFHD169 = VideoParameters(
description: 'FHD(1920x1080) 16:9',
width: 1920,
height: 1080,
encoding: VideoEncoding(
maxBitrate: 4000000,
maxFramerate: 30,
),
);
static const presetQVGA43 = VideoParameters(
description: 'QVGA(240x180) 4:3',
width: 240,
height: 180,
encoding: VideoEncoding(
maxBitrate: 100000,
maxFramerate: 15,
),
);
static const presetVGA43 = VideoParameters(
description: 'VGA(480x360) 4:3',
width: 480,
height: 360,
encoding: VideoEncoding(
maxBitrate: 320000,
maxFramerate: 30,
),
);
static const presetQHD43 = VideoParameters(
description: 'QHD(720x540) 4:3',
width: 720,
height: 540,
encoding: VideoEncoding(
maxBitrate: 640000,
maxFramerate: 30,
),
);
static const presetHD43 = VideoParameters(
description: 'HD(960x720) 4:3',
width: 960,
height: 720,
encoding: VideoEncoding(
maxBitrate: 2000000,
maxFramerate: 30,
),
);
static const presetFHD43 = VideoParameters(
description: 'FHD(1440x1080) 4:3',
width: 1440,
height: 1080,
encoding: VideoEncoding(
maxBitrate: 3200000,
maxFramerate: 30,
),
);
static final List<VideoParameters> presets169 = [
presetQVGA169,
presetVGA169,
presetQHD169,
presetHD169,
presetFHD169,
];
static final List<VideoParameters> presets43 = [
presetQVGA43,
presetVGA43,
presetQHD43,
presetHD43,
presetFHD43,
];
//
// TODO: Return constraints that will work for all platforms (Web & Mobile)
// https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia
//
Map<String, dynamic> toMediaConstraintsMap() => <String, dynamic>{
'maxWidth': width,
'maxHeight': height,
'maxFrameRate': encoding.maxFramerate,
};
}
/// Options when creating an LocalAudioTrack. Placeholder for now.
class LocalAudioTrackOptions {}
class LocalAudioTrackOptions {
const LocalAudioTrackOptions();
}
+14 -9
View File
@@ -1,6 +1,6 @@
import '../proto/livekit_models.pb.dart';
import '../proto/livekit_rtc.pbserver.dart';
import '../participant/remote_participant.dart';
import '../proto/livekit_models.pb.dart' as lk_models;
import '../proto/livekit_rtc.pb.dart' as lk_rtc;
import 'track.dart';
import 'track_publication.dart';
@@ -10,10 +10,11 @@ class RemoteTrackPublication extends TrackPublication {
final RemoteParticipant _participant;
bool _unsubscribed = false;
bool _disabled = false;
VideoQuality _videoQuality = VideoQuality.HIGH;
lk_rtc.VideoQuality _videoQuality = lk_rtc.VideoQuality.HIGH;
VideoQuality get videoQuality => _videoQuality;
set videoQuality(VideoQuality val) {
lk_rtc.VideoQuality get videoQuality => _videoQuality;
set videoQuality(lk_rtc.VideoQuality val) {
if (val == _videoQuality) return;
_videoQuality = val;
_sendUpdateTrackSettings();
@@ -56,21 +57,25 @@ class RemoteTrackPublication extends TrackPublication {
_participant.roomDelegate?.onTrackUnmuted(_participant, this);
}
if (subscribed) {
track?.mediaTrack.enabled = !val;
track?.mediaStreamTrack.enabled = !val;
}
_participant.muteChanged();
}
RemoteTrackPublication(TrackInfo info, this._participant, [Track? track]) : super.fromInfo(info) {
RemoteTrackPublication(
lk_models.TrackInfo info,
this._participant, [
Track? track,
]) : super.fromInfo(info) {
this.track = track;
}
void _sendUpdateTrackSettings() {
final settings = UpdateTrackSettings(
final settings = lk_rtc.UpdateTrackSettings(
trackSids: [sid],
disabled: _disabled,
);
if (kind == TrackType.VIDEO) {
if (kind == lk_models.TrackType.VIDEO) {
settings.quality = _videoQuality;
}
_participant.client.sendUpdateTrackSettings(settings);
+11 -10
View File
@@ -1,7 +1,7 @@
import 'package:flutter_webrtc/flutter_webrtc.dart';
import 'package:uuid/uuid.dart';
import '../proto/livekit_models.pb.dart';
import '../proto/livekit_models.pb.dart' as lk_models;
class TrackDimension {
int width;
@@ -12,24 +12,25 @@ class TrackDimension {
/// Wrapper around a MediaStreamTrack with additional metadata.
class Track {
static const cameraName = 'camera';
static const screenShareName = 'screen';
String name;
TrackType kind;
MediaStreamTrack mediaTrack;
lk_models.TrackType kind;
MediaStreamTrack mediaStreamTrack;
String? sid;
RTCRtpTransceiver? transceiver;
String? _cid;
Track(this.kind, this.name, this.mediaTrack);
Track(this.kind, this.name, this.mediaStreamTrack);
bool get muted => mediaTrack.muted == null ? false : mediaTrack.muted!;
bool get muted => mediaStreamTrack.muted == null ? false : mediaStreamTrack.muted!;
RTCRtpMediaType get mediaType {
switch (kind) {
case TrackType.AUDIO:
case lk_models.TrackType.AUDIO:
return RTCRtpMediaType.RTCRtpMediaTypeAudio;
case TrackType.VIDEO:
case lk_models.TrackType.VIDEO:
return RTCRtpMediaType.RTCRtpMediaTypeVideo;
// this should never happen
default:
@@ -38,7 +39,7 @@ class Track {
}
String getCid() {
var cid = _cid ?? mediaTrack.id;
var cid = _cid ?? mediaStreamTrack.id;
if (cid == null) {
const uuid = Uuid();
@@ -48,7 +49,7 @@ class Track {
return cid;
}
void stop() {
mediaTrack.stop();
Future<void> stop() async {
await mediaStreamTrack.stop();
}
}
+6 -6
View File
@@ -1,4 +1,4 @@
import '../proto/livekit_models.pb.dart';
import '../proto/livekit_models.pb.dart' as lk_models;
import 'track.dart';
/// Represents a track that's published to the server. This class contains
@@ -7,14 +7,14 @@ class TrackPublication {
Track? track;
String name;
String sid;
TrackType kind;
lk_models.TrackType kind;
bool muted = false;
bool simulcasted = false;
TrackDimension? dimension;
bool get subscribed => track != null;
TrackPublication.fromInfo(TrackInfo info)
TrackPublication.fromInfo(lk_models.TrackInfo info)
: sid = info.sid,
name = info.name,
kind = info.type {
@@ -22,12 +22,12 @@ class TrackPublication {
}
/// True when the track is published with name [Track.screenShareName].
bool get isScreenShare => kind == TrackType.VIDEO && name == Track.screenShareName;
bool get isScreenShare => kind == lk_models.TrackType.VIDEO && name == Track.screenShareName;
void updateFromInfo(TrackInfo info) {
void updateFromInfo(lk_models.TrackInfo info) {
muted = info.muted;
simulcasted = info.simulcast;
if (info.type == TrackType.VIDEO) {
if (info.type == lk_models.TrackType.VIDEO) {
dimension = TrackDimension(info.width, info.height);
}
}
+18 -10
View File
@@ -1,29 +1,37 @@
import 'package:flutter/foundation.dart';
import 'package:flutter/material.dart';
import 'package:flutter_webrtc/flutter_webrtc.dart';
import '../proto/livekit_models.pb.dart';
import '../proto/livekit_models.pb.dart' as lk_models;
import 'track.dart';
/// A video track will notify when its mediaTrack has changed.
class VideoTrack extends Track with ChangeNotifier {
MediaStream? _mediaStream;
MediaStream _mediaStream;
VideoTrack(String name, MediaStreamTrack mediaTrack, this._mediaStream)
: super(TrackType.VIDEO, name, mediaTrack);
VideoTrack(
String name,
MediaStreamTrack mediaTrack,
this._mediaStream,
) : super(
lk_models.TrackType.VIDEO,
name,
mediaTrack,
);
MediaStream? get mediaStream => _mediaStream;
MediaStream get mediaStream => _mediaStream;
/// internal use
/// {@nodoc}
set mediaStream(MediaStream? stream) {
void setMediaStream(MediaStream stream) {
_mediaStream = stream;
notifyListeners();
}
@override
stop() {
super.stop();
_mediaStream?.dispose();
_mediaStream = null;
Future<void> stop() async {
await super.stop();
await _mediaStream.dispose();
// _mediaStream = null;
}
}