Simulcast, Screen sharing & Various improvements (#4)

* Respect `RTCIceTransportPolicy` enum and organize * Simplify syntax where possible etc. * Combine `VideoPreset` and `VideoPresets` * Default values for `ConnectOptions` * Build URI instead of String manipulation * Slight modifications to Exception * `LiveKitTheme` for example * `VideoEncoding` class * Organize imports * First simulcast implementation * Remove unnecessary try-catches * Update Android settings * Remember uri and token * example improvements * `fit` parameter for VideoTrackRenderer * Simulcast option for example * Pass `defaultPublishOptions` * Show only `VideoQuality` * Pass tests * Better buildUri logic * Named parameter to positional * Explicit imports * `VideoParameter` instead of `VideoPreset` * Use `mediaTrack.getSettings` when possible * Safer dispose logic * Safer `PCTransport` Update transport.dart * Synchronized events for `SignalClient` * Use logger instead of print * Make example compile for iOS * First screen share implementation * Make example work with screen share * Example improvement * Code optimization * Don't depend on web_socket_channel * Fix: Unpublish track bug * Show participant mute state & identity * Update protos * Remote mute/unmute * iOS Background mode * Separate `createCameraTrack` and `createScreenTrack` * Clean up * PB fix * format * Fix analyzer warning * Android clean up * Update README.md * Clean up
2021-09-11 03:14:14 +09:00
parent 8e66e324f6
commit 50f56c5e1e
69 changed files with 2653 additions and 2351 deletions
@@ -1,23 +1,28 @@
+import 'package:flutter/foundation.dart';
 import 'package:flutter_webrtc/flutter_webrtc.dart';

 import '../errors.dart';
-import '../proto/livekit_models.pb.dart';
-import '../proto/livekit_rtc.pbserver.dart';
+import '../logger.dart';
+import '../options.dart';
+import '../proto/livekit_models.pb.dart' as lk_models;
 import '../rtc_engine.dart';
 import '../track/local_audio_track.dart';
 import '../track/local_track_publication.dart';
 import '../track/local_video_track.dart';
 import '../track/track.dart';
 import '../track/track_publication.dart';
+import '../utils.dart';
 import 'participant.dart';

 /// Represents the current participant in the room.
 class LocalParticipant extends Participant {
  final RTCEngine _engine;
+  final TrackPublishOptions? defaultPublishOptions;

  LocalParticipant({
    required RTCEngine engine,
-    required ParticipantInfo info,
+    required lk_models.ParticipantInfo info,
+    this.defaultPublishOptions,
  })  : _engine = engine,
        super(info.sid, info.identity) {
    updateFromInfo(info);
@@ -29,97 +34,134 @@ class LocalParticipant extends Participant {

  /// publish an audio track to the room
  Future<TrackPublication> publishAudioTrack(LocalAudioTrack track) async {
-    if (audioTracks.values.any((element) => element.track?.mediaTrack.id == track.mediaTrack.id)) {
-      return Future.error(TrackPublishError('track already exists'));
+    if (audioTracks.any((e) => e.track?.mediaStreamTrack.id == track.mediaStreamTrack.id)) {
+      throw TrackPublishError('track already exists');
    }

-    try {
-      final trackInfo =
-          await _engine.addTrack(cid: track.getCid(), name: track.name, kind: track.kind);
-      final transceiverInit = RTCRtpTransceiverInit(
-        direction: TransceiverDirection.SendOnly,
-      );
-      // addTransceiver cannot pass in a kind parameter due to a bug in flutter-webrtc (web)
-      track.transceiver = await _engine.publisher?.pc.addTransceiver(
-        track: track.mediaTrack,
-        init: transceiverInit,
-      );
+    // try {
+    final trackInfo = await _engine.addTrack(
+      cid: track.getCid(),
+      name: track.name,
+      kind: track.kind,
+    );

-      final pub = LocalTrackPublication(trackInfo, track, this);
-      addTrackPublication(pub);
-      notifyListeners();
+    final transceiverInit = RTCRtpTransceiverInit(
+      direction: TransceiverDirection.SendOnly,
+    );
+    // addTransceiver cannot pass in a kind parameter due to a bug in flutter-webrtc (web)
+    track.transceiver = await _engine.publisher?.pc.addTransceiver(
+      track: track.mediaStreamTrack,
+      init: transceiverInit,
+    );

-      return pub;
-    } catch (e) {
-      return Future.error(e);
-    }
+    final pub = LocalTrackPublication(trackInfo, track, this);
+    addTrackPublication(pub);
+    notifyListeners();
+
+    return pub;
  }

  /// Publish a video track to the room
-  Future<TrackPublication> publishVideoTrack(LocalVideoTrack track) async {
-    if (videoTracks.values.any((element) => element.track?.mediaTrack.id == track.mediaTrack.id)) {
-      return Future.error(TrackPublishError('track already exists'));
+  Future<TrackPublication> publishVideoTrack(
+    LocalVideoTrack track, {
+    TrackPublishOptions? options,
+  }) async {
+    if (videoTracks.any((e) => e.track?.mediaStreamTrack.id == track.mediaStreamTrack.id)) {
+      throw TrackPublishError('track already exists');
    }

-    try {
-      final trackInfo =
-          await _engine.addTrack(cid: track.getCid(), name: track.name, kind: track.kind);
-      final transceiverInit = RTCRtpTransceiverInit(
-        direction: TransceiverDirection.SendOnly,
-      );
-      // TODO: video encodings and simulcasts
-      // addTransceiver cannot pass in a kind parameter due to a bug in flutter-webrtc (web)
-      track.transceiver = await _engine.publisher?.pc.addTransceiver(
-        track: track.mediaTrack,
-        init: transceiverInit,
-      );
+    // Use default options from `ConnectOptions` if options is null
+    options = options ?? defaultPublishOptions;

-      final pub = LocalTrackPublication(trackInfo, track, this);
-      addTrackPublication(pub);
-      notifyListeners();
+    final trackInfo = await _engine.addTrack(
+      cid: track.getCid(),
+      name: track.name,
+      kind: track.kind,
+    );

-      return pub;
-    } catch (e) {
-      return Future.error(e);
+    //
+    // Video encodings and simulcasts
+    //
+
+    // use constraints passed to getUserMedia by default
+    int? width = track.currentOptions.params.width;
+    int? height = track.currentOptions.params.height;
+
+    if (kIsWeb) {
+      // getSettings() is only implemented for Web
+      try {
+        // try to use getSettings for more accurate resolution
+        final settings = track.mediaStreamTrack.getSettings();
+        width = settings['width'] as int?;
+        height = settings['height'] as int?;
+        // TODO: Get actual video dimensions to compute more accurately
+        // mediaTrack.getConsstraints() is not implemented for mobile
+      } catch (_) {
+        logger.warning('Failed to call `mediaStreamTrack.getSettings()`');
+      }
    }
+
+    logger.fine('Compute encodings with resolution: ${width}x${height}, options: ${options}');
+
+    final encodings = Utils.computeVideoEncodings(
+      width: width,
+      height: height,
+      options: options,
+    );
+
+    logger.fine('Using encodings: ${encodings?.map((e) => e.toMap())}');
+
+    final transceiverInit = RTCRtpTransceiverInit(
+      direction: TransceiverDirection.SendOnly,
+      sendEncodings: encodings,
+      streams: [track.mediaStream],
+    );
+
+    //
+    // addTransceiver cannot pass in a kind parameter due to a bug in flutter-webrtc (web)
+    //
+    track.transceiver = await _engine.publisher?.pc.addTransceiver(
+      track: track.mediaStreamTrack,
+      init: transceiverInit,
+    );
+
+    final pub = LocalTrackPublication(trackInfo, track, this);
+    addTrackPublication(pub);
+    notifyListeners();
+
+    return pub;
  }

  /// Unpublish a track that's already published
-  void unpublishTrack(Track track) {
+  Future<void> unpublishTrack(Track track) async {
    final existing = tracks.values.where((element) => element.track == track);
-    if (existing.isEmpty) {
-      return;
-    }
+    if (existing.isEmpty) return;
+
    final pub = existing.first;

-    track.stop();
+    await track.stop();
+
    final sender = track.transceiver?.sender;
    if (sender != null) {
-      engine.publisher?.pc.removeTrack(sender);
+      await engine.publisher?.pc.removeTrack(sender);
    }

    tracks.remove(pub.sid);
-    switch (pub.kind) {
-      case TrackType.AUDIO:
-        audioTracks.remove(pub.sid);
-        break;
-      case TrackType.VIDEO:
-        videoTracks.remove(pub.sid);
-        break;
-      default:
-        break;
-    }
  }

  /// Publish a new data payload to the room.
  /// @param destinationSids When empty, data will be forwarded to each participant in the room.
-  void publishData(List<int> data, DataPacket_Kind reliability, {List<String>? destinationSids}) {
+  void publishData(
+    List<int> data,
+    lk_models.DataPacket_Kind reliability, {
+    List<String>? destinationSids,
+  }) {
    RTCDataChannel? channel;
    switch (reliability) {
-      case DataPacket_Kind.RELIABLE:
+      case lk_models.DataPacket_Kind.RELIABLE:
        channel = engine.reliableDC;
        break;
-      case DataPacket_Kind.LOSSY:
+      case lk_models.DataPacket_Kind.LOSSY:
        channel = engine.lossyDC;
        break;
    }
@@ -127,9 +169,9 @@ class LocalParticipant extends Participant {
      return;
    }

-    final packet = DataPacket(
+    final packet = lk_models.DataPacket(
      kind: reliability,
-      user: UserPacket(
+      user: lk_models.UserPacket(
        payload: data,
        participantSid: sid,
        destinationSids: destinationSids,
@@ -143,7 +185,7 @@ class LocalParticipant extends Participant {
  /// for internal use
  /// {@nodoc}
  @override
-  void updateFromInfo(ParticipantInfo info) {
+  void updateFromInfo(lk_models.ParticipantInfo info) {
    super.updateFromInfo(info);
  }
 }
@@ -1,10 +1,10 @@
 import 'package:flutter/foundation.dart';

-import 'remote_participant.dart';
-import '../proto/livekit_models.pb.dart';
+import '../proto/livekit_models.pb.dart' as lk_models;
 import '../track/remote_track_publication.dart';
 import '../track/track.dart';
 import '../track/track_publication.dart';
+import 'remote_participant.dart';

 /// Callbacks for participant changes
 mixin ParticipantDelegate {
@@ -51,9 +51,6 @@ mixin ParticipantDelegate {
 /// - added/removed subscribed tracks
 /// - metadata changed
 class Participant extends ChangeNotifier {
-  Map<String, TrackPublication> audioTracks = {};
-  Map<String, TrackPublication> videoTracks = {};
-
  /// map of track sid => published track
  Map<String, TrackPublication> tracks = {};

@@ -77,7 +74,7 @@ class Participant extends ChangeNotifier {
  /// delegate to receive participant callbacks
  ParticipantDelegate? delegate;

-  ParticipantInfo? _participantInfo;
+  lk_models.ParticipantInfo? _participantInfo;
  bool _isSpeaking = false;

  /// when the participant joined the room
@@ -94,10 +91,8 @@ class Participant extends ChangeNotifier {

  /// true if participant is publishing an audio track and is muted
  bool get isMuted {
-    if (audioTracks.values.isEmpty) {
-      return false;
-    }
-    return audioTracks.values.first.muted;
+    if (audioTracks.isEmpty) return false;
+    return audioTracks.first.muted;
  }

  bool get hasAudio => audioTracks.isNotEmpty;
@@ -105,15 +100,7 @@ class Participant extends ChangeNotifier {
  bool get hasVideo => videoTracks.isNotEmpty;

  /// tracks that are subscribed to
-  List<TrackPublication> get subscribedTracks {
-    List<TrackPublication> result = [];
-    for (final track in tracks.values) {
-      if (track.subscribed) {
-        result.add(track);
-      }
-    }
-    return result;
-  }
+  List<TrackPublication> get subscribedTracks => tracks.values.where((e) => e.subscribed).toList();

  /// for internal use
  /// {@nodoc}
@@ -148,7 +135,7 @@ class Participant extends ChangeNotifier {

  /// for internal use
  /// {@nodoc}
-  void updateFromInfo(ParticipantInfo info) {
+  void updateFromInfo(lk_models.ParticipantInfo info) {
    identity = info.identity;
    sid = info.sid;
    if (info.metadata.isNotEmpty) {
@@ -168,15 +155,14 @@ class Participant extends ChangeNotifier {
  void addTrackPublication(TrackPublication pub) {
    pub.track?.sid = pub.sid;
    tracks[pub.sid] = pub;
-    switch (pub.kind) {
-      case TrackType.AUDIO:
-        audioTracks[pub.sid] = pub;
-        break;
-      case TrackType.VIDEO:
-        videoTracks[pub.sid] = pub;
-        break;
-      default:
-      // nothing
-    }
  }
 }
+
+// Convenience extension
+extension LKParticipantExt on Participant {
+  List<TrackPublication> get videoTracks =>
+      tracks.values.where((e) => e.kind == lk_models.TrackType.VIDEO).toList();
+
+  List<TrackPublication> get audioTracks =>
+      tracks.values.where((e) => e.kind == lk_models.TrackType.AUDIO).toList();
+}
@@ -1,7 +1,9 @@
 import 'package:flutter_webrtc/flutter_webrtc.dart';
-import 'package:livekit_client/src/track/audio_track.dart';
-import '../proto/livekit_models.pb.dart';
+
+import '../logger.dart';
+import '../proto/livekit_models.pb.dart' as lk_models;
 import '../signal_client.dart';
+import '../track/audio_track.dart';
 import '../track/remote_track_publication.dart';
 import '../track/track.dart';
 import '../track/video_track.dart';
@@ -13,17 +15,22 @@ class RemoteParticipant extends Participant {

  SignalClient get client => _client;

-  RemoteParticipant(this._client, String sid, String identity) : super(sid, identity);
+  RemoteParticipant(
+    this._client,
+    String sid,
+    String identity,
+  ) : super(sid, identity);

-  RemoteParticipant.fromInfo(this._client, ParticipantInfo info) : super(info.sid, info.identity) {
+  RemoteParticipant.fromInfo(
+    this._client,
+    lk_models.ParticipantInfo info,
+  ) : super(info.sid, info.identity) {
    updateFromInfo(info);
  }

  RemoteTrackPublication? getTrackPublication(String sid) {
    final pub = tracks[sid];
-    if (pub is RemoteTrackPublication) {
-      return pub;
-    }
+    if (pub is RemoteTrackPublication) return pub;
  }

  /// for internal use
@@ -49,11 +56,11 @@ class RemoteParticipant extends Participant {
    }

    Track? track;
-    if (pub.kind == TrackType.AUDIO) {
+    if (pub.kind == lk_models.TrackType.AUDIO) {
      final audioTrack = AudioTrack(pub.name, mediaTrack, stream);
      audioTrack.start();
      track = audioTrack;
-    } else if (pub.kind == TrackType.VIDEO) {
+    } else if (pub.kind == lk_models.TrackType.VIDEO) {
      track = VideoTrack(pub.name, mediaTrack, stream);
    } else {
      final msg = 'unsupported track type ${pub.kind}';
@@ -73,7 +80,7 @@ class RemoteParticipant extends Participant {
  /// for internal use
  /// {@nodoc}
  @override
-  void updateFromInfo(ParticipantInfo info) {
+  void updateFromInfo(lk_models.ParticipantInfo info) async {
    final hadInfo = hasInfo;
    super.updateFromInfo(info);

@@ -105,30 +112,28 @@ class RemoteParticipant extends Participant {
    }

    // remove tracks
-    for (final pub in tracks.values) {
-      if (!validPubs.containsKey(pub.sid)) {
-        unpublishTrack(sid, true);
-      }
+    final removeTrackSids =
+        tracks.values.where((e) => !validPubs.containsKey(e.sid)).map((e) => e.sid).toList();
+
+    for (final sid in removeTrackSids) {
+      await unpublishTrack(sid, true);
    }
  }

-  void unpublishTrack(String sid, [bool sendUnpublish = false]) {
+  Future<void> unpublishTrack(String sid, [bool notify = false]) async {
+    logger.finer('Unpublish track sid: $sid, notify: $notify');
    final pub = tracks.remove(sid);
-    if (pub == null || pub is! RemoteTrackPublication) {
-      return;
-    }
-
-    audioTracks.remove(sid);
-    videoTracks.remove(sid);
+    if (pub == null || pub is! RemoteTrackPublication) return;

    final track = pub.track;
    if (track != null) {
-      track.stop();
+      await track.stop();
      delegate?.onTrackUnsubscribed(this, track, pub);
      roomDelegate?.onTrackUnsubscribed(this, track, pub);
      notifyListeners();
    }
-    if (sendUnpublish) {
+
+    if (notify) {
      delegate?.onTrackUnpublished(this, pub);
      roomDelegate?.onTrackUnpublished(this, pub);
    }
@@ -141,9 +146,8 @@ class RemoteParticipant extends Participant {
          await Future<RemoteTrackPublication?>.delayed(const Duration(milliseconds: 100), () {
        return getTrackPublication(sid);
      });
-      if (pub != null) {
-        return pub;
-      }
+
+      if (pub != null) return pub;
    }
  }
 }