video rendering, local track creation

lib/livekit_client.dart

@@ -16,3 +16,4 @@ export 'src/track/local_video_track.dart';
 export 'src/track/track_publication.dart';
 export 'src/track/local_track_publication.dart';
 export 'src/track/remote_track_publication.dart';
+export 'src/widget/video_track_renderer.dart';

lib/src/errors.dart

@@ -13,6 +13,10 @@ class ConnectError extends LiveKitError {
   ConnectError([String msg = 'Failed to connect to server']) : super(msg);
 }
 
+class TrackCreateError extends LiveKitError {
+  TrackCreateError([String msg = 'Failed to create track']) : super(msg);
+}
+
 class TrackPublishError extends LiveKitError {
   TrackPublishError([String msg = 'Failed to publish track']) : super(msg);
 }

lib/src/participant/remote_participant.dart

@@ -26,7 +26,8 @@ class RemoteParticipant extends Participant {
     }
   }
 
-  addSubscribedMediaTrack(MediaStreamTrack mediaTrack, String? sid) async {
+  addSubscribedMediaTrack(
+      MediaStreamTrack mediaTrack, MediaStream stream, String? sid) async {
     if (sid == null) {
       var msg = 'addSubscribedMediaTrack received null sid';
       delegate?.onTrackSubscriptionFailed(this, '', msg);
@@ -50,7 +51,7 @@ class RemoteParticipant extends Participant {
     if (pub.kind == TrackType.AUDIO) {
       track = new Track(pub.kind, pub.name, mediaTrack);
     } else if (pub.kind == TrackType.VIDEO) {
-      track = new VideoTrack(pub.name, mediaTrack);
+      track = new VideoTrack(pub.name, mediaTrack, stream);
     } else {
       var msg = 'unsupported track type ${pub.kind}';
       delegate?.onTrackSubscriptionFailed(this, sid, msg);

lib/src/room.dart

@@ -144,7 +144,7 @@ class Room extends ChangeNotifier with ParticipantDelegate {
 
   RemoteParticipant _getOrCreateRemoteParticipant(
       String sid, ParticipantInfo? info) {
-    var participant = participants[sid];
+    var participant = _participants[sid];
     if (participant != null) {
       return participant;
     }
@@ -155,7 +155,7 @@ class Room extends ChangeNotifier with ParticipantDelegate {
       participant = RemoteParticipant.fromInfo(_engine.client, info);
     }
     participant.roomDelegate = this;
-    participants[sid] = participant;
+    _participants[sid] = participant;
 
     return participant;
   }
@@ -283,7 +283,7 @@ class Room extends ChangeNotifier with ParticipantDelegate {
     }
 
     var participant = _getOrCreateRemoteParticipant(parsed.item1, null);
-    participant.addSubscribedMediaTrack(track, trackSid);
+    participant.addSubscribedMediaTrack(track, stream, trackSid);
   }
 
   _handleParticipantDisconnect(String sid) {
@@ -292,7 +292,8 @@ class Room extends ChangeNotifier with ParticipantDelegate {
       return;
     }
 
-    for (var track in participant.tracks.values) {
+    var toRemove = List.from(participant.tracks.values);
+    for (var track in toRemove) {
       participant.unpublishTrack(track.sid, true);
     }
     delegate?.onParticipantDisconnected(participant);

lib/src/signal_client.dart

@@ -1,6 +1,5 @@
 import 'dart:convert';
 import 'dart:developer';
-import 'dart:typed_data';
 
 import 'package:flutter_webrtc/flutter_webrtc.dart';
 import 'package:web_socket_channel/web_socket_channel.dart';

lib/src/track/local_audio_track.dart

@@ -1,8 +1,30 @@
 import 'package:flutter_webrtc/flutter_webrtc.dart';
 
+import '../errors.dart';
 import '../proto/livekit_models.pb.dart';
+import 'options.dart';
 import 'track.dart';
 
 class LocalAudioTrack extends Track {
-  LocalAudioTrack(String name, MediaStreamTrack track) : super(TrackType.AUDIO, name, track);
+  MediaStream mediaStream;
+
+  LocalAudioTrack(String name, MediaStreamTrack track, this.mediaStream)
+      : super(TrackType.AUDIO, name, track);
+
+  Future<LocalAudioTrack> createTrack(LocalAudioTrackOptions? options) async {
+    try {
+      var stream = await navigator.mediaDevices.getUserMedia({
+        "audio": true,
+        "video": false,
+      });
+
+      if (stream.getAudioTracks().length == 0) {
+        return Future.error(TrackCreateError());
+      }
+
+      return LocalAudioTrack("", stream.getVideoTracks().first, stream);
+    } catch (e) {
+      return Future.error(e);
+    }
+  }
 }

lib/src/track/local_video_track.dart

@@ -1,8 +1,32 @@
 import 'package:flutter_webrtc/flutter_webrtc.dart';
 
+import '../errors.dart';
+import 'options.dart';
 import 'video_track.dart';
 
 class LocalVideoTrack extends VideoTrack {
-  LocalVideoTrack(String name, MediaStreamTrack mediaTrack)
-      : super(name, mediaTrack);
+  LocalVideoTrack(String name, MediaStreamTrack mediaTrack, MediaStream stream)
+      : super(name, mediaTrack, stream);
+
+  Future<LocalVideoTrack> createCameraTrack(
+      LocalVideoTrackOptions? options) async {
+    if (options == null) {
+      options = LocalVideoTrackOptions(params: VideoPresets.qhd);
+    }
+
+    try {
+      var stream = await navigator.mediaDevices.getUserMedia({
+        "audio": false,
+        "video": options.mediaConstraints,
+      });
+
+      if (stream.getVideoTracks().length == 0) {
+        return Future.error(TrackCreateError());
+      }
+
+      return LocalVideoTrack("", stream.getVideoTracks().first, stream);
+    } catch (e) {
+      return Future.error(e);
+    }
+  }
 }

lib/src/track/options.dart

@@ -0,0 +1,65 @@
+class LocalVideoTrackOptions {
+  CameraPosition position = CameraPosition.FRONT;
+  VideoParameter params;
+
+  LocalVideoTrackOptions({
+    required VideoParameter params,
+    CameraPosition? position,
+  }) : this.params = params {
+    if (position != null) {
+      this.position = position;
+    }
+  }
+
+  Map<String, dynamic> get mediaConstraints {
+    return {
+      "mandatory": params.mediaConstraints,
+      "facingMode": position == CameraPosition.FRONT ? "user" : "environment",
+    };
+  }
+}
+
+enum CameraPosition {
+  FRONT,
+  BACK,
+}
+
+class VideoParameter {
+  int width;
+  int height;
+  int fps;
+  int? bitrate;
+
+  VideoParameter(
+    this.width,
+    this.height,
+    this.fps, {
+    int? bitrate,
+  }) : this.bitrate = bitrate;
+
+  Map<String, dynamic> get mediaConstraints {
+    return {
+      "minWidth": this.width,
+      "minHeight": this.height,
+      "minFrameRate": this.fps,
+    };
+  }
+}
+
+class VideoPresets {
+  static final qvga = VideoParameter(320, 180, 15);
+  static final vga = VideoParameter(640, 360, 30);
+  static final qhd = VideoParameter(960, 540, 30);
+  static final hd = VideoParameter(1280, 720, 30);
+  static final fhd = VideoParameter(1920, 1080, 30);
+
+  static final List<VideoParameter> all = [
+    qvga,
+    vga,
+    qhd,
+    hd,
+    fhd,
+  ];
+}
+
+class LocalAudioTrackOptions {}

lib/src/track/track.dart

@@ -11,6 +11,8 @@ class TrackDimension {
 }
 
 class Track {
+  static const ScreenShareName = "screen";
+
   String name;
   TrackType kind;
   MediaStreamTrack mediaTrack;

lib/src/track/track_publication.dart

@@ -19,6 +19,9 @@ class TrackPublication {
     updateFromInfo(info);
   }
 
+  bool get isScreenShare =>
+      kind == TrackType.VIDEO && name == Track.ScreenShareName;
+
   updateFromInfo(TrackInfo info) {
     muted = info.muted;
     simulcasted = info.simulcast;

lib/src/track/video_track.dart

@@ -4,14 +4,8 @@ import '../proto/livekit_models.pb.dart';
 import 'track.dart';
 
 class VideoTrack extends Track {
-  VideoTrack(String name, MediaStreamTrack mediaTrack)
+  MediaStream mediaStream;
+
+  VideoTrack(String name, MediaStreamTrack mediaTrack, this.mediaStream)
       : super(TrackType.VIDEO, name, mediaTrack);
-
-  // TODO: keep list of video renderers
-
-  @override
-  stop() {
-    super.stop();
-    // TODO: remove renderer
-  }
 }

lib/src/widget/video_track_renderer.dart

@@ -0,0 +1,49 @@
+import 'package:flutter/material.dart';
+import 'package:flutter_webrtc/flutter_webrtc.dart';
+import '../track/video_track.dart';
+import '../track/local_video_track.dart';
+
+class VideoTrackRenderer extends StatefulWidget {
+  final VideoTrack track;
+  final RTCVideoRenderer renderer;
+
+  VideoTrackRenderer(this.track)
+      : renderer = RTCVideoRenderer(),
+        super(key: ValueKey(track.sid));
+
+  @override
+  State<StatefulWidget> createState() {
+    return _VideoTrackRendererState();
+  }
+}
+
+class _VideoTrackRendererState extends State<VideoTrackRenderer> {
+  final _renderer = RTCVideoRenderer();
+
+  @override
+  void initState() {
+    super.initState();
+    _initRenderer();
+  }
+
+  @override
+  void dispose() {
+    _renderer.dispose();
+    super.dispose();
+  }
+
+  _initRenderer() async {
+    await _renderer.initialize();
+    _renderer.srcObject = widget.track.mediaStream;
+  }
+
+  @override
+  Widget build(BuildContext context) {
+    var isLocal = widget.track is LocalVideoTrack;
+    return RTCVideoView(
+      _renderer,
+      mirror: isLocal,
+      filterQuality: FilterQuality.medium,
+    );
+  }
+}