feat: Stats monitor for Track. (#290)

* feat: Stats monitor for Track.

* fix analyze.

* fix.

* import sorter.

* add monitor layer for example.

* update.

* update.

* fix import sorter.

* update.

* import sorter.

* fix typo for filename.

* revert changes for VideoPublishOptions.videoCodec.

* chore: improve code.

* update.
This commit is contained in:
CloudWebRTC
2023-06-01 16:15:37 +08:00
committed by GitHub
parent f21236b213
commit 272484ce7e
11 changed files with 803 additions and 36 deletions
+55
View File
@@ -7,6 +7,7 @@ import 'participant/remote.dart';
import 'publication/local.dart';
import 'publication/remote.dart';
import 'publication/track_publication.dart';
import 'track/stats.dart';
import 'track/track.dart';
import 'types/other.dart';
import 'types/participant_permissions.dart';
@@ -435,3 +436,57 @@ class AudioPlaybackStatusChanged with RoomEvent {
String toString() => '${runtimeType}'
'Audio Playback Status Changed, isPlaying: ${isPlaying})';
}
class AudioSenderStatsEvent with TrackEvent {
final AudioSenderStats stats;
final num currentBitrate;
const AudioSenderStatsEvent({
required this.stats,
required this.currentBitrate,
});
@override
String toString() => '${runtimeType}'
'stats: ${stats})';
}
class VideoSenderStatsEvent with TrackEvent {
final Map<String, VideoSenderStats> stats;
final Map<String, num> bitrateForLayers;
final num currentBitrate;
const VideoSenderStatsEvent({
required this.stats,
required this.currentBitrate,
required this.bitrateForLayers,
});
@override
String toString() => '${runtimeType}'
'stats: ${stats})';
}
class AudioReceiverStatsEvent with TrackEvent {
final AudioReceiverStats stats;
final num currentBitrate;
const AudioReceiverStatsEvent({
required this.stats,
required this.currentBitrate,
});
@override
String toString() => '${runtimeType}'
'stats: ${stats})';
}
class VideoReceiverStatsEvent with TrackEvent {
final VideoReceiverStats stats;
final num currentBitrate;
const VideoReceiverStatsEvent({
required this.stats,
required this.currentBitrate,
});
@override
String toString() => '${runtimeType}'
'stats: ${stats})';
}
+56 -1
View File
@@ -1,12 +1,15 @@
import 'dart:async';
import 'package:collection/collection.dart';
import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
import 'package:meta/meta.dart';
import '../../events.dart';
import '../../proto/livekit_models.pb.dart' as lk_models;
import '../../types/other.dart';
import '../audio_management.dart';
import '../options.dart';
import '../stats.dart';
import 'local.dart';
class LocalAudioTrack extends LocalTrack
@@ -25,6 +28,58 @@ class LocalAudioTrack extends LocalTrack
}
}
num? _currentBitrate;
get currentBitrate => _currentBitrate;
AudioSenderStats? prevStats;
@override
Future<void> monitorSender() async {
if (sender == null) {
_currentBitrate = 0;
return;
}
final stats = await getSenderStats();
if (stats != null && prevStats != null && sender != null) {
_currentBitrate = computeBitrateForSenderStats(stats, prevStats);
events.emit(
AudioSenderStatsEvent(stats: stats, currentBitrate: currentBitrate));
}
prevStats = stats;
}
Future<AudioSenderStats?> getSenderStats() async {
if (sender == null) {
return null;
}
final stats = await sender!.getStats();
AudioSenderStats? senderStats;
for (var v in stats) {
if (v.type == 'outbound-rtp') {
senderStats ??= AudioSenderStats(v.id, v.timestamp);
senderStats.packetsSent = getNumValFromReport(v.values, 'packetsSent');
senderStats.packetsLost = getNumValFromReport(v.values, 'packetsLost');
senderStats.bytesSent = getNumValFromReport(v.values, 'bytesSent');
senderStats.roundTripTime =
getNumValFromReport(v.values, 'roundTripTime');
senderStats.jitter = getNumValFromReport(v.values, 'jitter');
final c = stats.firstWhereOrNull((element) => element.type == 'codec');
if (c != null) {
senderStats.mimeType = getStringValFromReport(c.values, 'mimeType');
senderStats.payloadType =
getNumValFromReport(c.values, 'payloadType');
senderStats.channels = getNumValFromReport(c.values, 'channels');
senderStats.clockRate = getNumValFromReport(c.values, 'clockRate');
}
break;
}
}
return senderStats;
}
// private constructor
@internal
LocalAudioTrack(
@@ -45,7 +100,7 @@ class LocalAudioTrack extends LocalTrack
static Future<LocalAudioTrack> create([
AudioCaptureOptions? options,
]) async {
options ??= const AudioCaptureOptions();
options = const AudioCaptureOptions();
final stream = await LocalTrack.createStream(options);
return LocalAudioTrack(
+23 -2
View File
@@ -1,3 +1,5 @@
import 'dart:async';
import 'package:flutter/material.dart';
import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
@@ -15,6 +17,7 @@ import '../../types/other.dart';
import '../options.dart';
import '../remote/audio.dart';
import '../remote/video.dart';
import '../stats.dart';
import '../track.dart';
import 'audio.dart';
import 'video.dart';
@@ -195,7 +198,7 @@ abstract class LocalTrack extends Track {
}
logger.fine('$objectId.publish()');
startMonitor();
_published = true;
return true;
}
@@ -209,8 +212,26 @@ abstract class LocalTrack extends Track {
}
logger.fine('$objectId.unpublish()');
stopMonitor();
_published = false;
return true;
}
Timer? _monitorTimer;
Future<void> monitorSender();
@internal
void startMonitor() {
_monitorTimer ??=
Timer.periodic(const Duration(milliseconds: monitorFrequency), (_) {
monitorSender();
});
}
@internal
void stopMonitor() {
_monitorTimer?.cancel();
_monitorTimer = null;
}
}
+99 -3
View File
@@ -1,9 +1,12 @@
import 'package:collection/collection.dart';
import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
import '../../events.dart';
import '../../logger.dart';
import '../../proto/livekit_models.pb.dart' as lk_models;
import '../../types/other.dart';
import '../options.dart';
import '../stats.dart';
import '../track.dart';
import 'audio.dart';
import 'local.dart';
@@ -15,6 +18,99 @@ class LocalVideoTrack extends LocalTrack with VideoTrack {
@override
covariant VideoCaptureOptions currentOptions;
num? _currentBitrate;
get currentBitrate => _currentBitrate;
Map<String, VideoSenderStats>? prevStats;
final Map<String, num> _bitrateFoLayers = {};
@override
Future<void> monitorSender() async {
if (sender == null) {
_currentBitrate = 0;
return;
}
List<VideoSenderStats> stats = [];
try {
stats = await getSenderStats();
} catch (e) {
logger.warning('Failed to get sender stats: $e');
return;
}
Map<String, VideoSenderStats> statsMap = {};
for (var s in stats) {
statsMap[s.rid ?? 'f'] = s;
}
if (prevStats != null) {
num totalBitrate = 0;
statsMap.forEach((key, s) {
final prev = prevStats![key];
var bitRateForlayer = computeBitrateForSenderStats(s, prev).toInt();
_bitrateFoLayers[key] = bitRateForlayer;
totalBitrate += bitRateForlayer;
});
_currentBitrate = totalBitrate;
events.emit(VideoSenderStatsEvent(
stats: statsMap,
currentBitrate: currentBitrate,
bitrateForLayers: _bitrateFoLayers,
));
}
prevStats = statsMap;
}
Future<List<VideoSenderStats>> getSenderStats() async {
if (sender == null) {
return [];
}
final stats = await sender!.getStats();
List<VideoSenderStats> items = [];
for (var v in stats) {
if (v.type == 'outbound-rtp') {
VideoSenderStats vs = VideoSenderStats(v.id, v.timestamp);
vs.frameHeight = getNumValFromReport(v.values, 'frameHeight');
vs.frameWidth = getNumValFromReport(v.values, 'frameWidth');
vs.framesPerSecond = getNumValFromReport(v.values, 'framesPerSecond');
vs.firCount = getNumValFromReport(v.values, 'firCount');
vs.pliCount = getNumValFromReport(v.values, 'pliCount');
vs.nackCount = getNumValFromReport(v.values, 'nackCount');
vs.packetsSent = getNumValFromReport(v.values, 'packetsSent');
vs.bytesSent = getNumValFromReport(v.values, 'bytesSent');
vs.framesSent = getNumValFromReport(v.values, 'framesSent');
vs.rid = getStringValFromReport(v.values, 'rid');
vs.encoderImplementation =
getStringValFromReport(v.values, 'encoderImplementation');
vs.retransmittedPacketsSent =
getNumValFromReport(v.values, 'retransmittedPacketsSent');
vs.qualityLimitationReason =
getStringValFromReport(v.values, 'qualityLimitationReason');
vs.qualityLimitationResolutionChanges =
getNumValFromReport(v.values, 'qualityLimitationResolutionChanges');
// locate the appropriate remote-inbound-rtp item
final remoteId = getStringValFromReport(v.values, 'remoteId');
final r = stats.firstWhereOrNull((element) => element.id == remoteId);
if (r != null) {
vs.jitter = getNumValFromReport(r.values, 'jitter');
vs.packetsLost = getNumValFromReport(r.values, 'packetsLost');
vs.roundTripTime = getNumValFromReport(r.values, 'roundTripTime');
}
final c = stats.firstWhereOrNull((element) => element.type == 'codec');
if (c != null) {
vs.mimeType = getStringValFromReport(c.values, 'mimeType');
vs.payloadType = getNumValFromReport(c.values, 'payloadType');
vs.channels = getNumValFromReport(c.values, 'channels');
vs.clockRate = getNumValFromReport(c.values, 'clockRate');
}
items.add(vs);
}
}
return items;
}
// Private constructor
LocalVideoTrack._(
String name,
@@ -34,7 +130,7 @@ class LocalVideoTrack extends LocalTrack with VideoTrack {
static Future<LocalVideoTrack> createCameraTrack([
CameraCaptureOptions? options,
]) async {
options ??= const CameraCaptureOptions();
options = const CameraCaptureOptions();
final stream = await LocalTrack.createStream(options);
return LocalVideoTrack._(
@@ -53,7 +149,7 @@ class LocalVideoTrack extends LocalTrack with VideoTrack {
static Future<LocalVideoTrack> createScreenShareTrack([
ScreenShareCaptureOptions? options,
]) async {
options ??= const ScreenShareCaptureOptions();
options = const ScreenShareCaptureOptions();
final stream = await LocalTrack.createStream(options);
return LocalVideoTrack._(
@@ -73,7 +169,7 @@ class LocalVideoTrack extends LocalTrack with VideoTrack {
static Future<List<LocalTrack>> createScreenShareTracksWithAudio([
ScreenShareCaptureOptions? options,
]) async {
options ??= const ScreenShareCaptureOptions(captureScreenAudio: true);
options = const ScreenShareCaptureOptions(captureScreenAudio: true);
final stream = await LocalTrack.createStream(options);
+71
View File
@@ -1,10 +1,13 @@
import 'package:collection/collection.dart';
import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
import '../../events.dart';
import '../../internal/events.dart';
import '../../proto/livekit_models.pb.dart' as lk_models;
import '../../types/other.dart';
import '../audio_management.dart';
import '../local/local.dart';
import '../stats.dart';
import 'remote.dart';
import '../web/_audio_api.dart' if (dart.library.html) '../web/_audio_html.dart'
@@ -58,4 +61,72 @@ class RemoteAudioTrack extends RemoteTrack
audio.setSinkId(getCid(), deviceId);
_deviceId = deviceId;
}
AudioReceiverStats? prevStats;
num? _currentBitrate;
get currentBitrate => _currentBitrate;
@override
Future<void> monitorReceiver() async {
if (receiver == null) {
_currentBitrate = 0;
return;
}
final stats = await getReceiverStats();
if (stats != null && prevStats != null && receiver != null) {
_currentBitrate = computeBitrateForReceiverStats(stats, prevStats);
events.emit(AudioReceiverStatsEvent(
stats: stats, currentBitrate: currentBitrate));
}
prevStats = stats;
}
Future<AudioReceiverStats?> getReceiverStats() async {
if (receiver == null) {
return null;
}
final stats = await receiver!.getStats();
AudioReceiverStats? receiverStats;
for (var v in stats) {
if (v.type == 'inbound-rtp') {
receiverStats ??= AudioReceiverStats(v.id, v.timestamp);
receiverStats.jitter = getNumValFromReport(v.values, 'jitter');
receiverStats.packetsLost =
getNumValFromReport(v.values, 'packetsLost');
receiverStats.jitterBufferDelay =
getNumValFromReport(v.values, 'jitterBufferDelay');
receiverStats.bytesReceived =
getNumValFromReport(v.values, 'bytesReceived');
receiverStats.packetsReceived =
getNumValFromReport(v.values, 'packetsReceived');
receiverStats.concealedSamples =
getNumValFromReport(v.values, 'concealedSamples');
receiverStats.concealmentEvents =
getNumValFromReport(v.values, 'concealmentEvents');
receiverStats.silentConcealedSamples =
getNumValFromReport(v.values, 'silentConcealedSamples');
receiverStats.silentConcealmentEvents =
getNumValFromReport(v.values, 'silentConcealmentEvents');
receiverStats.totalAudioEnergy =
getNumValFromReport(v.values, 'totalAudioEnergy');
receiverStats.totalSamplesDuration =
getNumValFromReport(v.values, 'totalSamplesDuration');
final c = stats.firstWhereOrNull((element) => element.type == 'codec');
if (c != null) {
receiverStats.mimeType = getStringValFromReport(c.values, 'mimeType');
receiverStats.payloadType =
getNumValFromReport(c.values, 'payloadType');
receiverStats.channels = getNumValFromReport(c.values, 'channels');
receiverStats.clockRate = getNumValFromReport(c.values, 'clockRate');
}
break;
}
}
return receiverStats;
}
}
+24
View File
@@ -1,7 +1,11 @@
import 'dart:async';
import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
import 'package:meta/meta.dart';
import '../../proto/livekit_models.pb.dart' as lk_models;
import '../../types/other.dart';
import '../stats.dart';
import '../track.dart';
abstract class RemoteTrack extends Track {
@@ -22,6 +26,7 @@ abstract class RemoteTrack extends Track {
final didStart = await super.start();
if (didStart) {
await enable();
startMonitor();
}
return didStart;
}
@@ -32,6 +37,25 @@ abstract class RemoteTrack extends Track {
if (didStop) {
await disable();
}
stopMonitor();
return didStop;
}
Timer? _monitorTimer;
Future<void> monitorReceiver();
@internal
void startMonitor() {
_monitorTimer ??=
Timer.periodic(const Duration(milliseconds: monitorFrequency), (_) {
monitorReceiver();
});
}
@internal
void stopMonitor() {
_monitorTimer?.cancel();
_monitorTimer = null;
}
}
+81 -1
View File
@@ -1,12 +1,15 @@
import 'package:collection/collection.dart';
import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
import 'package:meta/meta.dart';
import '../../events.dart';
import '../../proto/livekit_models.pb.dart' as lk_models;
import '../../types/other.dart';
import '../local/local.dart';
import '../stats.dart';
import 'remote.dart';
class RemoteVideoTrack extends RemoteTrack with VideoTrack {
//
RemoteVideoTrack(String name, TrackSource source, rtc.MediaStream stream,
rtc.MediaStreamTrack track,
{rtc.RTCRtpReceiver? receiver})
@@ -18,4 +21,81 @@ class RemoteVideoTrack extends RemoteTrack with VideoTrack {
track,
receiver: receiver,
);
VideoReceiverStats? prevStats;
num? _currentBitrate;
get currentBitrate => _currentBitrate;
@internal
String? getDecoderImplementation() {
return prevStats?.decoderImplementation;
}
@override
Future<void> monitorReceiver() async {
if (receiver == null) {
_currentBitrate = 0;
return;
}
final stats = await getReceiverStats();
if (stats != null && prevStats != null && receiver != null) {
_currentBitrate = computeBitrateForReceiverStats(stats, prevStats);
events.emit(VideoReceiverStatsEvent(
stats: stats, currentBitrate: currentBitrate));
}
prevStats = stats;
}
Future<VideoReceiverStats?> getReceiverStats() async {
if (receiver == null) {
return null;
}
final stats = await receiver!.getStats();
VideoReceiverStats? receiverStats;
for (var v in stats) {
if (v.type == 'inbound-rtp') {
receiverStats ??= VideoReceiverStats(v.id, v.timestamp);
receiverStats.jitter = getNumValFromReport(v.values, 'jitter');
receiverStats.jitterBufferDelay =
getNumValFromReport(v.values, 'jitterBufferDelay');
receiverStats.bytesReceived =
getNumValFromReport(v.values, 'bytesReceived');
receiverStats.packetsLost =
getNumValFromReport(v.values, 'packetsLost');
receiverStats.framesDecoded =
getNumValFromReport(v.values, 'framesDecoded');
receiverStats.framesDropped =
getNumValFromReport(v.values, 'framesDropped');
receiverStats.framesReceived =
getNumValFromReport(v.values, 'framesReceived');
receiverStats.packetsReceived =
getNumValFromReport(v.values, 'packetsReceived');
receiverStats.framesPerSecond =
getNumValFromReport(v.values, 'framesPerSecond');
receiverStats.frameWidth = getNumValFromReport(v.values, 'frameWidth');
receiverStats.frameHeight =
getNumValFromReport(v.values, 'frameHeight');
receiverStats.pliCount = getNumValFromReport(v.values, 'pliCount');
receiverStats.firCount = getNumValFromReport(v.values, 'firCount');
receiverStats.nackCount = getNumValFromReport(v.values, 'nackCount');
receiverStats.decoderImplementation =
getStringValFromReport(v.values, 'decoderImplementation');
final c = stats.firstWhereOrNull((element) => element.type == 'codec');
if (c != null) {
receiverStats.mimeType = getStringValFromReport(c.values, 'mimeType');
receiverStats.payloadType =
getNumValFromReport(c.values, 'payloadType');
receiverStats.channels = getNumValFromReport(c.values, 'channels');
receiverStats.clockRate = getNumValFromReport(c.values, 'clockRate');
}
break;
}
}
return receiverStats;
}
}
+200
View File
@@ -0,0 +1,200 @@
import 'package:flutter/foundation.dart';
import '../proto/livekit_models.pb.dart';
const monitorFrequency = 2000;
class CodecStats {
String? mimeType;
num? payloadType;
num? channels;
num? clockRate;
}
// key stats for senders and receivers
class SenderStats extends CodecStats {
SenderStats(this.streamId, this.timestamp);
/// number of packets sent
num? packetsSent;
/// number of bytes sent
num? bytesSent;
/// jitter as perceived by remote
num? jitter;
/// packets reported lost by remote
num? packetsLost;
/// RTT reported by remote
num? roundTripTime;
/// ID of the outbound stream
String streamId;
String? encoderImplementation;
num timestamp;
}
class AudioSenderStats extends SenderStats {
AudioSenderStats(String streamId, num timestamp) : super(streamId, timestamp);
TrackType type = TrackType.AUDIO;
}
class VideoSenderStats extends SenderStats {
VideoSenderStats(String streamId, num timestamp) : super(streamId, timestamp);
TrackType type = TrackType.VIDEO;
num? firCount;
num? pliCount;
num? nackCount;
String? rid;
num? frameWidth;
num? frameHeight;
num? framesSent;
num? framesPerSecond;
// bandwidth, cpu, other, none
String? qualityLimitationReason;
num? qualityLimitationResolutionChanges;
num? retransmittedPacketsSent;
}
class ReceiverStats extends CodecStats {
ReceiverStats(this.streamId, this.timestamp);
num? jitterBufferDelay;
/// packets reported lost by remote
num? packetsLost;
/// number of packets sent
num? packetsReceived;
num? bytesReceived;
String streamId;
num? jitter;
num timestamp;
}
class AudioReceiverStats extends ReceiverStats {
AudioReceiverStats(String streamId, num timestamp)
: super(streamId, timestamp);
TrackType type = TrackType.AUDIO;
num? concealedSamples;
num? concealmentEvents;
num? silentConcealedSamples;
num? silentConcealmentEvents;
num? totalAudioEnergy;
num? totalSamplesDuration;
}
class VideoReceiverStats extends ReceiverStats {
VideoReceiverStats(String streamId, num timestamp)
: super(streamId, timestamp);
TrackType type = TrackType.VIDEO;
num? framesDecoded;
num? framesDropped;
num? framesReceived;
num? framesPerSecond;
num? frameWidth;
num? frameHeight;
num? firCount;
num? pliCount;
num? nackCount;
String? decoderImplementation;
}
num computeBitrateForSenderStats(
SenderStats currentStats,
SenderStats? prevStats,
) {
if (prevStats == null) {
return 0;
}
num? bytesNow;
num? bytesPrev;
bytesNow = currentStats.bytesSent;
bytesPrev = prevStats.bytesSent;
if (bytesNow == null || bytesPrev == null) {
return 0;
}
if (kIsWeb) {
return ((bytesNow - bytesPrev) * 8) /
(currentStats.timestamp - prevStats.timestamp);
}
return ((bytesNow - bytesPrev) * 8 * 1000) /
(currentStats.timestamp - prevStats.timestamp);
}
num computeBitrateForReceiverStats(
ReceiverStats currentStats,
ReceiverStats? prevStats,
) {
if (prevStats == null) {
return 0;
}
num? bytesNow;
num? bytesPrev;
bytesNow = currentStats.bytesReceived;
bytesPrev = prevStats.bytesReceived;
if (bytesNow == null || bytesPrev == null) {
return 0;
}
if (kIsWeb) {
return ((bytesNow - bytesPrev) * 8) /
(currentStats.timestamp - prevStats.timestamp);
}
return ((bytesNow - bytesPrev) * 8 * 1000) /
(currentStats.timestamp - prevStats.timestamp);
}
num? getNumValFromReport(Map<dynamic, dynamic> values, String key) {
if (values.containsKey(key)) {
return (values[key] is String)
? num.tryParse(values[key])
: values[key] as num;
}
return null;
}
String? getStringValFromReport(Map<dynamic, dynamic> values, String key) {
if (values.containsKey(key)) {
return values[key] as String;
}
return null;
}