diff --git a/README.md b/README.md index 2df20b8..77e4814 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Table of Contents: - [MediaStreamTrack Entry](#mediastreamtrack-entry) - [InboundRTP Entry](#inboundrtp-entry) - [OutboundRTP Entry](#outboundrtp-entry) -- [Detectors and Issues](#issues-and-detectors) +- [Detectors and Issues](#detectors-and-issues) - [Congestion Detector](#congestion-detector) - [Audio Desync Detector](#audio-desync-detector) - [CPU Performance Detector](#cpu-performance-detector) @@ -30,7 +30,7 @@ Table of Contents: - [Getting Involved](#getting-involved) - [License](#license) -## Qucik Start +## Quick Start Install it from [npm](https://www.npmjs.com/package/@observertc/client-monitor-js) package repository. @@ -534,6 +534,33 @@ detector.on('statechanged', onStateChanged); ``` +### Video Freeze Detector + +```javascript +const detector = monitor.createVideoFreezesDetector({ + createIssueOnDetection: { + severity: 'major', + attachments: { + // various custom data + }, + } +}); +detector.on('freezedVideoStarted', event => { + console.log('Freezed video started'); + console.log('TrackId', event.trackId); + console.log('PeerConnectionId', event.peerConnectionId); + console.log('SSRC:', event.ssrc); +}); + +detector.on('freezedVideoEnded', event => { + console.log('Freezed video ended'); + console.log('TrackId', event.trackId); + console.log('Freeze duration in Seconds', event.durationInS); + console.log('PeerConnectionId', event.peerConnectionId); + console.log('SSRC:', event.ssrc); +}); +``` + ## Configurations ```javascript diff --git a/package.json b/package.json index f5c96cc..7847248 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@observertc/client-monitor-js", - "version": "3.6.0", + "version": "3.7.0", "description": "ObserveRTC Client Integration Javascript Library", "main": "lib/index.js", "types": "lib/index.d.ts", diff --git a/src/ClientMonitor.ts b/src/ClientMonitor.ts index 7ebc3e9..03215bf 100644 --- a/src/ClientMonitor.ts +++ b/src/ClientMonitor.ts @@ -11,6 +11,12 @@ import { PeerConnectionEntry, TrackStats } from './entries/StatsEntryInterfaces' import { AudioDesyncDetector, AudioDesyncDetectorConfig } from './detectors/AudioDesyncDetector'; import { CongestionDetector, CongestionDetectorEvents } from './detectors/CongestionDetector'; import { CpuPerformanceDetector, CpuPerformanceDetectorConfig } from './detectors/CpuPerformanceDetector'; +import { + VideoFreezesDetector, + VideoFreezesDetectorConfig, + FreezedVideoStartedEvent, + FreezedVideoEndedEvent, +} from './detectors/VideoFreezesDetector'; const logger = createLogger('ClientMonitor'); @@ -62,8 +68,14 @@ export interface ClientMonitorEvents { outgoingBitrateAfterCongestion: number | undefined; outgoingBitrateBeforeCongestion: number | undefined; }, + 'usermediaerror': string, 'cpulimitation': AlertState, 'audio-desync': AlertState, + 'freezed-video': { + trackId: string, + peerConnectionId: string | undefined, + }, + 'using-turn': boolean, 'issue': ClientIssue, } @@ -145,19 +157,28 @@ export class ClientMonitor extends TypedEventEmitter { public async collect(): Promise { if (this._closed) throw new Error('ClientMonitor is closed'); - + const wasUsingTURN = this.peerConnections.some(pc => pc.usingTURN); const collectedStats = await this.collectors.collect(); this.storage.update(collectedStats); const timestamp = Date.now(); + this.emit('stats-collected', { collectedStats, elapsedSinceLastCollectedInMs: timestamp - this._lastCollectedAt, }); + this._lastCollectedAt = timestamp; + if (this._config.samplingTick && this._config.samplingTick <= ++this._actualCollectingTick ) { this._actualCollectingTick = 0; this.sample(); } + + const isUsingTURN = this.peerConnections.some(pc => pc.usingTURN); + + if (wasUsingTURN !== isUsingTURN) { + this.emit('using-turn', isUsingTURN); + } return collectedStats; } @@ -207,6 +228,10 @@ export class ClientMonitor extends TypedEventEmitter { this._sampler.setMarker(value); } + public setUserId(userId?: string) { + this._sampler.setUserId(userId); + } + public setMediaDevices(...devices: MediaDevice[]): void { if (!devices) return; this.meta.mediaDevices = devices; @@ -222,7 +247,12 @@ export class ClientMonitor extends TypedEventEmitter { } public addUserMediaError(err: unknown): void { - this._sampler.addUserMediaError(`${err}`); + const message = `${err}`; + + if(0 < (this._config.samplingTick ?? 0)) + this._sampler.addUserMediaError(message); + + this.emit('usermediaerror', message); } public setMediaConstraints(constrains: MediaStreamConstraints | MediaTrackConstraints): void { @@ -393,6 +423,59 @@ export class ClientMonitor extends TypedEventEmitter { return detector; } + public createVideoFreezesDetector(config?: VideoFreezesDetectorConfig & { + createIssueOnDetection?: { + attachments?: Record, + severity: 'critical' | 'major' | 'minor', + }, + }): VideoFreezesDetector { + const existingDetector = this._detectors.get(VideoFreezesDetector.name); + + if (existingDetector) return existingDetector as VideoFreezesDetector; + + const detector = new VideoFreezesDetector({ + }); + const onUpdate = () => detector.update(this.storage.inboundRtps()); + const { + createIssueOnDetection, + } = config ?? {}; + + const onFreezeStarted = (event: FreezedVideoStartedEvent) => { + this.emit('freezed-video', { + peerConnectionId: event.peerConnectionId, + trackId: event.trackId, + }); + }; + const onFreezeEnded = (event: FreezedVideoEndedEvent) => { + if (createIssueOnDetection) { + this.addIssue({ + severity: createIssueOnDetection.severity, + description: 'Video Freeze detected', + timestamp: Date.now(), + peerConnectionId: event.peerConnectionId, + mediaTrackId: event.trackId, + attachments: { + durationInS: event.durationInS, + ...(createIssueOnDetection.attachments ?? {}) + }, + }); + } + } + + detector.once('close', () => { + this.off('stats-collected', onUpdate); + detector.off('freezedVideoStarted', onFreezeStarted); + detector.off('freezedVideoEnded', onFreezeEnded); + this._detectors.delete(VideoFreezesDetector.name); + }); + detector.on('freezedVideoStarted', onFreezeStarted); + detector.on('freezedVideoEnded', onFreezeEnded); + + this._detectors.set(VideoFreezesDetector.name, detector); + + return detector; + } + public createCpuPerformanceIssueDetector(config?: CpuPerformanceDetectorConfig & { createIssueOnDetection?: { attachments?: Record, @@ -656,6 +739,14 @@ export class ClientMonitor extends TypedEventEmitter { return this.storage.highestSeenAvailableIncomingBitrate; } + public get sendingFractionLost() { + return this.storage.sendingFractionLost; + } + + public get receivingFractionLost() { + return this.storage.receivingFractionLost; + } + private _setupTimer(): void { this._timer && clearInterval(this._timer); this._timer = undefined; diff --git a/src/Sampler.ts b/src/Sampler.ts index d15b8ea..42fd9b3 100644 --- a/src/Sampler.ts +++ b/src/Sampler.ts @@ -42,6 +42,7 @@ export class Sampler { private _localSDP?: string[]; private _marker?: string; private _sampleSeq = 0; + private _userId?: string; private readonly _timezoneOffset: number = new Date().getTimezoneOffset(); public constructor( @@ -104,6 +105,10 @@ export class Sampler { this._marker = value; } + public setUserId(userId?: string) { + this._userId = userId; + } + public clear() { this._engine = undefined; this._platform = undefined; @@ -124,8 +129,8 @@ export class Sampler { callId: 'NULL', clientId: 'NULL', roomId: 'NULL', - userId: 'NULL', - + + userId: this._userId, marker: this._marker, sampleSeq: this._sampleSeq, timeZoneOffsetInHours: this._timezoneOffset, diff --git a/src/detectors/CongestionDetector.ts b/src/detectors/CongestionDetector.ts index 10d194d..c3bd534 100644 --- a/src/detectors/CongestionDetector.ts +++ b/src/detectors/CongestionDetector.ts @@ -2,6 +2,7 @@ import EventEmitter from "events"; import { IceCandidatePairEntry, PeerConnectionEntry } from "../entries/StatsEntryInterfaces"; type PeerConnectionState = { + peerConnectionId: string; congested: boolean; outgoingBitrateBeforeCongestion?: number; outgoingBitrateAfterCongestion?: number; @@ -32,6 +33,10 @@ export class CongestionDetector extends EventEmitter { } + public get states(): ReadonlyMap { + return this._states; + } + public update(peerConnections: IterableIterator) { const visitedPeerConnectionIds = new Set(); let gotCongested = false; @@ -44,6 +49,7 @@ export class CongestionDetector extends EventEmitter { if (!state) { state = { + peerConnectionId, congested: false, // outgoingBitrateBeforeCongestion: 0, // outgoingBitrateAfterCongestion: 0, diff --git a/src/detectors/VideoFreezesDetector.ts b/src/detectors/VideoFreezesDetector.ts new file mode 100644 index 0000000..2cac107 --- /dev/null +++ b/src/detectors/VideoFreezesDetector.ts @@ -0,0 +1,118 @@ +import EventEmitter from "events"; +import { InboundRtpEntry } from "../entries/StatsEntryInterfaces"; + +export type VideoFreezesDetectorConfig = { + // empty +} + +export type FreezedVideoStartedEvent = { + peerConnectionId: string | undefined, + trackId: string, + ssrc: number, +} + +export type FreezedVideoEndedEvent = { + peerConnectionId: string, + trackId: string, + durationInS: number, +} + +export type VideoFreezesDetectorEvents = { + freezedVideoStarted: [FreezedVideoStartedEvent], + freezedVideoEnded: [FreezedVideoEndedEvent], + close: [], +} + +type InboundRtpStatsTrace = { + ssrc: number, + lastFreezeCount: number, + freezedStartedDuration?: number, + freezed: boolean, + visited: boolean, +} + +export declare interface VideoFreezesDetector { + on(event: K, listener: (...events: VideoFreezesDetectorEvents[K]) => void): this; + off(event: K, listener: (...events: VideoFreezesDetectorEvents[K]) => void): this; + once(event: K, listener: (...events: VideoFreezesDetectorEvents[K]) => void): this; + emit(event: K, ...events: VideoFreezesDetectorEvents[K]): boolean; +} + +export class VideoFreezesDetector extends EventEmitter { + private _closed = false; + private readonly _traces = new Map(); + + public constructor( + public readonly config: VideoFreezesDetectorConfig, + ) { + super(); + this.setMaxListeners(Infinity); + + } + + public close() { + if (this._closed) return; + this._closed = true; + + this._traces.clear(); + this.emit('close'); + } + + public update(inboundRtps: IterableIterator) { + for (const inboundRtp of inboundRtps) { + const stats = inboundRtp.stats; + const trackId = inboundRtp.getTrackId(); + const ssrc = stats.ssrc; + if (stats.kind !== 'video' || trackId === undefined) { + continue; + } + + let trace = this._traces.get(ssrc); + if (!trace) { + trace = { + ssrc, + lastFreezeCount: 0, + freezed: false, + freezedStartedDuration: 0, + visited: false, + }; + this._traces.set(ssrc, trace); + } + + const wasFreezed = trace.freezed; + + trace.visited = true; + trace.freezed = 0 < Math.max(0, (stats.freezeCount ?? 0) - trace.lastFreezeCount); + trace.lastFreezeCount = stats.freezeCount ?? 0; + + if (!wasFreezed && trace.freezed) { + trace.freezedStartedDuration = stats.totalFreezesDuration ?? 0; + this.emit('freezedVideoStarted', { + peerConnectionId: inboundRtp.getPeerConnection()?.peerConnectionId, + trackId, + ssrc, + }) + } else if (wasFreezed && !trace.freezed) { + const durationInS = Math.max(0, (stats.totalFreezesDuration ?? 0) - (trace.freezedStartedDuration ?? 0)); + + trace.freezedStartedDuration = undefined; + + 0 < durationInS && this.emit('freezedVideoEnded', { + peerConnectionId: inboundRtp.getPeerConnection()?.peerConnectionId, + trackId, + durationInS, + }) + } + } + + for (const trace of this._traces.values()) { + if (trace.visited) { + trace.visited = false; + + continue; + } + + this._traces.delete(trace.ssrc); + } + } +} diff --git a/src/entries/OutboundTrackStats.ts b/src/entries/OutboundTrackStats.ts index 9b0602c..fe8fa71 100644 --- a/src/entries/OutboundTrackStats.ts +++ b/src/entries/OutboundTrackStats.ts @@ -12,7 +12,12 @@ export function createOutboundTrackStats(peerConnection: PeerConnectionEntry, tr } } } + + const outboundRtps = Array.from(iterator()); + const layers = outboundRtps.filter(rtp => rtp.getSsrc() !== undefined && rtp.stats.rid !== undefined && rtp.sendingBitrate !== undefined) + .map(rtp => ({ ssrc: rtp.getSsrc()!, rid: rtp.stats.rid!, sendingBitrate: rtp.sendingBitrate! })); + let sfuStreamId = outboundRtps.find(outboundRtp => outboundRtp.sfuStreamId !== undefined)?.sfuStreamId; const result = { direction: 'outbound', @@ -24,8 +29,9 @@ export function createOutboundTrackStats(peerConnection: PeerConnectionEntry, tr set sfuStreamId(value: string | undefined) { sfuStreamId = value; }, - - sendingBitrate: outboundRtps.reduce((acc, outboundRtp) => acc + (outboundRtp.sendingBitrate ?? 0), 0), + + layers, + sendingBitrate:layers.reduce((acc, layer) => acc + (layer.sendingBitrate ?? 0), 0), sentPackets: outboundRtps.reduce((acc, outboundRtp) => acc + (outboundRtp.sentPackets ?? 0), 0), remoteLostPackets: outboundRtps.reduce((acc, outboundRtp) => acc + (outboundRtp.getRemoteInboundRtp()?.lostPackets ?? 0), 0), remoteReceivedPackets: outboundRtps.reduce((acc, outboundRtp) => acc + (outboundRtp.getRemoteInboundRtp()?.receivedPackets ?? 0), 0), @@ -39,14 +45,22 @@ export function createOutboundTrackStats(peerConnection: PeerConnectionEntry, tr result.sentPackets = 0; result.remoteLostPackets = 0; result.remoteReceivedPackets = 0; + result.layers = []; for (const outboundRtp of iterator()) { + const ssrc = outboundRtp.getSsrc(); + const rid = outboundRtp.stats.rid; + const sendingBitrate = outboundRtp.sendingBitrate; + outboundRtp.sfuStreamId = sfuStreamId; - result.sendingBitrate += outboundRtp.sendingBitrate ?? 0; + result.sendingBitrate += sendingBitrate ?? 0; result.sentPackets += outboundRtp.sentPackets ?? 0; result.remoteLostPackets += outboundRtp.getRemoteInboundRtp()?.lostPackets ?? 0; result.remoteReceivedPackets += outboundRtp.getRemoteInboundRtp()?.receivedPackets ?? 0; + if (ssrc !== undefined && rid !== undefined && sendingBitrate !== undefined) { + result.layers.push({ ssrc, rid, sendingBitrate }); + } } } }; diff --git a/src/entries/PeerConnectionEntryManifest.ts b/src/entries/PeerConnectionEntryManifest.ts index 6ef2759..99672af 100644 --- a/src/entries/PeerConnectionEntryManifest.ts +++ b/src/entries/PeerConnectionEntryManifest.ts @@ -66,8 +66,11 @@ export class PeerConnectionEntryManifest implements PeerConnectionEntry { public avgRttInS?: number; public sendingAudioBitrate?: number; public sendingVideoBitrate?: number; + public sendingFractionLost?: number; + public receivingAudioBitrate?: number; public receivingVideoBitrate?: number; + public receivingFractionLost?: number; public readonly config: { outbStabilityScoresLength: number; @@ -157,6 +160,10 @@ export class PeerConnectionEntryManifest implements PeerConnectionEntry { return this._emitter; } + public get usingTURN(): boolean { + return this.getSelectedIceCandidatePair()?.getRemoteCandidate()?.stats.candidateType === 'relay'; + } + public update(statsMap: StatsMap) { for (const statsValue of statsMap) { this._visit(statsValue); @@ -564,6 +571,7 @@ export class PeerConnectionEntryManifest implements PeerConnectionEntry { roundTripTimesInS.push(currentRoundTripTime) } } + const avgRttInS = (roundTripTimesInS.length < 1 ? this.avgRttInS : Math.max(0, roundTripTimesInS.reduce((acc, rtt) => acc + rtt, 0) / roundTripTimesInS.length)); for (const outboundRtpEntry of this._outboundRtps.values()) { @@ -588,6 +596,14 @@ export class PeerConnectionEntryManifest implements PeerConnectionEntry { this.totalDataChannelBytesReceived += this.deltaDataChannelBytesReceived; this.totalDataChannelBytesSent += this.deltaDataChannelBytesSent; this.avgRttInS = avgRttInS; + + if (0 < this.deltaOutboundPacketsLost + this.deltaOutboundPacketsReceived) { + this.sendingFractionLost = this.deltaOutboundPacketsLost / (this.deltaOutboundPacketsLost + this.deltaOutboundPacketsReceived); + } + + if (0 < this.deltaInboundPacketsLost + this.deltaInboundPacketsReceived) { + this.receivingFractionLost = this.deltaInboundPacketsLost / (this.deltaInboundPacketsLost + this.deltaInboundPacketsReceived); + } } private _createCodecEntry(stats: W3C.CodecStats): CodecEntry { diff --git a/src/entries/StatsEntryInterfaces.ts b/src/entries/StatsEntryInterfaces.ts index 5f104e1..06bfb82 100644 --- a/src/entries/StatsEntryInterfaces.ts +++ b/src/entries/StatsEntryInterfaces.ts @@ -328,6 +328,8 @@ export interface PeerConnectionEntry { readonly label: string | undefined; readonly events: TypedEvents; + readonly usingTURN: boolean; + readonly totalInboundPacketsLost: number; readonly totalInboundPacketsReceived: number; readonly totalOutboundPacketsLost: number; @@ -355,8 +357,11 @@ export interface PeerConnectionEntry { readonly avgRttInS?: number; readonly sendingAudioBitrate?: number; readonly sendingVideoBitrate?: number; + readonly sendingFractionalLoss?: number; + readonly receivingAudioBitrate?: number; readonly receivingVideoBitrate?: number; + readonly receivingFractionalLoss?: number; getSelectedIceCandidatePair(): IceCandidatePairEntry | undefined; codecs(): IterableIterator; diff --git a/src/entries/StatsStorage.ts b/src/entries/StatsStorage.ts index 58b93f7..bc44616 100644 --- a/src/entries/StatsStorage.ts +++ b/src/entries/StatsStorage.ts @@ -42,8 +42,11 @@ export class StatsStorage { public sendingAudioBitrate?: number; public sendingVideoBitrate?: number; + public sendingFractionLost?: number; + public receivingAudioBitrate?: number; public receivingVideoBitrate?: number; + public receivingFractionLost?: number; public totalInboundPacketsLost = 0; public totalInboundPacketsReceived = 0; @@ -298,8 +301,11 @@ export class StatsStorage { private _updateMetrics() { this.sendingAudioBitrate = 0; this.sendingVideoBitrate = 0; + this.sendingFractionLost = 0.0; + this.receivingAudioBitrate = 0; this.receivingVideoBitrate = 0; + this.receivingFractionLost = 0.0; this.deltaInboundPacketsLost = 0; this.deltaInboundPacketsReceived = 0; @@ -314,6 +320,7 @@ export class StatsStorage { this.deltaReceivedVideoBytes = 0; this.totalAvailableIncomingBitrate = 0; this.totalAvailableOutgoingBitrate = 0; + for (const peerConnectionEntry of this._peerConnections.values()) { for (const transport of peerConnectionEntry.transports()) { @@ -337,6 +344,9 @@ export class StatsStorage { this.deltaSentVideoBytes += peerConnectionEntry.deltaSentVideoBytes ?? 0; this.deltaReceivedAudioBytes += peerConnectionEntry.deltaReceivedAudioBytes ?? 0; this.deltaReceivedVideoBytes += peerConnectionEntry.deltaReceivedVideoBytes ?? 0; + + this.sendingFractionLost += peerConnectionEntry.sendingFractionLost ?? 0.0; + this.receivingFractionLost += peerConnectionEntry.receivingFractionLost ?? 0.0; } this.totalInboundPacketsLost += this.deltaInboundPacketsLost; this.totalInboundPacketsReceived += this.deltaInboundPacketsReceived; @@ -379,6 +389,8 @@ export class StatsStorage { } } this.avgRttInS = avgRttInS; + this.sendingFractionLost = Math.round(this.sendingFractionLost * 100) / 100; + this.receivingFractionLost = Math.round(this.receivingFractionLost * 100) / 100; } private _updateTracks() { diff --git a/src/index.ts b/src/index.ts index 3948e1e..7759a1a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,9 +9,21 @@ export type { ClientMonitorEvents, } from "./ClientMonitor"; +export type { + VideoFreezesDetector, + VideoFreezesDetectorConfig, + FreezedVideoStartedEvent, + FreezedVideoEndedEvent, +} from './detectors/VideoFreezesDetector'; export type { CongestionDetector } from './detectors/CongestionDetector'; -export type { CpuPerformanceDetectorConfig } from './detectors/CpuPerformanceDetector'; -export type { AudioDesyncDetector, AudioDesyncDetectorConfig } from './detectors/AudioDesyncDetector'; +export type { + CpuPerformanceDetector, + CpuPerformanceDetectorConfig +} from './detectors/CpuPerformanceDetector'; +export type { + AudioDesyncDetector, + AudioDesyncDetectorConfig +} from './detectors/AudioDesyncDetector'; export type { Collectors } from './Collectors'; export type { @@ -94,6 +106,7 @@ export function createClientMonitor(config?: ClientMonitorConfig & { }); } + export { createLogger, addLoggerProcess, diff --git a/src/schema/W3cStatsIdentifiers.ts b/src/schema/W3cStatsIdentifiers.ts index 8c51e17..c5989a8 100644 --- a/src/schema/W3cStatsIdentifiers.ts +++ b/src/schema/W3cStatsIdentifiers.ts @@ -218,13 +218,13 @@ type RtcInboundRtpStreamStats = { framesDropped?: number; // only video frameWidth?: number; // only video frameHeight?: number; // only video - framesPerSecond?: number; // only vidoe + framesPerSecond?: number; // only video qpSum?: number; // only video totalDecodeTime?: number; // only video totalInterFrameDelay?: number; // only video totalSquaredInterFrameDelay?: number; // only video - pauseCaount?: number; + pauseCount?: number; totalPausesDuration?: number; freezeCount?: number; totalFreezesDuration?: number;