From 705688d6f57459103fac674800cb9a5b1d321499 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 31 May 2022 01:09:07 +0100 Subject: [PATCH 001/137] untested WIP for calling SFUs --- src/webrtc/call.ts | 3 + src/webrtc/groupCall.ts | 169 +++++++++++++++++++++++++++++----------- 2 files changed, 127 insertions(+), 45 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 7ae188aabbe..5d8c01349a1 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -328,6 +328,7 @@ export class MatrixCall extends EventEmitter { private opponentDeviceId: string; private opponentSessionId: string; public groupCallId: string; + private dataChannel: DataChannel; constructor(opts: CallOpts) { super(); @@ -339,6 +340,7 @@ export class MatrixCall extends EventEmitter { this.opponentDeviceId = opts.opponentDeviceId; this.opponentSessionId = opts.opponentSessionId; this.groupCallId = opts.groupCallId; + // Array of Objects with urls, username, credential keys this.turnServers = opts.turnServers || []; if (this.turnServers.length === 0 && this.client.isFallbackICEServerAllowed()) { @@ -376,6 +378,7 @@ export class MatrixCall extends EventEmitter { public createDataChannel(label: string, options: RTCDataChannelInit) { const dataChannel = this.peerConn.createDataChannel(label, options); this.emit(CallEvent.DataChannel, dataChannel); + this.dataChannel = dataChannel; return dataChannel; } diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 32f6fa29554..84ec86ddae1 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -144,6 +144,8 @@ export class GroupCall extends EventEmitter { public type: GroupCallType, public isPtt: boolean, public intent: GroupCallIntent, + public localSfu?: string, + public localSfuDeviceId?: string, groupCallId?: string, private dataChannelsEnabled?: boolean, private dataChannelOptions?: IGroupCallDataChannelOptions, @@ -152,6 +154,11 @@ export class GroupCall extends EventEmitter { this.reEmitter = new ReEmitter(this); this.groupCallId = groupCallId || genCallID(); + if (this.localSfu) { + // we have to use DCs to talk to the SFU + this.dataChannelsEnabled = true; + } + const roomState = this.room.currentState; const memberStateEvents = roomState.getStateEvents(EventType.GroupCallMemberPrefix); @@ -596,7 +603,12 @@ export class GroupCall extends EventEmitter { "device_id": deviceId, "session_id": this.client.getSessionId(), "feeds": this.getLocalFeeds().map((feed) => ({ - purpose: feed.purpose, + "purpose": feed.purpose, + "id": feed.stream.id, + "tracks": feed.stream.getTracks().map((track) => ({ + "id": track.id, + "settings": track.settings, + })), })), // TODO: Add data channels }, @@ -695,64 +707,120 @@ export class GroupCall extends EventEmitter { return; } - // Only initiate a call with a user who has a userId that is lexicographically - // less than your own. Otherwise, that user will call you. - if (member.userId < localUserId) { - logger.log(`Waiting for ${member.userId} to send call invite.`); - return; - } + let opponentDevice: IGroupCallRoomMemberDevice; + let peerUserId: string; + let existingCall: MatrixCall; - const opponentDevice = this.getDeviceForMember(member.userId); + if (!this.localSfu) { + // Only initiate a call with a user who has a userId that is lexicographically + // less than your own. Otherwise, that user will call you. + if (member.userId < localUserId) { + logger.log(`Waiting for ${member.userId} to send call invite.`); + return; + } - if (!opponentDevice) { - logger.warn(`No opponent device found for ${member.userId}, ignoring.`); - this.emit( - GroupCallEvent.Error, - new GroupCallError( - GroupCallErrorCode.UnknownDevice, - `Outgoing Call: No opponent device found for ${member.userId}, ignoring.`, - ), - ); - return; - } + opponentDevice = this.getDeviceForMember(member.userId); - const existingCall = this.getCallByUserId(member.userId); + if (!opponentDevice) { + logger.warn(`No opponent device found for ${member.userId}, ignoring.`); + this.emit( + GroupCallEvent.Error, + new GroupCallError( + GroupCallErrorCode.UnknownDevice, + `Outgoing Call: No opponent device found for ${member.userId}, ignoring.`, + ), + ); + return; + } - if ( - existingCall && - existingCall.getOpponentSessionId() === opponentDevice.session_id - ) { - return; + peerUserId = member.userId; + existingCall = this.getCallByUserId(peerUserId); + + // if we already have an existing call to the same session on the other side, then + // use it - it must have already called us first. + if ( + existingCall && + existingCall.getOpponentSessionId() === opponentDevice.session_id + ) { + return; + } + } + else { + peerUserId = this.localSfu; + opponentDevice = { + "device_id": this.localSfuDeviceId; + "session_id": ""; // we can use a blank session_id, as the SFU is stateless + "feeds": []; + } + existingCall = this.getCallByUserId(peerUserId); } - const newCall = createNewMatrixCall( - this.client, - this.room.roomId, - { - invitee: member.userId, - opponentDeviceId: opponentDevice.device_id, - opponentSessionId: opponentDevice.session_id, - groupCallId: this.groupCallId, - }, - ); + if (!this.localSfu || !existingCall) { + const newCall = createNewMatrixCall( + this.client, + this.room.roomId, + { + invitee: peerUserId, + opponentDeviceId: opponentDevice.device_id, + opponentSessionId: opponentDevice.session_id, + groupCallId: this.groupCallId, + }, + ); - const requestScreenshareFeed = opponentDevice.feeds.some( - (feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); + const requestScreenshareFeed = opponentDevice.feeds.some( + (feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); - // Safari can't send a MediaStream to multiple sources, so clone it - newCall.placeCallWithCallFeeds(this.getLocalFeeds().map(feed => feed.clone()), requestScreenshareFeed); + // Safari can't send a MediaStream to multiple sources, so clone it + newCall.placeCallWithCallFeeds(this.getLocalFeeds().map(feed => feed.clone()), requestScreenshareFeed); + + if (this.dataChannelsEnabled) { + newCall.createDataChannel("datachannel", this.dataChannelOptions); + } - if (this.dataChannelsEnabled) { - newCall.createDataChannel("datachannel", this.dataChannelOptions); + if (existingCall) { + this.replaceCall(existingCall, newCall, CallErrorCode.NewSession); + } else { + this.addCall(newCall); + } } - if (existingCall) { - this.replaceCall(existingCall, newCall, CallErrorCode.NewSession); - } else { - this.addCall(newCall); + if (this.localSfu) { + // TODO: only subscribe to the streams you care about + remoteDevice = this.getDeviceForMember(member.userId); + // subscribe if we have a call established to the SFU + // if not, we'll trigger a subscription when the call sets up. + this.subscribeStream(existingCall || newCall, member.userId, remoteDevice); } }; + private subscribeStream(call: MatrixCall, userId: string, device: IGroupCallRoomMemberDevice, streamId?: string) { + // Asks the SFU to subscribe to the given streams originating from a given device. + // if streamId is undefined, subscribe to all of them. + + const streams = []; + for (const feed of device.feeds) { + if (streamId && feed.id !== streamId) continue; + for (const track of feed.tracks) { + streams.push({ + "stream_id": feed.id, + "track_id": track.id + }); + } + } + + if (streams.length == 0) { + logger.warn("Failed to find any streams to subscribe to"); + return; + } + + // FIXME: actually wait until dataChannel has fired onopen + call.dataChannel.send(JSON.stringify({ + "op": "select", + "conf_id": this.groupCallId, + "start": streams + })); + } + public getDeviceForMember(userId: string): IGroupCallRoomMemberDevice { const memberStateEvent = this.room.currentState.getStateEvents(EventType.GroupCallMemberPrefix, userId); @@ -968,6 +1036,17 @@ export class GroupCall extends EventEmitter { if (state === CallState.Connected) { this.retryCallCounts.delete(getCallUserId(call)); + + // if we're calling an SFU, subscribe to its feeds + // XXX: check that datachannel is open first? + if (this.localSfu) { + const memberStateEvents = this.room.currentState.getStateEvents(EventType.GroupCallMemberPrefix); + for (const stateEvent of memberStateEvents) { + const userId = stateEvent.getStateKey(); + const device = this.getDeviceForMember(userId); + this.subscribeStream(call, userId, device); + } + } } }; From 36eeb27343fead8e9c68ab2d884f074fb2f9b7f5 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 31 May 2022 11:49:26 +0100 Subject: [PATCH 002/137] fix types --- src/client.ts | 4 +++ src/webrtc/call.ts | 2 +- src/webrtc/groupCall.ts | 58 +++++++++++++++++++++++++---------------- 3 files changed, 40 insertions(+), 24 deletions(-) diff --git a/src/client.ts b/src/client.ts index 3ce67168ce9..7132e8bb5b4 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1326,6 +1326,8 @@ export class MatrixClient extends EventEmitter { intent: GroupCallIntent, dataChannelsEnabled?: boolean, dataChannelOptions?: IGroupCallDataChannelOptions, + localSfu?: string, + localSfuDeviceId?: string, ): Promise { if (this.getGroupCallForRoom(roomId)) { throw new Error(`${roomId} already has an existing group call`); @@ -1346,6 +1348,8 @@ export class MatrixClient extends EventEmitter { undefined, dataChannelsEnabled, dataChannelOptions, + localSfu, + localSfuDeviceId, ).create(); } diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 5d8c01349a1..c3ee8202594 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -328,7 +328,7 @@ export class MatrixCall extends EventEmitter { private opponentDeviceId: string; private opponentSessionId: string; public groupCallId: string; - private dataChannel: DataChannel; + public dataChannel: RTCDataChannel; constructor(opts: CallOpts) { super(); diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 84ec86ddae1..841b5e7077f 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -72,25 +72,34 @@ export interface IGroupCallDataChannelOptions { protocol: string; } -export interface IGroupCallRoomMemberFeed { +export interface IGroupCallMemberTrack { + id: string; + kind: string; // TODO: use an enum + label: string; + settings: MediaTrackSettings; +} + +export interface IGroupCallMemberFeed { + id: string; purpose: SDPStreamMetadataPurpose; + tracks: IGroupCallMemberTrack[]; // TODO: Sources for adaptive bitrate } -export interface IGroupCallRoomMemberDevice { +export interface IGroupCallMemberDevice { "device_id": string; "session_id": string; - "feeds": IGroupCallRoomMemberFeed[]; + "feeds": IGroupCallMemberFeed[]; } -export interface IGroupCallRoomMemberCallState { +export interface IGroupCallMemberCallState { "m.call_id": string; "m.foci"?: string[]; - "m.devices": IGroupCallRoomMemberDevice[]; + "m.devices": IGroupCallMemberDevice[]; } -export interface IGroupCallRoomMemberState { - "m.calls": IGroupCallRoomMemberCallState[]; +export interface IGroupCallMemberState { + "m.calls": IGroupCallMemberCallState[]; } export enum GroupCallState { @@ -144,11 +153,11 @@ export class GroupCall extends EventEmitter { public type: GroupCallType, public isPtt: boolean, public intent: GroupCallIntent, - public localSfu?: string, - public localSfuDeviceId?: string, groupCallId?: string, private dataChannelsEnabled?: boolean, private dataChannelOptions?: IGroupCallDataChannelOptions, + private localSfu?: string, + private localSfuDeviceId?: string, ) { super(); this.reEmitter = new ReEmitter(this); @@ -607,7 +616,9 @@ export class GroupCall extends EventEmitter { "id": feed.stream.id, "tracks": feed.stream.getTracks().map((track) => ({ "id": track.id, - "settings": track.settings, + "kind": track.kind, + "label": track.label, + "settings": track.getSettings(), })), })), // TODO: Add data channels @@ -621,13 +632,13 @@ export class GroupCall extends EventEmitter { return this.updateMemberCallState(undefined); } - private async updateMemberCallState(memberCallState?: IGroupCallRoomMemberCallState): Promise { + private async updateMemberCallState(memberCallState?: IGroupCallMemberCallState): Promise { const localUserId = this.client.getUserId(); const currentStateEvent = this.room.currentState.getStateEvents(EventType.GroupCallMemberPrefix, localUserId); - const memberStateEvent = currentStateEvent?.getContent(); + const memberStateEvent = currentStateEvent?.getContent(); - let calls: IGroupCallRoomMemberCallState[] = []; + let calls: IGroupCallMemberCallState[] = []; // Sanitize existing member state event if (memberStateEvent && Array.isArray(memberStateEvent["m.calls"])) { @@ -665,7 +676,7 @@ export class GroupCall extends EventEmitter { return; } - let callsState = event.getContent()["m.calls"]; + let callsState = event.getContent()["m.calls"]; if (Array.isArray(callsState)) { callsState = callsState.filter((call) => !!call); @@ -707,7 +718,7 @@ export class GroupCall extends EventEmitter { return; } - let opponentDevice: IGroupCallRoomMemberDevice; + let opponentDevice: IGroupCallMemberDevice; let peerUserId: string; let existingCall: MatrixCall; @@ -748,15 +759,16 @@ export class GroupCall extends EventEmitter { else { peerUserId = this.localSfu; opponentDevice = { - "device_id": this.localSfuDeviceId; - "session_id": ""; // we can use a blank session_id, as the SFU is stateless - "feeds": []; + "device_id": this.localSfuDeviceId, + "session_id": "", // we can use a blank session_id, as the SFU is stateless + "feeds": [], } existingCall = this.getCallByUserId(peerUserId); } + let newCall: MatrixCall; if (!this.localSfu || !existingCall) { - const newCall = createNewMatrixCall( + newCall = createNewMatrixCall( this.client, this.room.roomId, { @@ -786,14 +798,14 @@ export class GroupCall extends EventEmitter { if (this.localSfu) { // TODO: only subscribe to the streams you care about - remoteDevice = this.getDeviceForMember(member.userId); + const remoteDevice = this.getDeviceForMember(member.userId); // subscribe if we have a call established to the SFU // if not, we'll trigger a subscription when the call sets up. this.subscribeStream(existingCall || newCall, member.userId, remoteDevice); } }; - private subscribeStream(call: MatrixCall, userId: string, device: IGroupCallRoomMemberDevice, streamId?: string) { + private subscribeStream(call: MatrixCall, userId: string, device: IGroupCallMemberDevice, streamId?: string) { // Asks the SFU to subscribe to the given streams originating from a given device. // if streamId is undefined, subscribe to all of them. @@ -821,14 +833,14 @@ export class GroupCall extends EventEmitter { })); } - public getDeviceForMember(userId: string): IGroupCallRoomMemberDevice { + public getDeviceForMember(userId: string): IGroupCallMemberDevice { const memberStateEvent = this.room.currentState.getStateEvents(EventType.GroupCallMemberPrefix, userId); if (!memberStateEvent) { return undefined; } - const memberState = memberStateEvent.getContent(); + const memberState = memberStateEvent.getContent(); const memberGroupCallState = memberState["m.calls"]?.find( (call) => call && call["m.call_id"] === this.groupCallId); From 0958b45faa861b6ea264a910f4c2eedaad2d4010 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 31 May 2022 12:36:11 +0100 Subject: [PATCH 003/137] fix lint --- src/webrtc/groupCall.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 841b5e7077f..4053fbdc306 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -83,7 +83,6 @@ export interface IGroupCallMemberFeed { id: string; purpose: SDPStreamMetadataPurpose; tracks: IGroupCallMemberTrack[]; - // TODO: Sources for adaptive bitrate } export interface IGroupCallMemberDevice { @@ -755,14 +754,13 @@ export class GroupCall extends EventEmitter { ) { return; } - } - else { + } else { peerUserId = this.localSfu; opponentDevice = { "device_id": this.localSfuDeviceId, "session_id": "", // we can use a blank session_id, as the SFU is stateless "feeds": [], - } + }; existingCall = this.getCallByUserId(peerUserId); } @@ -815,7 +813,7 @@ export class GroupCall extends EventEmitter { for (const track of feed.tracks) { streams.push({ "stream_id": feed.id, - "track_id": track.id + "track_id": track.id, }); } } @@ -829,7 +827,7 @@ export class GroupCall extends EventEmitter { call.dataChannel.send(JSON.stringify({ "op": "select", "conf_id": this.groupCallId, - "start": streams + "start": streams, })); } From bc3d326fbf2004ffbb0984e65157264891f64a17 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 4 Jun 2022 02:20:16 +0100 Subject: [PATCH 004/137] defloatify settings --- src/webrtc/groupCall.ts | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index ac18a203990..0dfa19119ea 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -139,6 +139,23 @@ function getCallUserId(call: MatrixCall): string | null { return call.getOpponentMember()?.userId || call.invitee || null; } +function defloat(json: Object): Object { + for (const key of Object.keys(json)) { + if (isFloat(json[key])) { + json[key] = "" + json[key]; + } + } + return json; +} + +function isFloat(value) { + return ( + typeof value === 'number' && + !Number.isNaN(value) && + !Number.isInteger(value) + ); +} + export class GroupCall extends TypedEventEmitter { // Config public activeSpeakerInterval = 1000; @@ -658,8 +675,8 @@ export class GroupCall extends TypedEventEmitter ({ "id": track.id, "kind": track.kind, - "label": track.label, - "settings": track.getSettings(), + // "label": track.label, // feels a bit invasive + "settings": defloat(track.getSettings()), })), })), // TODO: Add data channels From cab5a704585ed767567d66c71aa11f54804f6121 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 4 Jun 2022 13:16:11 +0100 Subject: [PATCH 005/137] fix lint --- src/webrtc/groupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 0dfa19119ea..448ddf9e9a7 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -93,7 +93,7 @@ export interface IGroupCallDataChannelOptions { export interface IGroupCallMemberTrack { id: string; kind: string; // TODO: use an enum - label: string; + // label: string; // removing as too privacy invasive settings: MediaTrackSettings; } From fb671cf359576e6d1e0849a3737b7ba9cba20483 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 4 Jun 2022 23:30:27 +0100 Subject: [PATCH 006/137] switch SFU config to matrixClient opts --- src/client.ts | 25 +++++++++++++++++++++++-- src/webrtc/call.ts | 5 +++-- src/webrtc/groupCall.ts | 16 +++++++--------- 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/client.ts b/src/client.ts index 380e2751c5f..d9159752925 100644 --- a/src/client.ts +++ b/src/client.ts @@ -355,6 +355,21 @@ export interface ICreateClientOpts { fallbackICEServerAllowed?: boolean; cryptoCallbacks?: ICryptoCallbacks; + + /** + * The user ID for the local SFU to use for group calling, if any + */ + localSfu?: string; + + /** + * The device ID for the local SFU to use for group calling, if any + */ + localSfuDeviceId?: string; + + /** + * False to disable E2EE for to-device calls. Default true. + */ + encryptedCalls?: boolean; } export interface IMatrixClientCreateOpts extends ICreateClientOpts { @@ -953,6 +968,10 @@ export class MatrixClient extends TypedEventEmitter>(); + public localSfu: string; + public localSfuDeviceId: string; + public encryptedCalls: boolean; + constructor(opts: IMatrixClientCreateOpts) { super(); @@ -1022,6 +1041,10 @@ export class MatrixClient extends TypedEventEmitter Date: Sun, 5 Jun 2022 01:32:33 +0100 Subject: [PATCH 007/137] fix configurable call encryption --- src/webrtc/call.ts | 50 +++++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index e2748a63b12..db9b231c011 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2063,38 +2063,42 @@ export class MatrixCall extends TypedEventEmitter Date: Sun, 5 Jun 2022 13:16:18 +0100 Subject: [PATCH 008/137] hook up SDP nego over DC --- src/webrtc/call.ts | 2 +- src/webrtc/groupCall.ts | 100 ++++++++++++++++++++++++++++++++++------ 2 files changed, 88 insertions(+), 14 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index db9b231c011..ec5f818cc4b 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2095,7 +2095,7 @@ export class MatrixCall extends TypedEventEmitter = new Map(); private reEmitter: ReEmitter; private transmitTimer: ReturnType | null = null; + private dcTid: number; constructor( private client: MatrixClient, @@ -821,9 +837,9 @@ export class GroupCall extends TypedEventEmitter = new Promise(resolve => {}); + call.dataChannel.onopen = () => { + Promise.resolve(p); + }; + await p; + } + if (call.dataChannel.readyState !== 'open') { + logger.warn("Can't sent to DC in state", call.dataChannel.readyState); + } + + // FIXME: play nice with application-layer use of the DC + // + // TODO: rather than gutwrenching into our MatrixCall's peerConnection, + // should this be handled inside MatrixCall instead? + // + // FIXME: RPC reliability over DC + const msg: ISfuDataChannelMessage = { "op": "select", "conf_id": this.groupCallId, + "id": "" + (this.dcTid++), "start": streams, - })); + "sdp": call.peerConn.localDescription.sdp, + }; + + call.dataChannel.send(JSON.stringify(msg)); + } + + private async onDataChannelMessage(call: MatrixCall, event: MessageEvent) { + // FIXME: this feels like it should be on MatrixCall rather than gutwrenching + let json: ISfuDataChannelMessage; + try { + json = JSON.parse(event.data); + } catch (e) { + logger.warn("Ignoring non-JSON DC event"); + return; + } + + if (!json.op) { + logger.warn("Ignoring unrecognised DC event"); + return; + } + + if (json.op !== "answer") { + logger.warn("Ignoring unrecognised DC event op ", json.op); + return; + } + + try { + await call.peerConn.setRemoteDescription({ + "type": "answer", + "sdp": json.sdp, + }); + } catch (e) { + logger.debug(`Call ${call.callId} Failed to set remote description`, e); + // fixme: terminate is private + //call.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); + return; + } } public getDeviceForMember(userId: string): IGroupCallMemberDevice { From e3e1633b21145e732f03fe9b42425c4104756908 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 5 Jun 2022 18:29:37 +0100 Subject: [PATCH 009/137] better DC TIDs --- src/webrtc/groupCall.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index a74640edbdf..2631a663982 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -14,6 +14,7 @@ import { EventType } from "../@types/event"; import { CallEventHandlerEvent } from "./callEventHandler"; import { RoomStateEvent } from "../matrix"; import { GroupCallEventHandlerEvent } from "./groupCallEventHandler"; +import { randomString } from "../randomstring"; export enum GroupCallIntent { Ring = "m.ring", @@ -195,7 +196,6 @@ export class GroupCall extends TypedEventEmitter = new Map(); private reEmitter: ReEmitter; private transmitTimer: ReturnType | null = null; - private dcTid: number; constructor( private client: MatrixClient, @@ -831,7 +831,11 @@ export class GroupCall extends TypedEventEmitter Date: Sun, 5 Jun 2022 20:08:04 +0100 Subject: [PATCH 010/137] publish IDs from PC; ditch stream_id --- src/webrtc/groupCall.ts | 56 +++++++++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 2631a663982..c62392d2e24 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -99,7 +99,7 @@ export interface IGroupCallMemberTrack { } export interface IGroupCallMemberFeed { - id: string; + // id: string; // pc.getLocalStreams() is deprecated so we can't get a stream ID any more... purpose: SDPStreamMetadataPurpose; tracks: IGroupCallMemberTrack[]; } @@ -677,22 +677,48 @@ export class GroupCall extends TypedEventEmitter { const deviceId = this.client.getDeviceId(); + let feeds; + if (this.client.localSfu) { + if (this.calls.length != 0) { + logger.warn("Can't publish accurate m.call.member event as SFU call not set up yet"); + // placeholder feed content + feeds = this.getLocalFeeds().map((feed) => ({ + "purpose": feed.purpose, + })); + } else { + feeds = this.getLocalFeeds().map((feed) => ({ + "purpose": SDPStreamMetadataPurpose.Usermedia, // FIXME - track + + // we have to advertise the actual tracks we're sending to the SFU from the PC + // we can't use the feeds' mediaStream IDs, as they are local rather than the copy + // sent over WebRTC + // + // TODO: correctly track which rtpSenders are associated with which feed + // rather than assuming that all our senders are from this feed. + "tracks": this.calls[0].peerConn.getSenders().map((rtpSender) => ({ + "id": rtpSender.track.id, + "kind": rtpSender.track.kind, + "settings": defloat(rtpSender.track.getSettings()), + })), + })); + } + } + else { + feeds = this.getLocalFeeds().map((feed) => ({ + "purpose": feed.purpose, + // don't bother publishing feed details + // if we're not using an SFU, given each + // call will have a different ID. + })); + } + return this.updateMemberCallState({ "m.call_id": this.groupCallId, "m.devices": [ { "device_id": deviceId, "session_id": this.client.getSessionId(), - "feeds": this.getLocalFeeds().map((feed) => ({ - "purpose": feed.purpose, - "id": feed.stream.id, - "tracks": feed.stream.getTracks().map((track) => ({ - "id": track.id, - "kind": track.kind, - // "label": track.label, // feels a bit invasive - "settings": defloat(track.getSettings()), - })), - })), + "feeds": feeds, // TODO: Add data channels }, ], @@ -902,16 +928,14 @@ export class GroupCall extends TypedEventEmitter Date: Sun, 5 Jun 2022 20:37:31 +0100 Subject: [PATCH 011/137] publish feeds correctly --- src/webrtc/groupCall.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index c62392d2e24..eac21782f70 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -679,7 +679,7 @@ export class GroupCall extends TypedEventEmitter ({ @@ -805,7 +805,7 @@ export class GroupCall extends TypedEventEmitter Date: Mon, 6 Jun 2022 02:57:33 +0100 Subject: [PATCH 012/137] scrape track IDs from SDP :/ --- src/webrtc/groupCall.ts | 83 +++++++++++++++++++++++++++++++++-------- 1 file changed, 67 insertions(+), 16 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index eac21782f70..e9e4aabed57 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -172,6 +172,48 @@ function isFloat(value) { ); } +interface IMediaBlock { + mid?: string; + trackDesc?: ISfuTrackDesc; +} + +function getTrackDesc(sdp: string, mid: string): ISfuTrackDesc | undefined { + // sdp mangling to grab the a=msid: line out of SDP for a given mid + if (!sdp) return; + + const mediaByMids: Map = new Map(); + let mediaBlock: IMediaBlock = {}; + let matches; + for (const line of sdp.split(/\r?\n/)) { + if (line.match(/^m=/)) { + if (mediaBlock.mid !== undefined) { + mediaByMids.set(mediaBlock.mid, mediaBlock); + } + mediaBlock = {}; + } + matches = line.match(/^a=mid:(.*?)$/); + if (matches) { + mediaBlock.mid = matches[1]; + } + matches = line.match(/^a=msid:(.*?) (.*?)$/); + if (matches) { + mediaBlock.trackDesc = { + stream_id: matches[1], + track_id: matches[2], + }; + } + } + if (mediaBlock.mid) { + mediaByMids.set(mediaBlock.mid, mediaBlock); + } + + if (mediaByMids.get(mid)) { + return mediaByMids.get(mid).trackDesc; + } else { + return; + } +} + export class GroupCall extends TypedEventEmitter { // Config public activeSpeakerInterval = 1000; @@ -683,8 +725,8 @@ export class GroupCall extends TypedEventEmitter ({ - "purpose": feed.purpose, - })); + "purpose": feed.purpose, + })); } else { feeds = this.getLocalFeeds().map((feed) => ({ "purpose": SDPStreamMetadataPurpose.Usermedia, // FIXME - track @@ -695,15 +737,15 @@ export class GroupCall extends TypedEventEmitter ({ - "id": rtpSender.track.id, - "kind": rtpSender.track.kind, - "settings": defloat(rtpSender.track.getSettings()), + "tracks": this.calls[0].peerConn.getTransceivers().map(transceiver => ({ + "id": getTrackDesc(this.calls[0].peerConn.localDescription?.sdp, transceiver.mid)?.track_id, + "kind": transceiver.sender.track.kind, + "settings": defloat(transceiver.sender.track.getSettings()), })), })); + console.warn("calculated SFU feeds as", feeds); } - } - else { + } else { feeds = this.getLocalFeeds().map((feed) => ({ "purpose": feed.purpose, // don't bother publishing feed details @@ -913,9 +955,6 @@ export class GroupCall extends TypedEventEmitter = new Promise(resolve => {}); @@ -1229,6 +1276,10 @@ export class GroupCall extends TypedEventEmitter Date: Mon, 6 Jun 2022 03:08:52 +0100 Subject: [PATCH 013/137] get signalling over DC working --- src/webrtc/groupCall.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index e9e4aabed57..31050ed1586 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -1007,6 +1007,10 @@ export class GroupCall extends TypedEventEmitter Date: Mon, 6 Jun 2022 03:24:06 +0100 Subject: [PATCH 014/137] don't barf on SFU having no room member --- src/webrtc/call.ts | 6 ++++++ src/webrtc/groupCall.ts | 2 ++ 2 files changed, 8 insertions(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index ec5f818cc4b..6dd17798320 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2463,6 +2463,12 @@ export class MatrixCall extends TypedEventEmitter { diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 31050ed1586..71e9550ac62 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -1285,8 +1285,10 @@ export class GroupCall extends TypedEventEmitter Date: Mon, 6 Jun 2022 03:26:17 +0100 Subject: [PATCH 015/137] fix logging --- src/webrtc/groupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 71e9550ac62..c751b67d90f 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -743,7 +743,7 @@ export class GroupCall extends TypedEventEmitter ({ From 67c79573cafd74d2c442b84d6eb9ac196f5f3b27 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 6 Jun 2022 03:27:02 +0100 Subject: [PATCH 016/137] lint --- src/webrtc/groupCall.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index c751b67d90f..8f4d7b40e69 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -977,8 +977,7 @@ export class GroupCall extends TypedEventEmitter Date: Tue, 12 Jul 2022 16:11:35 +0100 Subject: [PATCH 017/137] uncommited comments --- src/webrtc/groupCall.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 8f4d7b40e69..2902ddca01b 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -1275,17 +1275,17 @@ export class GroupCall extends TypedEventEmitter Date: Sat, 30 Jul 2022 18:10:47 +0200 Subject: [PATCH 018/137] Add useful logging and comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 746a489f75c..6ea155112ed 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -1054,6 +1054,14 @@ export class GroupCall extends TypedEventEmitter< logger.info("Subscribing to ", streams, userId); } + call.dataChannel.onclose = () => { + logger.error("DC was closed"); + }; + + call.dataChannel.onerror = (error) => { + logger.error("DC has errored", error); + }; + if (call.dataChannel.readyState === 'connecting') { const p: Promise = new Promise(resolve => {}); call.dataChannel.onopen = () => { @@ -1084,6 +1092,7 @@ export class GroupCall extends TypedEventEmitter< }; call.dataChannel.send(JSON.stringify(msg)); + logger.warn("Sent select message over DC", msg); } private async onDataChannelMessage(call: MatrixCall, event: MessageEvent) { @@ -1333,7 +1342,7 @@ export class GroupCall extends TypedEventEmitter< this.retryCallCounts.delete(getCallUserId(call)); if (this.client.localSfu) { - // now we know what our feed IDs are, we can publish them + // now we know what our stream IDs are, we can publish them // so others can subscribe to us... this.sendMemberStateEvent(); @@ -1343,6 +1352,7 @@ export class GroupCall extends TypedEventEmitter< for (const stateEvent of memberStateEvents) { const userId = stateEvent.getStateKey(); // don't try to subscribe to our own feed(!) + // TODO: What happens if we do? if (userId === localUserId) continue; const device = this.getDeviceForMember(userId); this.subscribeStream(call, userId, device); From f7ec3abbd74af9eb924a2aa7fcb50054697991c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 30 Jul 2022 18:55:29 +0200 Subject: [PATCH 019/137] Choose opponent more correctly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 336710eb0ac..d8dd2e73bd2 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2591,6 +2591,7 @@ export class MatrixCall extends TypedEventEmitter(); + const sender = ev.getSender(); logger.debug(`Call ${this.callId} choosing opponent party ID ${msg.party_id}`); @@ -2606,13 +2607,9 @@ export class MatrixCall extends TypedEventEmitter { From 89c96950f25df9f858635b2dd95e4c3a34133fa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 30 Jul 2022 18:59:09 +0200 Subject: [PATCH 020/137] Disable E2E when we are using an SFU MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/client.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/client.ts b/src/client.ts index 7468ad31db4..647ac353d9c 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1543,6 +1543,8 @@ export class MatrixClient extends TypedEventEmitter Date: Sat, 30 Jul 2022 19:00:53 +0200 Subject: [PATCH 021/137] Don't warn about missing `SDPStreamMetadata` on SFU calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index d8dd2e73bd2..755dd2f752d 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -843,7 +843,7 @@ export class MatrixCall extends TypedEventEmitter Date: Sun, 31 Jul 2022 21:43:56 +0200 Subject: [PATCH 022/137] Make basic group calling work MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 125 ++++++++++--- src/webrtc/callEventTypes.ts | 5 +- src/webrtc/groupCall.ts | 333 ++++++++++++----------------------- 3 files changed, 215 insertions(+), 248 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 755dd2f752d..a7dc903e5cd 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -50,7 +50,7 @@ import { MatrixClient } from "../client"; import { ISendEventResponse } from "../@types/requests"; import { EventEmitterEvents, TypedEventEmitter } from "../models/typed-event-emitter"; import { DeviceInfo } from '../crypto/deviceinfo'; -import { GroupCallUnknownDeviceError } from './groupCall'; +import { GroupCallUnknownDeviceError, ISfuDataChannelMessage } from './groupCall'; import { IScreensharingOpts } from "./mediaHandler"; // events: hangup, error(err), replaced(call), state(state, oldState) @@ -80,6 +80,7 @@ interface CallOpts { opponentDeviceId?: string; opponentSessionId?: string; groupCallId?: string; + initialRemoteSDPStreamMetadata?: SDPStreamMetadata; } interface TurnServer { @@ -404,6 +405,7 @@ export class MatrixCall extends TypedEventEmitter feed.purpose === purpose); - if (existingFeed) { - existingFeed.setNewStream(stream); - } else { - this.feeds.push(new CallFeed({ - client: this.client, - roomId: this.roomId, - userId, - stream, - purpose, - audioMuted, - videoMuted, - })); - this.emit(CallEvent.FeedsChanged, this.feeds); + if (this.getFeedByStreamId(stream.id)) { + logger.warn(`Ignoring stream with id ${stream.id} because we already have a feed for it`); + return; } - logger.info(`Call ${this.callId} Pushed remote stream (id="${ - stream.id}", active="${stream.active}", purpose=${purpose})`); + this.feeds.push(new CallFeed({ + client: this.client, + roomId: this.roomId, + userId, + stream, + purpose, + audioMuted, + videoMuted, + })); + this.emit(CallEvent.FeedsChanged, this.feeds); + + logger.info( + `Call ${this.callId} Pushed remote stream (` + + `id="${stream.id}" ` + + `active="${stream.active}" ` + + `purpose=${purpose} ` + + `userId=${userId}` + + `)`, + ); } /** @@ -1811,7 +1818,62 @@ export class MatrixCall extends TypedEventEmitter => { + // TODO: play nice with application layer DC listeners + + let json: ISfuDataChannelMessage; + try { + json = JSON.parse(event.data); + } catch (e) { + logger.warn("Ignoring non-JSON DC event:", event.data); + return; + } + + if (!json.op) { + logger.warn("Ignoring unrecognized DC event:", json.op); + return; + } + + logger.warn(`Received DC ${json.op} event`, json); + + switch (json.op) { + case "offer": + try { + await this.peerConn.setRemoteDescription({ + "type": "offer", + "sdp": json.sdp, + }); + + const answer = await this.peerConn.createAnswer(); + await this.peerConn.setLocalDescription(answer); + + const msg: ISfuDataChannelMessage = { + "op": "answer", + "conf_id": this.groupCallId, + "id": Date.now() + randomString(5), + "sdp": answer.sdp, + }; + + this.dataChannel.send(JSON.stringify(msg)); + logger.warn("Sent answer message over DC", msg); + } catch (e) { + logger.debug(`Call ${this.callId} Failed to set remote description`, e); + this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); + return; + } + break; + case "error": + logger.error("Received DC error event", json.message); + + break; + default: + logger.warn("Ignoring unrecognized DC event op ", json.op); + + break; + } + }; + + public updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { this.remoteSDPStreamMetadata = utils.recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); for (const feed of this.getRemoteFeeds()) { const streamId = feed.stream.id; @@ -2078,9 +2140,15 @@ export class MatrixCall extends TypedEventEmitter { - this.emit(CallEvent.DataChannel, ev.channel); + this.setupDataChannel(ev.channel); }; + private setupDataChannel(dataChannel: RTCDataChannel): void { + this.dataChannel = dataChannel; + this.emit(CallEvent.DataChannel, dataChannel); + this.dataChannel.addEventListener("message", this.onDataChannelMessage); + } + /** * This method removes all video/rtx codecs from screensharing video * transceivers. This is necessary since they can cause problems. Without @@ -2714,6 +2782,7 @@ export function createNewMatrixCall(client: any, roomId: string, options?: CallO opponentDeviceId: options?.opponentDeviceId, opponentSessionId: options?.opponentSessionId, groupCallId: options?.groupCallId, + initialRemoteSDPStreamMetadata: options?.initialRemoteSDPStreamMetadata, }; const call = new MatrixCall(opts); diff --git a/src/webrtc/callEventTypes.ts b/src/webrtc/callEventTypes.ts index 4f43a70a1d5..aae2f54880e 100644 --- a/src/webrtc/callEventTypes.ts +++ b/src/webrtc/callEventTypes.ts @@ -13,8 +13,9 @@ export enum SDPStreamMetadataPurpose { export interface SDPStreamMetadataObject { purpose: SDPStreamMetadataPurpose; - audio_muted: boolean; - video_muted: boolean; + userId?: string; + audio_muted?: boolean; + video_muted?: boolean; } export interface SDPStreamMetadata { diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 6ea155112ed..1631f867357 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -13,7 +13,7 @@ import { RoomMember } from "../models/room-member"; import { Room } from "../models/room"; import { logger } from "../logger"; import { ReEmitter } from "../ReEmitter"; -import { SDPStreamMetadataPurpose } from "./callEventTypes"; +import { SDPStreamMetadata, SDPStreamMetadataPurpose } from "./callEventTypes"; import { createNewMatrixCall } from "./call"; import { ISendEventResponse } from "../@types/requests"; import { MatrixEvent } from "../models/event"; @@ -22,6 +22,7 @@ import { CallEventHandlerEvent } from "./callEventHandler"; import { GroupCallEventHandlerEvent } from "./groupCallEventHandler"; import { randomString } from "../randomstring"; import { IScreensharingOpts } from "./mediaHandler"; +import { recursivelyAssign } from "../utils"; export enum GroupCallIntent { Ring = "m.ring", @@ -112,7 +113,7 @@ export interface IGroupCallMemberTrack { } export interface IGroupCallMemberFeed { - // id: string; // pc.getLocalStreams() is deprecated so we can't get a stream ID any more... + id: string; purpose: SDPStreamMetadataPurpose; tracks: IGroupCallMemberTrack[]; } @@ -131,7 +132,7 @@ export interface IGroupCallMemberCallState { export interface ISfuTrackDesc { "stream_id": string; - "track_id": string; + "track_id"?: string; } export interface ISfuDataChannelMessage { @@ -266,6 +267,7 @@ export class GroupCall extends TypedEventEmitter< private transmitTimer: ReturnType | null = null; private memberStateExpirationTimers: Map> = new Map(); private resendMemberStateTimer: ReturnType | null = null; + private subscribedStreams: ISfuTrackDesc[] = []; constructor( private client: MatrixClient, @@ -764,40 +766,24 @@ export class GroupCall extends TypedEventEmitter< } private async sendMemberStateEvent(): Promise { - let feeds; - if (this.client.localSfu) { - if (this.calls.length == 0) { - logger.warn("Can't publish accurate m.call.member event as SFU call not set up yet"); - // placeholder feed content - feeds = this.getLocalFeeds().map((feed) => ({ - "purpose": feed.purpose, - })); - } else { - feeds = this.getLocalFeeds().map((feed) => ({ - "purpose": SDPStreamMetadataPurpose.Usermedia, // FIXME - track - - // we have to advertise the actual tracks we're sending to the SFU from the PC - // we can't use the feeds' mediaStream IDs, as they are local rather than the copy - // sent over WebRTC - // - // TODO: correctly track which rtpSenders are associated with which feed - // rather than assuming that all our senders are from this feed. - "tracks": this.calls[0].peerConn.getTransceivers().map(transceiver => ({ - "id": getTrackDesc(this.calls[0].peerConn.localDescription?.sdp, transceiver.mid)?.track_id, - "kind": transceiver.sender.track.kind, - "settings": defloat(transceiver.sender.track.getSettings()), - })), - })); - logger.warn("calculated SFU feeds as", feeds); - } - } else { - feeds = this.getLocalFeeds().map((feed) => ({ - "purpose": feed.purpose, - // don't bother publishing feed details - // if we're not using an SFU, given each - // call will have a different ID. - })); - } + const feeds = this.getLocalFeeds().map((feed) => ({ + purpose: feed.purpose, + id: feed.stream.id, + + // we have to advertise the actual tracks we're sending to the SFU from the PC + // we can't use the feeds' mediaStream IDs, as they are local rather than the copy + // sent over WebRTC + // + // TODO: correctly track which rtpSenders are associated with which feed + // rather than assuming that all our senders are from this feed. + tracks: this.calls[0] + ? this.calls[0].peerConn.getTransceivers().map(transceiver => ({ + "id": getTrackDesc(this.calls[0].peerConn.localDescription?.sdp, transceiver.mid)?.track_id, + "kind": transceiver.sender.track.kind, + "settings": defloat(transceiver.sender.track.getSettings()), + })) + : undefined, + })); const send = () => this.updateMemberCallState({ "m.call_id": this.groupCallId, @@ -861,6 +847,33 @@ export class GroupCall extends TypedEventEmitter< return this.client.sendStateEvent(this.room.roomId, EventType.GroupCallMemberPrefix, content, localUserId); } + private getRemoteFeedsFromState(): IGroupCallMemberFeed[] { + return this.getMemberStateEvents()?.reduce((feeds, event) => { + if (event.getSender() === this.client.getUserId()) return feeds; // Ignore local + const newFeeds = event.getContent()?.["m.calls"]?.[0]?.["m.devices"]?.[0]?.feeds; + if (!newFeeds) return feeds; + return [...feeds, ...newFeeds]; + }, []) ?? []; + } + + private getRemoteSDPStreamMetadataForCall(): SDPStreamMetadata { + return this.getMemberStateEvents().reduce((metaAcc: SDPStreamMetadata, event: MatrixEvent) => { + if (event.getSender() === this.client.getUserId()) return metaAcc; // Ignore local + const feeds = event.getContent()?.["m.calls"]?.[0]?.["m.devices"]?.[0]?.feeds; + if (!feeds) return metaAcc; + const metadata = feeds.reduce((feedAcc: SDPStreamMetadata, feed: IGroupCallMemberFeed) => { + if (!feed?.id) return feedAcc; + return recursivelyAssign(feedAcc, { + [feed.id]: { + purpose: feed.purpose, + userId: event.getSender(), + }, + }, true); + }, {}); + return recursivelyAssign(metaAcc, metadata, true); + }, {}); + } + public onMemberStateChanged = async (event: MatrixEvent) => { // The member events may be received for another room, which we will ignore. if (event.getRoomId() !== this.room.roomId) return; @@ -920,11 +933,24 @@ export class GroupCall extends TypedEventEmitter< return; } - let opponentDevice; + let opponentDevice: IGroupCallMemberDevice; let peerUserId: string; let existingCall: MatrixCall; - if (!this.client.localSfu) { + if (this.client.localSfu) { + peerUserId = this.client.localSfu; + opponentDevice = { + "device_id": this.client.localSfuDeviceId, + // XXX: the SFU might need to specify a session_id so that if it + // restarts and starts sending invites to us, we know that it's + // forgotten who we were? But then we need a way to communicate + // the session_id to the clients, which is tough if the SFU is + // not in the right room. + "session_id": "sfu", + "feeds": [], + }; + existingCall = this.getCallByUserId(peerUserId); + } else { // Only initiate a call with a user who has a userId that is // lexicographically less than your own. Otherwise, that user will // call you. @@ -954,23 +980,12 @@ export class GroupCall extends TypedEventEmitter< ) { return; } - } else { - peerUserId = this.client.localSfu; - opponentDevice = { - "device_id": this.client.localSfuDeviceId, - // XXX: the SFU might need to specify a session_id so that if it - // restarts and starts sending invites to us, we know that it's - // forgotten who we were? But then we need a way to communicate - // the session_id to the clients, which is tough if the SFU is - // not in the right room. - "session_id": "sfu", - "feeds": [], - }; - existingCall = this.getCallByUserId(peerUserId); } - if (!this.client.localSfu || - (this.client.localSfu && !existingCall)) { + if ( + !this.client.localSfu || + (this.client.localSfu && !existingCall) + ) { const newCall = createNewMatrixCall( this.client, this.room.roomId, @@ -979,6 +994,9 @@ export class GroupCall extends TypedEventEmitter< opponentDeviceId: opponentDevice.device_id, opponentSessionId: opponentDevice.session_id, groupCallId: this.groupCallId, + initialRemoteSDPStreamMetadata: this.client.localSfu + ? this.getRemoteSDPStreamMetadataForCall() + : undefined, }, ); @@ -988,9 +1006,10 @@ export class GroupCall extends TypedEventEmitter< (feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); try { - // Safari can't send a MediaStream to multiple sources, so clone it await newCall.placeCallWithCallFeeds( - this.getLocalFeeds().map(feed => feed.clone()), + this.client.localSfu === peerUserId + ? this.getLocalFeeds() // TODO: We should just setup the datachannel + : this.getLocalFeeds().map(feed => feed.clone()), // Safari can't send a MediaStream to multiple sources, so clone it requestScreenshareFeed, ); } catch (e) { @@ -1014,71 +1033,46 @@ export class GroupCall extends TypedEventEmitter< } else { this.addCall(newCall); } - - if (this.client.localSfu) { - // TODO: play nice with application layer DC listeners - newCall.dataChannel.onmessage = this.onDataChannelMessage.bind(this, newCall); - } - } - - if (this.client.localSfu && existingCall) { + } else if (this.client.localSfu && existingCall) { // subscribe if we already had an existing call (otherwise // we'll subscribe on the new call being set up) - const remoteDevice = this.getDeviceForMember(member.userId); - // TODO: only subscribe to the streams you care about - this.subscribeStream(existingCall, member.userId, remoteDevice); + existingCall.updateRemoteSDPStreamMetadata(this.getRemoteSDPStreamMetadataForCall()); + this.subscribeToSFU(existingCall, this.getRemoteFeedsFromState()); } }; - private async subscribeStream(call: MatrixCall, userId: string, device: IGroupCallMemberDevice) { - // Asks the SFU to subscribe to the tracks originating from a given device. - - const streams: ISfuTrackDesc[] = []; - for (const feed of device.feeds) { - if (!feed.tracks) { - logger.warn(`missing feeds for ${userId}`); - } else { - for (const track of feed.tracks) { - streams.push({ - "stream_id": "unknown", - "track_id": track.id, - }); - } - } + private async waitForDatachannelToBeOpen(call: MatrixCall): Promise { + if (call.dataChannel.readyState === 'connecting') { + const p = new Promise(resolve => { + call.dataChannel.onopen = () => resolve(); + call.dataChannel.onclose = () => resolve(); + }); + await p; } + return; + } - if (streams.length == 0) { - logger.warn("Failed to find any streams to subscribe to"); + private async subscribeToSFU(call: MatrixCall, feeds: IGroupCallMemberFeed[]) { + await this.waitForDatachannelToBeOpen(call); + if (call.dataChannel.readyState !== "open") { + logger.warn("Can't sent to DC in state:", call.dataChannel.readyState); return; - } else { - logger.info("Subscribing to ", streams, userId); } - call.dataChannel.onclose = () => { - logger.error("DC was closed"); - }; - - call.dataChannel.onerror = (error) => { - logger.error("DC has errored", error); - }; + // Only subscribe to streams we aren't already subscribed to + const streams: ISfuTrackDesc[] = feeds.filter((feed) => { + if (!feed.tracks) return false; // If we don't have info about tracks, the SFU won't have them either + return !this.subscribedStreams.find((stream) => stream.stream_id === feed.id); + }).map((feed) => ({ stream_id: feed.id })); - if (call.dataChannel.readyState === 'connecting') { - const p: Promise = new Promise(resolve => {}); - call.dataChannel.onopen = () => { - Promise.resolve(p); - }; - await p; - } - if (call.dataChannel.readyState !== 'open') { - logger.warn("Can't sent to DC in state", call.dataChannel.readyState); + if (streams.length === 0) { + logger.warn("Failed to find any new streams to subscribe to"); + return; + } else { + this.subscribedStreams.push(...streams); + logger.warn("Subscribing to:", streams); } - // we have to create a new offer first so we can answer it - const offer = await call.peerConn.createOffer(); - call.peerConn.setLocalDescription(offer); - - // FIXME: play nice with application-layer use of the DC - // // TODO: rather than gutwrenching into our MatrixCall's peerConnection, // should this be handled inside MatrixCall instead? // @@ -1088,46 +1082,12 @@ export class GroupCall extends TypedEventEmitter< "conf_id": this.groupCallId, "id": Date.now() + randomString(5), "start": streams, - "sdp": call.peerConn.localDescription.sdp, }; call.dataChannel.send(JSON.stringify(msg)); logger.warn("Sent select message over DC", msg); } - private async onDataChannelMessage(call: MatrixCall, event: MessageEvent) { - // FIXME: this feels like it should be on MatrixCall rather than gutwrenching - let json: ISfuDataChannelMessage; - try { - json = JSON.parse(event.data); - } catch (e) { - logger.warn("Ignoring non-JSON DC event"); - return; - } - - if (!json.op) { - logger.warn("Ignoring unrecognised DC event"); - return; - } - - if (json.op !== "answer") { - logger.warn("Ignoring unrecognised DC event op ", json.op); - return; - } - - try { - await call.peerConn.setRemoteDescription({ - "type": "answer", - "sdp": json.sdp, - }); - } catch (e) { - logger.debug(`Call ${call.callId} Failed to set remote description`, e); - // fixme: terminate is private - //call.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); - return; - } - } - public getDeviceForMember(userId: string): IGroupCallMemberDevice { const memberStateEvent = this.getMemberStateEvents(userId); @@ -1284,39 +1244,12 @@ export class GroupCall extends TypedEventEmitter< } private onCallFeedsChanged = (call: MatrixCall) => { - const opponentMemberId = getCallUserId(call); - - if (!opponentMemberId) { - throw new Error("Cannot change call feeds without user id"); - } - - const currentUserMediaFeed = this.getUserMediaFeedByUserId(opponentMemberId); - const remoteUsermediaFeed = call.remoteUsermediaFeed; - const remoteFeedChanged = remoteUsermediaFeed !== currentUserMediaFeed; - - if (remoteFeedChanged) { - if (!currentUserMediaFeed && remoteUsermediaFeed) { - this.addUserMediaFeed(remoteUsermediaFeed); - } else if (currentUserMediaFeed && remoteUsermediaFeed) { - this.replaceUserMediaFeed(currentUserMediaFeed, remoteUsermediaFeed); - } else if (currentUserMediaFeed && !remoteUsermediaFeed) { - this.removeUserMediaFeed(currentUserMediaFeed); - } - } - - const currentScreenshareFeed = this.getScreenshareFeedByUserId(opponentMemberId); - const remoteScreensharingFeed = call.remoteScreensharingFeed; - const remoteScreenshareFeedChanged = remoteScreensharingFeed !== currentScreenshareFeed; - - if (remoteScreenshareFeedChanged) { - if (!currentScreenshareFeed && remoteScreensharingFeed) { - this.addScreenshareFeed(remoteScreensharingFeed); - } else if (currentScreenshareFeed && remoteScreensharingFeed) { - this.replaceScreenshareFeed(currentScreenshareFeed, remoteScreensharingFeed); - } else if (currentScreenshareFeed && !remoteScreensharingFeed) { - this.removeScreenshareFeed(currentScreenshareFeed); - } - } + call.getRemoteFeeds().filter((cf) => { + return !this.userMediaFeeds.find((gf) => gf.stream.id === cf.stream.id); + }).forEach((feed) => { + if (feed.purpose === SDPStreamMetadataPurpose.Usermedia) this.addUserMediaFeed(feed); + else if (feed.purpose === SDPStreamMetadataPurpose.Screenshare) this.addScreenshareFeed(feed); + }); }; private onCallStateChanged = (call: MatrixCall, state: CallState, _oldState: CallState) => { @@ -1341,22 +1274,13 @@ export class GroupCall extends TypedEventEmitter< if (state === CallState.Connected) { this.retryCallCounts.delete(getCallUserId(call)); - if (this.client.localSfu) { - // now we know what our stream IDs are, we can publish them - // so others can subscribe to us... - this.sendMemberStateEvent(); - - // if we're calling an SFU, subscribe to its feeds - const memberStateEvents = this.room.currentState.getStateEvents(EventType.GroupCallMemberPrefix); - const localUserId = this.client.getUserId(); - for (const stateEvent of memberStateEvents) { - const userId = stateEvent.getStateKey(); - // don't try to subscribe to our own feed(!) - // TODO: What happens if we do? - if (userId === localUserId) continue; - const device = this.getDeviceForMember(userId); - this.subscribeStream(call, userId, device); - } + // Now we know what our track IDs are, we can publish them so others + // can subscribe to us... + this.sendMemberStateEvent(); + + // if we're calling an SFU, subscribe to its feeds + if (call.getOpponentMember().userId === this.client.localSfu) { + this.subscribeToSFU(call, this.getRemoteFeedsFromState()); } } }; @@ -1383,20 +1307,6 @@ export class GroupCall extends TypedEventEmitter< this.emit(GroupCallEvent.UserMediaFeedsChanged, this.userMediaFeeds); } - private replaceUserMediaFeed(existingFeed: CallFeed, replacementFeed: CallFeed) { - const feedIndex = this.userMediaFeeds.findIndex((feed) => feed.userId === existingFeed.userId); - - if (feedIndex === -1) { - throw new Error("Couldn't find user media feed to replace"); - } - - this.userMediaFeeds.splice(feedIndex, 1, replacementFeed); - - existingFeed.dispose(); - replacementFeed.measureVolumeActivity(true); - this.emit(GroupCallEvent.UserMediaFeedsChanged, this.userMediaFeeds); - } - private removeUserMediaFeed(callFeed: CallFeed) { const feedIndex = this.userMediaFeeds.findIndex((feed) => feed.userId === callFeed.userId); @@ -1466,19 +1376,6 @@ export class GroupCall extends TypedEventEmitter< this.emit(GroupCallEvent.ScreenshareFeedsChanged, this.screenshareFeeds); } - private replaceScreenshareFeed(existingFeed: CallFeed, replacementFeed: CallFeed) { - const feedIndex = this.screenshareFeeds.findIndex((feed) => feed.userId === existingFeed.userId); - - if (feedIndex === -1) { - throw new Error("Couldn't find screenshare feed to replace"); - } - - this.screenshareFeeds.splice(feedIndex, 1, replacementFeed); - - existingFeed.dispose(); - this.emit(GroupCallEvent.ScreenshareFeedsChanged, this.screenshareFeeds); - } - private removeScreenshareFeed(callFeed: CallFeed) { const feedIndex = this.screenshareFeeds.findIndex((feed) => feed.userId === callFeed.userId); From b3177d52e0ec9ddf3f162b86580e17d0f8fae4aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 2 Aug 2022 09:18:25 +0200 Subject: [PATCH 023/137] Add `disposed` prop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/callFeed.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index c9133895212..e3a901a36ed 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -76,6 +76,7 @@ export class CallFeed extends TypedEventEmitter private speakingThreshold = SPEAKING_THRESHOLD; private speaking = false; private volumeLooperTimeout: ReturnType; + private disposed = false; constructor(opts: ICallFeedOpts) { super(); @@ -293,6 +294,11 @@ export class CallFeed extends TypedEventEmitter this.analyser = null; releaseContext(); } + this.disposed = true; + } + + public isDisposed(): boolean { + return this.disposed; } public getLocalVolume(): number { From 572c8549915c72a4b33105511dc64014e551a681 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 2 Aug 2022 09:55:38 +0200 Subject: [PATCH 024/137] Remove `disposed` feeds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 1631f867357..813ae6bd95d 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -1244,6 +1244,13 @@ export class GroupCall extends TypedEventEmitter< } private onCallFeedsChanged = (call: MatrixCall) => { + // Find removed feeds + [...this.userMediaFeeds, ...this.screenshareFeeds].filter((gf) => gf.isDisposed()).forEach((feed) => { + if (feed.purpose === SDPStreamMetadataPurpose.Usermedia) this.removeUserMediaFeed(feed); + else if (feed.purpose === SDPStreamMetadataPurpose.Screenshare) this.removeScreenshareFeed(feed); + }); + + // Find new feeds call.getRemoteFeeds().filter((cf) => { return !this.userMediaFeeds.find((gf) => gf.stream.id === cf.stream.id); }).forEach((feed) => { From 05be4663305c7a7a545977ca7f32e2411e71a035 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 2 Aug 2022 09:56:25 +0200 Subject: [PATCH 025/137] Reformat MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 813ae6bd95d..02e2230fbfd 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -1,7 +1,8 @@ import { TypedEventEmitter } from "../models/typed-event-emitter"; import { CallFeed, SPEAKING_THRESHOLD } from "./callFeed"; import { MatrixClient } from "../client"; -import { CallErrorCode, +import { + CallErrorCode, CallEvent, CallEventHandlerMap, CallState, From 854736a63216d628c29490cb7ece0b0a819b3f35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 2 Aug 2022 09:56:44 +0200 Subject: [PATCH 026/137] Clear `subscribedStreams` when ending call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 02e2230fbfd..ae736bb50cd 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -445,6 +445,7 @@ export class GroupCall extends TypedEventEmitter< } this.client.getMediaHandler().stopAllStreams(); + this.subscribedStreams = []; if (this.state !== GroupCallState.Entered) { return; From 7fca5611bc1dfe016694a2fa4f53d0739a9407bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 2 Aug 2022 09:57:07 +0200 Subject: [PATCH 027/137] A **very** ugly hack to make leaving work MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index ae736bb50cd..83ca2b2345a 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -1210,12 +1210,16 @@ export class GroupCall extends TypedEventEmitter< throw new Error("Cannot dispose call without user id"); } + // FIXME: We need a queue for onMemberStateEvent as if two events are + // received in rapid succession, we get two calls + const callHandlers = this.callHandlers.get(opponentMemberId); + if (!callHandlers) return; const { onCallFeedsChanged, onCallStateChanged, onCallHangup, onCallReplaced, - } = this.callHandlers.get(opponentMemberId); + } = callHandlers; call.removeListener(CallEvent.FeedsChanged, onCallFeedsChanged); call.removeListener(CallEvent.State, onCallStateChanged); From 3c2eb17b57b52b2f0c64e8ed1bd898f96a92c21a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 3 Aug 2022 09:42:27 +0200 Subject: [PATCH 028/137] Create a call with the SFU right when entering the conference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 214 ++++++++++++++++++++++------------------ 1 file changed, 120 insertions(+), 94 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 83ca2b2345a..43e65354e32 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -429,6 +429,52 @@ export class GroupCall extends TypedEventEmitter< this.retryCallLoopTimeout = setTimeout(this.onRetryCallLoop, this.retryCallInterval); this.onActiveSpeakerLoop(); + + if (this.client.localSfu) { + const opponentDevice = { + "device_id": this.client.localSfuDeviceId, + // XXX: the SFU might need to specify a session_id so that if it + // restarts and starts sending invites to us, we know that it's + // forgotten who we were? But then we need a way to communicate + // the session_id to the clients, which is tough if the SFU is + // not in the right room. + "session_id": "sfu", + "feeds": [], + }; + + const newCall = createNewMatrixCall( + this.client, + this.room.roomId, + { + invitee: this.client.localSfu, + opponentDeviceId: opponentDevice.device_id, + opponentSessionId: opponentDevice.session_id, + groupCallId: this.groupCallId, + initialRemoteSDPStreamMetadata: this.client.localSfu + ? this.getRemoteSDPStreamMetadataForCall() + : undefined, + }, + ); + + newCall.isPtt = this.isPtt; + + try { + await newCall.placeCallWithCallFeeds(this.getLocalFeeds()); // TODO: We should just setup the datachannel + newCall.createDataChannel("datachannel", this.dataChannelOptions); + } catch (e) { + logger.warn(`Failed to place call to ${this.client.localSfu}!`, e); + this.emit( + GroupCallEvent.Error, + new GroupCallError( + GroupCallErrorCode.PlaceCallFailed, + `Failed to place call to ${this.client.localSfu}.`, + ), + ); + return; + } + + this.addCall(newCall); + } } private dispose() { @@ -883,6 +929,17 @@ export class GroupCall extends TypedEventEmitter< const member = this.room.getMember(event.getStateKey()); if (!member) return; + if (this.client.localSfu) { + const sfuCall = this.getCallByUserId(this.client.localSfu); + if (!sfuCall) return; + // subscribe if we already had an existing call (otherwise + // we'll subscribe on the new call being set up) + sfuCall.updateRemoteSDPStreamMetadata(this.getRemoteSDPStreamMetadataForCall()); + this.subscribeToSFU(sfuCall, this.getRemoteFeedsFromState()); + + return; + } + const ignore = () => { this.removeParticipant(member); clearTimeout(this.memberStateExpirationTimers.get(member.userId)); @@ -935,111 +992,80 @@ export class GroupCall extends TypedEventEmitter< return; } - let opponentDevice: IGroupCallMemberDevice; - let peerUserId: string; - let existingCall: MatrixCall; - - if (this.client.localSfu) { - peerUserId = this.client.localSfu; - opponentDevice = { - "device_id": this.client.localSfuDeviceId, - // XXX: the SFU might need to specify a session_id so that if it - // restarts and starts sending invites to us, we know that it's - // forgotten who we were? But then we need a way to communicate - // the session_id to the clients, which is tough if the SFU is - // not in the right room. - "session_id": "sfu", - "feeds": [], - }; - existingCall = this.getCallByUserId(peerUserId); - } else { - // Only initiate a call with a user who has a userId that is - // lexicographically less than your own. Otherwise, that user will - // call you. - if (member.userId < localUserId) { - logger.log(`Waiting for ${member.userId} to send call invite.`); - return; - } - - opponentDevice = this.getDeviceForMember(member.userId); - if (!opponentDevice) { - logger.warn(`No opponent device found for ${member.userId}, ignoring.`); - this.emit( - GroupCallEvent.Error, - new GroupCallUnknownDeviceError(member.userId), - ); - return; - } + // Only initiate a call with a user who has a userId that is + // lexicographically less than your own. Otherwise, that user will + // call you. + if (member.userId < localUserId) { + logger.log(`Waiting for ${member.userId} to send call invite.`); + return; + } - peerUserId = member.userId; - existingCall = this.getCallByUserId(peerUserId); + const opponentDevice = this.getDeviceForMember(member.userId); - // if we already have an existing call to the same session on the - // other side, then use it - it must have already called us first. - if ( - existingCall && - existingCall.getOpponentSessionId() === opponentDevice.session_id - ) { - return; - } + if (!opponentDevice) { + logger.warn(`No opponent device found for ${member.userId}, ignoring.`); + this.emit( + GroupCallEvent.Error, + new GroupCallUnknownDeviceError(member.userId), + ); + return; } + const existingCall = this.getCallByUserId(member.userId); + + // if we already have an existing call to the same session on the + // other side, then use it - it must have already called us first. if ( - !this.client.localSfu || - (this.client.localSfu && !existingCall) + existingCall && + existingCall.getOpponentSessionId() === opponentDevice.session_id ) { - const newCall = createNewMatrixCall( - this.client, - this.room.roomId, - { - invitee: peerUserId, - opponentDeviceId: opponentDevice.device_id, - opponentSessionId: opponentDevice.session_id, - groupCallId: this.groupCallId, - initialRemoteSDPStreamMetadata: this.client.localSfu - ? this.getRemoteSDPStreamMetadataForCall() - : undefined, - }, - ); + return; + } - newCall.isPtt = this.isPtt; + const newCall = createNewMatrixCall( + this.client, + this.room.roomId, + { + invitee: member.userId, + opponentDeviceId: opponentDevice.device_id, + opponentSessionId: opponentDevice.session_id, + groupCallId: this.groupCallId, + initialRemoteSDPStreamMetadata: this.client.localSfu + ? this.getRemoteSDPStreamMetadataForCall() + : undefined, + }, + ); - const requestScreenshareFeed = opponentDevice.feeds.some( - (feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); + newCall.isPtt = this.isPtt; - try { - await newCall.placeCallWithCallFeeds( - this.client.localSfu === peerUserId - ? this.getLocalFeeds() // TODO: We should just setup the datachannel - : this.getLocalFeeds().map(feed => feed.clone()), // Safari can't send a MediaStream to multiple sources, so clone it - requestScreenshareFeed, - ); - } catch (e) { - logger.warn(`Failed to place call to ${member.userId}!`, e); - this.emit( - GroupCallEvent.Error, - new GroupCallError( - GroupCallErrorCode.PlaceCallFailed, - `Failed to place call to ${member.userId}.`, - ), - ); - return; - } + const requestScreenshareFeed = opponentDevice.feeds.some( + (feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); - if (this.dataChannelsEnabled) { - newCall.createDataChannel("datachannel", this.dataChannelOptions); - } + try { + await newCall.placeCallWithCallFeeds( + this.getLocalFeeds().map(feed => feed.clone()), + requestScreenshareFeed, + ); + } catch (e) { + logger.warn(`Failed to place call to ${member.userId}!`, e); + this.emit( + GroupCallEvent.Error, + new GroupCallError( + GroupCallErrorCode.PlaceCallFailed, + `Failed to place call to ${member.userId}.`, + ), + ); + return; + } - if (existingCall) { - this.replaceCall(existingCall, newCall, CallErrorCode.NewSession); - } else { - this.addCall(newCall); - } - } else if (this.client.localSfu && existingCall) { - // subscribe if we already had an existing call (otherwise - // we'll subscribe on the new call being set up) - existingCall.updateRemoteSDPStreamMetadata(this.getRemoteSDPStreamMetadataForCall()); - this.subscribeToSFU(existingCall, this.getRemoteFeedsFromState()); + if (this.dataChannelsEnabled) { + newCall.createDataChannel("datachannel", this.dataChannelOptions); + } + + if (existingCall) { + this.replaceCall(existingCall, newCall, CallErrorCode.NewSession); + } else { + this.addCall(newCall); } }; From b96534349a36e42dc0d5aa12d71342bc01988b18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 3 Aug 2022 09:45:42 +0200 Subject: [PATCH 029/137] Reduce diff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 43e65354e32..15cb9d429c7 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -992,8 +992,8 @@ export class GroupCall extends TypedEventEmitter< return; } - // Only initiate a call with a user who has a userId that is - // lexicographically less than your own. Otherwise, that user will + // Only initiate a call with a user who has a userId that is lexicographically + // less than your own. Otherwise, that user will // call you. if (member.userId < localUserId) { logger.log(`Waiting for ${member.userId} to send call invite.`); From ca7532e3185c6818a958c1caee6d7859f4eac9cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 3 Aug 2022 09:46:53 +0200 Subject: [PATCH 030/137] More diff reduction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 15cb9d429c7..20950a1ec3e 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -993,8 +993,7 @@ export class GroupCall extends TypedEventEmitter< } // Only initiate a call with a user who has a userId that is lexicographically - // less than your own. Otherwise, that user will - // call you. + // less than your own. Otherwise, that user will call you. if (member.userId < localUserId) { logger.log(`Waiting for ${member.userId} to send call invite.`); return; @@ -1013,8 +1012,6 @@ export class GroupCall extends TypedEventEmitter< const existingCall = this.getCallByUserId(member.userId); - // if we already have an existing call to the same session on the - // other side, then use it - it must have already called us first. if ( existingCall && existingCall.getOpponentSessionId() === opponentDevice.session_id From 9f7672c11e0266d7e407d8b837f65758cdbf25d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 3 Aug 2022 09:49:32 +0200 Subject: [PATCH 031/137] Remove a no-longer necessary hack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 20950a1ec3e..6fb39fcf58e 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -1233,16 +1233,12 @@ export class GroupCall extends TypedEventEmitter< throw new Error("Cannot dispose call without user id"); } - // FIXME: We need a queue for onMemberStateEvent as if two events are - // received in rapid succession, we get two calls - const callHandlers = this.callHandlers.get(opponentMemberId); - if (!callHandlers) return; const { onCallFeedsChanged, onCallStateChanged, onCallHangup, onCallReplaced, - } = callHandlers; + } = this.callHandlers.get(opponentMemberId); call.removeListener(CallEvent.FeedsChanged, onCallFeedsChanged); call.removeListener(CallEvent.State, onCallStateChanged); From 1190bd024567700a6b2238b627cad2784aeac2b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 3 Aug 2022 09:51:42 +0200 Subject: [PATCH 032/137] Remove unused prop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/client.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/client.ts b/src/client.ts index 647ac353d9c..85c56dd7cd7 100644 --- a/src/client.ts +++ b/src/client.ts @@ -365,11 +365,6 @@ export interface ICreateClientOpts { * The device ID for the local SFU to use for group calling, if any */ localSfuDeviceId?: string; - - /** - * False to disable E2EE for to-device calls. Default true. - */ - encryptedCalls?: boolean; } export interface IMatrixClientCreateOpts extends ICreateClientOpts { @@ -978,7 +973,6 @@ export class MatrixClient extends TypedEventEmitter Date: Wed, 3 Aug 2022 11:13:43 +0200 Subject: [PATCH 033/137] Introduce `getSFU()` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/client.ts | 19 ++++++++++++++++--- src/webrtc/call.ts | 6 +++--- src/webrtc/groupCall.ts | 22 +++++++++++----------- 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/src/client.ts b/src/client.ts index 85c56dd7cd7..192e8d9567e 100644 --- a/src/client.ts +++ b/src/client.ts @@ -794,6 +794,12 @@ interface ITimestampToEventResponse { event_id: string; origin_server_ts: string; } + +interface ISFUInfo { + user_id: string; + device_id: string; +} + /* eslint-enable camelcase */ // We're using this constant for methods overloading and inspect whether a variable @@ -971,8 +977,8 @@ export class MatrixClient extends TypedEventEmitter>(); - public localSfu: string; - public localSfuDeviceId: string; + private localSfu: string; + private localSfuDeviceId: string; private useE2eForGroupCall = true; constructor(opts: IMatrixClientCreateOpts) { @@ -1537,10 +1543,17 @@ export class MatrixClient extends TypedEventEmitter Date: Wed, 3 Aug 2022 12:59:17 +0200 Subject: [PATCH 034/137] Move `subscribeToSFU()` to `MatrixCall` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 51 +++++++++++++++++++++++++++++++++++- src/webrtc/groupCall.ts | 57 +++-------------------------------------- 2 files changed, 53 insertions(+), 55 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index e7d7ce6d46e..0919f3c999a 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -50,7 +50,7 @@ import { MatrixClient } from "../client"; import { ISendEventResponse } from "../@types/requests"; import { EventEmitterEvents, TypedEventEmitter } from "../models/typed-event-emitter"; import { DeviceInfo } from '../crypto/deviceinfo'; -import { GroupCallUnknownDeviceError, ISfuDataChannelMessage } from './groupCall'; +import { GroupCallUnknownDeviceError, IGroupCallMemberFeed, ISfuDataChannelMessage, ISfuTrackDesc } from './groupCall'; import { IScreensharingOpts } from "./mediaHandler"; // events: hangup, error(err), replaced(call), state(state, oldState) @@ -349,6 +349,7 @@ export class MatrixCall extends TypedEventEmitter = []; private usermediaSenders: Array = []; private screensharingSenders: Array = []; + private subscribedStreams: ISfuTrackDesc[] = []; private inviteOrAnswerSent = false; private waitForLocalAVStream: boolean; private successor: MatrixCall; @@ -1873,6 +1874,53 @@ export class MatrixCall extends TypedEventEmitter { + if (this.dataChannel.readyState === 'connecting') { + const p = new Promise(resolve => { + this.dataChannel.onopen = () => resolve(); + this.dataChannel.onclose = () => resolve(); + }); + await p; + } + return; + } + + public async subscribeToSFU(feeds: IGroupCallMemberFeed[]): Promise { + await this.waitForDatachannelToBeOpen(); + if (this.dataChannel.readyState !== "open") { + logger.warn("Can't sent to DC in state:", this.dataChannel.readyState); + return; + } + + // Only subscribe to streams we aren't already subscribed to + const streams: ISfuTrackDesc[] = feeds.filter((feed) => { + if (!feed.tracks) return false; // If we don't have info about tracks, the SFU won't have them either + return !this.subscribedStreams.find((stream) => stream.stream_id === feed.id); + }).map((feed) => ({ stream_id: feed.id })); + + if (streams.length === 0) { + logger.warn("Failed to find any new streams to subscribe to"); + return; + } else { + this.subscribedStreams.push(...streams); + logger.warn("Subscribing to:", streams); + } + + // TODO: rather than gutwrenching into our MatrixCall's peerConnection, + // should this be handled inside MatrixCall instead? + // + // FIXME: RPC reliability over DC + const msg: ISfuDataChannelMessage = { + "op": "select", + "conf_id": this.groupCallId, + "id": Date.now() + randomString(5), + "start": streams, + }; + + this.dataChannel.send(JSON.stringify(msg)); + logger.warn("Sent select message over DC", msg); + } + public updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { this.remoteSDPStreamMetadata = utils.recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); for (const feed of this.getRemoteFeeds()) { @@ -2462,6 +2510,7 @@ export class MatrixCall extends TypedEventEmitter | null = null; private memberStateExpirationTimers: Map> = new Map(); private resendMemberStateTimer: ReturnType | null = null; - private subscribedStreams: ISfuTrackDesc[] = []; constructor( private client: MatrixClient, @@ -491,7 +489,6 @@ export class GroupCall extends TypedEventEmitter< } this.client.getMediaHandler().stopAllStreams(); - this.subscribedStreams = []; if (this.state !== GroupCallState.Entered) { return; @@ -932,10 +929,9 @@ export class GroupCall extends TypedEventEmitter< if (this.client.getSFU().user_id) { const sfuCall = this.getCallByUserId(this.client.getSFU().user_id); if (!sfuCall) return; - // subscribe if we already had an existing call (otherwise - // we'll subscribe on the new call being set up) + sfuCall.updateRemoteSDPStreamMetadata(this.getRemoteSDPStreamMetadataForCall()); - this.subscribeToSFU(sfuCall, this.getRemoteFeedsFromState()); + sfuCall.subscribeToSFU(this.getRemoteFeedsFromState()); return; } @@ -1066,53 +1062,6 @@ export class GroupCall extends TypedEventEmitter< } }; - private async waitForDatachannelToBeOpen(call: MatrixCall): Promise { - if (call.dataChannel.readyState === 'connecting') { - const p = new Promise(resolve => { - call.dataChannel.onopen = () => resolve(); - call.dataChannel.onclose = () => resolve(); - }); - await p; - } - return; - } - - private async subscribeToSFU(call: MatrixCall, feeds: IGroupCallMemberFeed[]) { - await this.waitForDatachannelToBeOpen(call); - if (call.dataChannel.readyState !== "open") { - logger.warn("Can't sent to DC in state:", call.dataChannel.readyState); - return; - } - - // Only subscribe to streams we aren't already subscribed to - const streams: ISfuTrackDesc[] = feeds.filter((feed) => { - if (!feed.tracks) return false; // If we don't have info about tracks, the SFU won't have them either - return !this.subscribedStreams.find((stream) => stream.stream_id === feed.id); - }).map((feed) => ({ stream_id: feed.id })); - - if (streams.length === 0) { - logger.warn("Failed to find any new streams to subscribe to"); - return; - } else { - this.subscribedStreams.push(...streams); - logger.warn("Subscribing to:", streams); - } - - // TODO: rather than gutwrenching into our MatrixCall's peerConnection, - // should this be handled inside MatrixCall instead? - // - // FIXME: RPC reliability over DC - const msg: ISfuDataChannelMessage = { - "op": "select", - "conf_id": this.groupCallId, - "id": Date.now() + randomString(5), - "start": streams, - }; - - call.dataChannel.send(JSON.stringify(msg)); - logger.warn("Sent select message over DC", msg); - } - public getDeviceForMember(userId: string): IGroupCallMemberDevice { const memberStateEvent = this.getMemberStateEvents(userId); @@ -1312,7 +1261,7 @@ export class GroupCall extends TypedEventEmitter< // if we're calling an SFU, subscribe to its feeds if (call.getOpponentMember().userId === this.client.getSFU().user_id) { - this.subscribeToSFU(call, this.getRemoteFeedsFromState()); + call.subscribeToSFU(this.getRemoteFeedsFromState()); } } }; From cd2bcafd410b0a6ac8474347a0804506eb4d1c10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 3 Aug 2022 12:59:57 +0200 Subject: [PATCH 035/137] Remove stale comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 0919f3c999a..afbc1cd1a37 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1906,9 +1906,6 @@ export class MatrixCall extends TypedEventEmitter Date: Wed, 3 Aug 2022 13:05:58 +0200 Subject: [PATCH 036/137] Reduce diff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index f7edbecbbb4..983dd700d89 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -118,16 +118,16 @@ export interface IGroupCallMemberFeed { tracks: IGroupCallMemberTrack[]; } -export interface IGroupCallMemberDevice { +export interface IGroupCallRoomMemberDevice { "device_id": string; "session_id": string; "feeds": IGroupCallMemberFeed[]; } -export interface IGroupCallMemberCallState { +export interface IGroupRoomCallMemberCallState { "m.call_id": string; "m.foci"?: string[]; - "m.devices": IGroupCallMemberDevice[]; + "m.devices": IGroupCallRoomMemberDevice[]; } export interface ISfuTrackDesc { @@ -146,7 +146,7 @@ export interface ISfuDataChannelMessage { } export interface IGroupCallRoomMemberState { - "m.calls": IGroupCallMemberCallState[]; + "m.calls": IGroupRoomCallMemberCallState[]; "m.expires_ts": number; } @@ -860,12 +860,12 @@ export class GroupCall extends TypedEventEmitter< return await this.updateMemberCallState(undefined); } - private async updateMemberCallState(memberCallState?: IGroupCallMemberCallState): Promise { + private async updateMemberCallState(memberCallState?: IGroupRoomCallMemberCallState): Promise { const localUserId = this.client.getUserId(); const memberState = this.getMemberStateEvents(localUserId)?.getContent(); - let calls: IGroupCallMemberCallState[] = []; + let calls: IGroupRoomCallMemberCallState[] = []; // Sanitize existing member state event if (memberState && Array.isArray(memberState["m.calls"])) { @@ -1062,7 +1062,7 @@ export class GroupCall extends TypedEventEmitter< } }; - public getDeviceForMember(userId: string): IGroupCallMemberDevice { + public getDeviceForMember(userId: string): IGroupCallRoomMemberDevice { const memberStateEvent = this.getMemberStateEvents(userId); if (!memberStateEvent) { From 2f8207fab37d4be233fee7ff4dce36acfafe0425 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 3 Aug 2022 13:06:59 +0200 Subject: [PATCH 037/137] Naming fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 983dd700d89..506359e2d58 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -124,7 +124,7 @@ export interface IGroupCallRoomMemberDevice { "feeds": IGroupCallMemberFeed[]; } -export interface IGroupRoomCallMemberCallState { +export interface IGroupCallRoomMemberCallState { "m.call_id": string; "m.foci"?: string[]; "m.devices": IGroupCallRoomMemberDevice[]; @@ -146,7 +146,7 @@ export interface ISfuDataChannelMessage { } export interface IGroupCallRoomMemberState { - "m.calls": IGroupRoomCallMemberCallState[]; + "m.calls": IGroupCallRoomMemberCallState[]; "m.expires_ts": number; } @@ -860,12 +860,12 @@ export class GroupCall extends TypedEventEmitter< return await this.updateMemberCallState(undefined); } - private async updateMemberCallState(memberCallState?: IGroupRoomCallMemberCallState): Promise { + private async updateMemberCallState(memberCallState?: IGroupCallRoomMemberCallState): Promise { const localUserId = this.client.getUserId(); const memberState = this.getMemberStateEvents(localUserId)?.getContent(); - let calls: IGroupRoomCallMemberCallState[] = []; + let calls: IGroupCallRoomMemberCallState[] = []; // Sanitize existing member state event if (memberState && Array.isArray(memberState["m.calls"])) { From a884fecbc990f5137b8032d81268e1e203d32c3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 3 Aug 2022 13:09:09 +0200 Subject: [PATCH 038/137] Further reduce diff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 9 +++++++-- src/webrtc/groupCall.ts | 8 ++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index afbc1cd1a37..13d09588d7b 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -50,8 +50,13 @@ import { MatrixClient } from "../client"; import { ISendEventResponse } from "../@types/requests"; import { EventEmitterEvents, TypedEventEmitter } from "../models/typed-event-emitter"; import { DeviceInfo } from '../crypto/deviceinfo'; -import { GroupCallUnknownDeviceError, IGroupCallMemberFeed, ISfuDataChannelMessage, ISfuTrackDesc } from './groupCall'; import { IScreensharingOpts } from "./mediaHandler"; +import { + GroupCallUnknownDeviceError, + IGroupCallRoomMemberFeed, + ISfuDataChannelMessage, + ISfuTrackDesc, +} from "./groupCall"; // events: hangup, error(err), replaced(call), state(state, oldState) @@ -1885,7 +1890,7 @@ export class MatrixCall extends TypedEventEmitter { + public async subscribeToSFU(feeds: IGroupCallRoomMemberFeed[]): Promise { await this.waitForDatachannelToBeOpen(); if (this.dataChannel.readyState !== "open") { logger.warn("Can't sent to DC in state:", this.dataChannel.readyState); diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 506359e2d58..b31266ee901 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -112,7 +112,7 @@ export interface IGroupCallMemberTrack { settings: MediaTrackSettings; } -export interface IGroupCallMemberFeed { +export interface IGroupCallRoomMemberFeed { id: string; purpose: SDPStreamMetadataPurpose; tracks: IGroupCallMemberTrack[]; @@ -121,7 +121,7 @@ export interface IGroupCallMemberFeed { export interface IGroupCallRoomMemberDevice { "device_id": string; "session_id": string; - "feeds": IGroupCallMemberFeed[]; + "feeds": IGroupCallRoomMemberFeed[]; } export interface IGroupCallRoomMemberCallState { @@ -892,7 +892,7 @@ export class GroupCall extends TypedEventEmitter< return this.client.sendStateEvent(this.room.roomId, EventType.GroupCallMemberPrefix, content, localUserId); } - private getRemoteFeedsFromState(): IGroupCallMemberFeed[] { + private getRemoteFeedsFromState(): IGroupCallRoomMemberFeed[] { return this.getMemberStateEvents()?.reduce((feeds, event) => { if (event.getSender() === this.client.getUserId()) return feeds; // Ignore local const newFeeds = event.getContent()?.["m.calls"]?.[0]?.["m.devices"]?.[0]?.feeds; @@ -906,7 +906,7 @@ export class GroupCall extends TypedEventEmitter< if (event.getSender() === this.client.getUserId()) return metaAcc; // Ignore local const feeds = event.getContent()?.["m.calls"]?.[0]?.["m.devices"]?.[0]?.feeds; if (!feeds) return metaAcc; - const metadata = feeds.reduce((feedAcc: SDPStreamMetadata, feed: IGroupCallMemberFeed) => { + const metadata = feeds.reduce((feedAcc: SDPStreamMetadata, feed: IGroupCallRoomMemberFeed) => { if (!feed?.id) return feedAcc; return recursivelyAssign(feedAcc, { [feed.id]: { From 380578394c07079959340dcdd9180a3d0d8ae81e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 3 Aug 2022 13:54:15 +0200 Subject: [PATCH 039/137] Extract `sendSFUDataChannelMessage()` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 47 +++++++++++++++++++----------------- src/webrtc/callEventTypes.ts | 30 +++++++++++++++++++++++ src/webrtc/groupCall.ts | 17 +------------ 3 files changed, 56 insertions(+), 38 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 13d09588d7b..4df20c44620 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -44,6 +44,11 @@ import { MCallCandidates, MCallBase, MCallHangupReject, + ISfuTrackDesc, + ISfuBaseDataChannelMessage, + ISfuSelectDataChannelMessage, + ISfuAnswerDataChannelMessage, + ISfuPublishDataChannelMessage, } from './callEventTypes'; import { CallFeed } from './callFeed'; import { MatrixClient } from "../client"; @@ -54,8 +59,6 @@ import { IScreensharingOpts } from "./mediaHandler"; import { GroupCallUnknownDeviceError, IGroupCallRoomMemberFeed, - ISfuDataChannelMessage, - ISfuTrackDesc, } from "./groupCall"; // events: hangup, error(err), replaced(call), state(state, oldState) @@ -1827,7 +1830,7 @@ export class MatrixCall extends TypedEventEmitter => { // TODO: play nice with application layer DC listeners - let json: ISfuDataChannelMessage; + let json: ISfuBaseDataChannelMessage; try { json = JSON.parse(event.data); } catch (e) { @@ -1853,15 +1856,10 @@ export class MatrixCall extends TypedEventEmitter): void { + const realContent: ISfuBaseDataChannelMessage = Object.assign(content, { + id: randomString(24), + conf_id: this.groupCallId, + }); + + // FIXME: RPC reliability over DC + this.dataChannel.send(JSON.stringify(realContent)); + logger.warn(`Sent ${realContent.op} over DC:`, realContent); + } + private queueCandidate(content: RTCIceCandidate): void { // We partially de-trickle candidates by waiting for `delay` before sending them // amalgamated, in order to avoid sending too many m.call.candidates events and hitting diff --git a/src/webrtc/callEventTypes.ts b/src/webrtc/callEventTypes.ts index aae2f54880e..d6c2197e8ab 100644 --- a/src/webrtc/callEventTypes.ts +++ b/src/webrtc/callEventTypes.ts @@ -90,4 +90,34 @@ export interface MCallHangupReject extends MCallBase { reason?: CallErrorCode; } +export interface ISfuTrackDesc { + stream_id: string; + track_id?: string; +} + +export interface ISfuBaseDataChannelMessage { + op: string; + id: string; + conf_id: string; + sdp?: string; + message?: string; + start?: ISfuTrackDesc[]; + stop?: ISfuTrackDesc[]; +} + +export interface ISfuSelectDataChannelMessage extends ISfuBaseDataChannelMessage { + op: "select"; + start: ISfuTrackDesc[]; +} + +export interface ISfuAnswerDataChannelMessage extends ISfuBaseDataChannelMessage { + op: "answer"; + sdp: string; +} + +export interface ISfuPublishDataChannelMessage extends ISfuBaseDataChannelMessage { + op: "publish"; + sdp: string; +} + /* eslint-enable camelcase */ diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index b31266ee901..c1153fb1df9 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -14,7 +14,7 @@ import { RoomMember } from "../models/room-member"; import { Room } from "../models/room"; import { logger } from "../logger"; import { ReEmitter } from "../ReEmitter"; -import { SDPStreamMetadata, SDPStreamMetadataPurpose } from "./callEventTypes"; +import { ISfuTrackDesc, SDPStreamMetadata, SDPStreamMetadataPurpose } from "./callEventTypes"; import { createNewMatrixCall } from "./call"; import { ISendEventResponse } from "../@types/requests"; import { MatrixEvent } from "../models/event"; @@ -130,21 +130,6 @@ export interface IGroupCallRoomMemberCallState { "m.devices": IGroupCallRoomMemberDevice[]; } -export interface ISfuTrackDesc { - "stream_id": string; - "track_id"?: string; -} - -export interface ISfuDataChannelMessage { - "op": string; - "id": string; - "conf_id"?: string; - "sdp"?: string; - "message"?: string; - "start"?: ISfuTrackDesc[]; - "stop"?: ISfuTrackDesc[]; -} - export interface IGroupCallRoomMemberState { "m.calls": IGroupCallRoomMemberCallState[]; "m.expires_ts": number; From 1db93c36266d9ec4bfe08f0f01a9d77c3e78b567 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 3 Aug 2022 14:31:21 +0200 Subject: [PATCH 040/137] Select by both `streamId` and `trackId` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 21 ++++++++++----------- src/webrtc/callEventTypes.ts | 2 +- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 4df20c44620..d00bdb96705 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -357,7 +357,7 @@ export class MatrixCall extends TypedEventEmitter = []; private usermediaSenders: Array = []; private screensharingSenders: Array = []; - private subscribedStreams: ISfuTrackDesc[] = []; + private subscribedTracks: ISfuTrackDesc[] = []; private inviteOrAnswerSent = false; private waitForLocalAVStream: boolean; private successor: MatrixCall; @@ -1895,23 +1895,22 @@ export class MatrixCall extends TypedEventEmitter { - if (!feed.tracks) return false; // If we don't have info about tracks, the SFU won't have them either - return !this.subscribedStreams.find((stream) => stream.stream_id === feed.id); - }).map((feed) => ({ stream_id: feed.id })); + const tracks: ISfuTrackDesc[] = feeds + .filter((feed) => feed.tracks) // Skip trackless feeds + .reduce((tracks, f) => [...tracks, ...f.tracks.map((t) => ({ stream_id: f.id, track_id: t.id }))], []) // Get array of tracks from feeds + .filter((track) => !this.subscribedTracks.find((subscribed) => utils.deepCompare(track, subscribed))); // Filter out already subscribed tracks - if (streams.length === 0) { + if (tracks.length === 0) { logger.warn("Failed to find any new streams to subscribe to"); return; } else { - this.subscribedStreams.push(...streams); - logger.warn("Subscribing to:", streams); + this.subscribedTracks.push(...tracks); + logger.warn("Subscribing to:", tracks); } this.sendSFUDataChannelMessage({ op: "select", - start: streams, + start: tracks, } as ISfuSelectDataChannelMessage); } @@ -2515,7 +2514,7 @@ export class MatrixCall extends TypedEventEmitter Date: Wed, 3 Aug 2022 16:07:35 +0200 Subject: [PATCH 041/137] Remove unused interface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/callEventTypes.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/webrtc/callEventTypes.ts b/src/webrtc/callEventTypes.ts index eadfc4ab5c3..ba758bb48e5 100644 --- a/src/webrtc/callEventTypes.ts +++ b/src/webrtc/callEventTypes.ts @@ -115,9 +115,4 @@ export interface ISfuAnswerDataChannelMessage extends ISfuBaseDataChannelMessage sdp: string; } -export interface ISfuPublishDataChannelMessage extends ISfuBaseDataChannelMessage { - op: "publish"; - sdp: string; -} - /* eslint-enable camelcase */ From de26e64f7d5d3517484fb449499a7b5e08a62320 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 3 Aug 2022 16:09:55 +0200 Subject: [PATCH 042/137] Fix unused import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index d00bdb96705..0e4f44aff01 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -48,7 +48,6 @@ import { ISfuBaseDataChannelMessage, ISfuSelectDataChannelMessage, ISfuAnswerDataChannelMessage, - ISfuPublishDataChannelMessage, } from './callEventTypes'; import { CallFeed } from './callFeed'; import { MatrixClient } from "../client"; From ca26e665937d896be83088b7a78c073a4bd61c8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 4 Aug 2022 14:53:40 +0200 Subject: [PATCH 043/137] Make sure to not `undefined` tracks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index c1153fb1df9..3fb543ba418 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -807,10 +807,10 @@ export class GroupCall extends TypedEventEmitter< // TODO: correctly track which rtpSenders are associated with which feed // rather than assuming that all our senders are from this feed. tracks: this.calls[0] - ? this.calls[0].peerConn.getTransceivers().map(transceiver => ({ - "id": getTrackDesc(this.calls[0].peerConn.localDescription?.sdp, transceiver.mid)?.track_id, - "kind": transceiver.sender.track.kind, - "settings": defloat(transceiver.sender.track.getSettings()), + ? this.calls[0].peerConn.getTransceivers().filter((t) => t.sender.track).map(t => ({ + "id": getTrackDesc(this.calls[0].peerConn.localDescription?.sdp, t.mid)?.track_id, + "kind": t.sender.track.kind, + "settings": defloat(t.sender.track.getSettings()), })) : undefined, })); From 785db350ff0024cf2a11924f212e80eb6f2c588f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 4 Aug 2022 17:09:58 +0200 Subject: [PATCH 044/137] Simplify what we're sending in member state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 70 ++--------------------------------------- 1 file changed, 3 insertions(+), 67 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 3fb543ba418..862e625bada 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -14,7 +14,7 @@ import { RoomMember } from "../models/room-member"; import { Room } from "../models/room"; import { logger } from "../logger"; import { ReEmitter } from "../ReEmitter"; -import { ISfuTrackDesc, SDPStreamMetadata, SDPStreamMetadataPurpose } from "./callEventTypes"; +import { SDPStreamMetadata, SDPStreamMetadataPurpose } from "./callEventTypes"; import { createNewMatrixCall } from "./call"; import { ISendEventResponse } from "../@types/requests"; import { MatrixEvent } from "../models/event"; @@ -107,9 +107,6 @@ export interface IGroupCallDataChannelOptions { export interface IGroupCallMemberTrack { id: string; - kind: string; // TODO: use an enum - // label: string; // removing as too privacy invasive - settings: MediaTrackSettings; } export interface IGroupCallRoomMemberFeed { @@ -164,65 +161,6 @@ function getCallUserId(call: MatrixCall): string | null { return call.getOpponentMember()?.userId || call.invitee || null; } -function defloat(json: Object): Object { - for (const key of Object.keys(json)) { - if (isFloat(json[key])) { - json[key] = "" + json[key]; - } - } - return json; -} - -function isFloat(value) { - return ( - typeof value === 'number' && - !Number.isNaN(value) && - !Number.isInteger(value) - ); -} - -interface IMediaBlock { - mid?: string; - trackDesc?: ISfuTrackDesc; -} - -function getTrackDesc(sdp: string, mid: string): ISfuTrackDesc | undefined { - // sdp mangling to grab the a=msid: line out of SDP for a given mid - if (!sdp) return; - - const mediaByMids: Map = new Map(); - let mediaBlock: IMediaBlock = {}; - let matches; - for (const line of sdp.split(/\r?\n/)) { - if (line.match(/^m=/)) { - if (mediaBlock.mid !== undefined) { - mediaByMids.set(mediaBlock.mid, mediaBlock); - } - mediaBlock = {}; - } - matches = line.match(/^a=mid:(.*?)$/); - if (matches) { - mediaBlock.mid = matches[1]; - } - matches = line.match(/^a=msid:(.*?) (.*?)$/); - if (matches) { - mediaBlock.trackDesc = { - stream_id: matches[1], - track_id: matches[2], - }; - } - } - if (mediaBlock.mid) { - mediaByMids.set(mediaBlock.mid, mediaBlock); - } - - if (mediaByMids.get(mid)) { - return mediaByMids.get(mid).trackDesc; - } else { - return; - } -} - export class GroupCall extends TypedEventEmitter< GroupCallEvent | CallEvent, GroupCallEventHandlerMap & CallEventHandlerMap @@ -807,10 +745,8 @@ export class GroupCall extends TypedEventEmitter< // TODO: correctly track which rtpSenders are associated with which feed // rather than assuming that all our senders are from this feed. tracks: this.calls[0] - ? this.calls[0].peerConn.getTransceivers().filter((t) => t.sender.track).map(t => ({ - "id": getTrackDesc(this.calls[0].peerConn.localDescription?.sdp, t.mid)?.track_id, - "kind": t.sender.track.kind, - "settings": defloat(t.sender.track.getSettings()), + ? this.calls[0].peerConn.getSenders().filter((s) => s.track).map(s => ({ + "id": s.track.id, })) : undefined, })); From 31d3ba39298cf4ed7e5195d74be0ac9cbccd6910 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 4 Aug 2022 17:27:36 +0200 Subject: [PATCH 045/137] Reduce diff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 862e625bada..4495d6eb3b6 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -734,30 +734,28 @@ export class GroupCall extends TypedEventEmitter< } private async sendMemberStateEvent(): Promise { - const feeds = this.getLocalFeeds().map((feed) => ({ - purpose: feed.purpose, - id: feed.stream.id, - - // we have to advertise the actual tracks we're sending to the SFU from the PC - // we can't use the feeds' mediaStream IDs, as they are local rather than the copy - // sent over WebRTC - // - // TODO: correctly track which rtpSenders are associated with which feed - // rather than assuming that all our senders are from this feed. - tracks: this.calls[0] - ? this.calls[0].peerConn.getSenders().filter((s) => s.track).map(s => ({ - "id": s.track.id, - })) - : undefined, - })); - const send = () => this.updateMemberCallState({ "m.call_id": this.groupCallId, "m.devices": [ { "device_id": this.client.getDeviceId(), "session_id": this.client.getSessionId(), - "feeds": feeds, + "feeds": this.getLocalFeeds().map((feed) => ({ + purpose: feed.purpose, + id: feed.stream.id, + + // we have to advertise the actual tracks we're sending to the SFU from the PC + // we can't use the feeds' mediaStream IDs, as they are local rather than the copy + // sent over WebRTC + // + // TODO: correctly track which rtpSenders are associated with which feed + // rather than assuming that all our senders are from this feed. + tracks: this.calls[0] + ? this.calls[0].peerConn.getSenders().filter((s) => s.track).map(s => ({ + "id": s.track.id, + })) + : undefined, + })), // TODO: Add data channels }, ], From 24c7f09951e4cbfadf01ac71cda33c30a5aa690d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 6 Aug 2022 11:51:54 +0200 Subject: [PATCH 046/137] Use `transceivers` instead of `senders` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit They tend to be more accurate when it comes to `trackId`s (at least most of the time) They can give us `mid`s which might be the thing we switch to for identification of streams and tracks Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 79 ++++++++++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 35 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 0e4f44aff01..b6c9295882d 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -354,8 +354,8 @@ export class MatrixCall extends TypedEventEmitter = []; - private usermediaSenders: Array = []; - private screensharingSenders: Array = []; + private usermediaTransceivers: Array = []; + private screensharingTransceivers: Array = []; private subscribedTracks: ISfuTrackDesc[] = []; private inviteOrAnswerSent = false; private waitForLocalAVStream: boolean; @@ -737,10 +737,10 @@ export class MatrixCall extends TypedEventEmitter { return track.kind === "video"; }); - const sender = this.usermediaSenders.find((sender) => { - return sender.track?.kind === "video"; + const transceiver = this.usermediaTransceivers.find((transceiver) => { + return transceiver.sender.track?.kind === "video"; }); - sender.replaceTrack(track); + transceiver.sender.replaceTrack(track); this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare, false); @@ -1200,10 +1207,10 @@ export class MatrixCall extends TypedEventEmitter { return track.kind === "video"; }); - const sender = this.usermediaSenders.find((sender) => { - return sender.track?.kind === "video"; + const transceiver = this.usermediaTransceivers.find((transceiver) => { + return transceiver.sender.track?.kind === "video"; }); - sender.replaceTrack(track); + transceiver.sender.replaceTrack(track); this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream); this.deleteFeedByStream(this.localScreensharingStream); @@ -1236,14 +1243,14 @@ export class MatrixCall extends TypedEventEmitter { - return sender.track?.kind === track.kind; + const oldTransceiver = this.usermediaTransceivers.find((transceiver) => { + return transceiver.sender.track?.kind === track.kind; }); - let newSender: RTCRtpSender; + let newTransceiver: RTCRtpTransceiver; try { logger.info( @@ -1255,8 +1262,8 @@ export class MatrixCall extends TypedEventEmitter Date: Sat, 6 Aug 2022 11:53:59 +0200 Subject: [PATCH 047/137] `newCall` -> `sfuCall` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 4495d6eb3b6..eabdc9dbe19 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -363,7 +363,7 @@ export class GroupCall extends TypedEventEmitter< "feeds": [], }; - const newCall = createNewMatrixCall( + const sfuCall = createNewMatrixCall( this.client, this.room.roomId, { @@ -377,11 +377,11 @@ export class GroupCall extends TypedEventEmitter< }, ); - newCall.isPtt = this.isPtt; + sfuCall.isPtt = this.isPtt; try { - await newCall.placeCallWithCallFeeds(this.getLocalFeeds()); // TODO: We should just setup the datachannel - newCall.createDataChannel("datachannel", this.dataChannelOptions); + await sfuCall.placeCallWithCallFeeds(this.getLocalFeeds()); // TODO: We should just setup the datachannel + sfuCall.createDataChannel("datachannel", this.dataChannelOptions); } catch (e) { logger.warn(`Failed to place call to ${this.client.getSFU().user_id}!`, e); this.emit( @@ -394,7 +394,7 @@ export class GroupCall extends TypedEventEmitter< return; } - this.addCall(newCall); + this.addCall(sfuCall); } } From 205920eddac2c3eedfbb9677c2782f72647582f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 6 Aug 2022 12:02:47 +0200 Subject: [PATCH 048/137] Add `getGroupCallRoomMemberFeeds()` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 22 ++++++++++++++++++++++ src/webrtc/groupCall.ts | 18 ++---------------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index b6c9295882d..94f4e0ba4b5 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -599,6 +599,28 @@ export class MatrixCall extends TypedEventEmitter ({ + id: transceiver.sender.track.id, + })), + }); + } + return feeds; + } + /** * Returns true if there are no incoming feeds, * otherwise returns false diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index eabdc9dbe19..79aa7784df6 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -740,22 +740,8 @@ export class GroupCall extends TypedEventEmitter< { "device_id": this.client.getDeviceId(), "session_id": this.client.getSessionId(), - "feeds": this.getLocalFeeds().map((feed) => ({ - purpose: feed.purpose, - id: feed.stream.id, - - // we have to advertise the actual tracks we're sending to the SFU from the PC - // we can't use the feeds' mediaStream IDs, as they are local rather than the copy - // sent over WebRTC - // - // TODO: correctly track which rtpSenders are associated with which feed - // rather than assuming that all our senders are from this feed. - tracks: this.calls[0] - ? this.calls[0].peerConn.getSenders().filter((s) => s.track).map(s => ({ - "id": s.track.id, - })) - : undefined, - })), + // We assume that all calls are sending the same feeds + "feeds": this.calls[0]?.getGroupCallRoomMemberFeeds(), // TODO: Add data channels }, ], From cca9eda7ad958396a2d96d30c1d520d71b97533b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 6 Aug 2022 12:16:42 +0200 Subject: [PATCH 049/137] Call `sendMemberStateEvent()` more intelligently MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 79aa7784df6..61c54184039 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -642,8 +642,6 @@ export class GroupCall extends TypedEventEmitter< this.localScreenshareFeed.clone(), ))); - await this.sendMemberStateEvent(); - return true; } catch (error) { logger.error("Enabling screensharing error", error); @@ -658,7 +656,6 @@ export class GroupCall extends TypedEventEmitter< this.removeScreenshareFeed(this.localScreenshareFeed); this.localScreenshareFeed = undefined; this.localDesktopCapturerSourceId = undefined; - await this.sendMemberStateEvent(); this.emit(GroupCallEvent.LocalScreenshareStateChanged, false, undefined, undefined); return false; } @@ -1123,6 +1120,8 @@ export class GroupCall extends TypedEventEmitter< } private onCallFeedsChanged = (call: MatrixCall) => { + this.sendMemberStateEvent(); + // Find removed feeds [...this.userMediaFeeds, ...this.screenshareFeeds].filter((gf) => gf.isDisposed()).forEach((feed) => { if (feed.purpose === SDPStreamMetadataPurpose.Usermedia) this.removeUserMediaFeed(feed); @@ -1160,10 +1159,6 @@ export class GroupCall extends TypedEventEmitter< if (state === CallState.Connected) { this.retryCallCounts.delete(getCallUserId(call)); - // Now we know what our track IDs are, we can publish them so others - // can subscribe to us... - this.sendMemberStateEvent(); - // if we're calling an SFU, subscribe to its feeds if (call.getOpponentMember().userId === this.client.getSFU().user_id) { call.subscribeToSFU(this.getRemoteFeedsFromState()); From 98156e06db8a811ac8e3ada2a1b5d8e0fc0e5462 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 6 Aug 2022 12:31:49 +0200 Subject: [PATCH 050/137] Add screen-sharing support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 50 +++++++++++++++++++++++++++++++----- src/webrtc/callEventTypes.ts | 20 ++++++++++++--- src/webrtc/groupCall.ts | 10 ++++++-- 3 files changed, 68 insertions(+), 12 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 94f4e0ba4b5..48641075bec 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -48,6 +48,9 @@ import { ISfuBaseDataChannelMessage, ISfuSelectDataChannelMessage, ISfuAnswerDataChannelMessage, + ISfuPublishDataChannelMessage, + ISfuUnpublishDataChannelMessage, + ISfuOfferDataChannelMessage, } from './callEventTypes'; import { CallFeed } from './callFeed'; import { MatrixClient } from "../client"; @@ -789,7 +792,21 @@ export class MatrixCall extends TypedEventEmitter { + this.peerConn.setLocalDescription(offer); + + this.sendSFUDataChannelMessage({ + op: "publish", + sdp: offer.sdp, + } as ISfuPublishDataChannelMessage); + this.emit(CallEvent.FeedsChanged, this.feeds); + }); + } else { + this.emit(CallEvent.FeedsChanged, this.feeds); + } } /** @@ -818,6 +835,18 @@ export class MatrixCall extends TypedEventEmitter { + this.peerConn.setLocalDescription(offer); + + this.sendSFUDataChannelMessage({ + op: "unpublish", + sdp: offer.sdp, + stop: tracksToUnpublish, + } as ISfuUnpublishDataChannelMessage); + }); + } } private deleteAllFeeds(): void { @@ -1876,11 +1905,12 @@ export class MatrixCall extends TypedEventEmitter => { + // Other than the initial offer, we handle negotiation manually when calling with an SFU + if (this.client.getSFU() && this.state !== CallState.CreateOffer) return; + logger.info(`Call ${this.callId} Negotiation is needed!`); if (this.state !== CallState.CreateOffer && this.opponentVersion === 0) { diff --git a/src/webrtc/callEventTypes.ts b/src/webrtc/callEventTypes.ts index ba758bb48e5..a49591d3f25 100644 --- a/src/webrtc/callEventTypes.ts +++ b/src/webrtc/callEventTypes.ts @@ -99,10 +99,6 @@ export interface ISfuBaseDataChannelMessage { op: string; id: string; conf_id: string; - sdp?: string; - message?: string; - start?: ISfuTrackDesc[]; - stop?: ISfuTrackDesc[]; } export interface ISfuSelectDataChannelMessage extends ISfuBaseDataChannelMessage { @@ -110,9 +106,25 @@ export interface ISfuSelectDataChannelMessage extends ISfuBaseDataChannelMessage start: ISfuTrackDesc[]; } +export interface ISfuOfferDataChannelMessage extends ISfuBaseDataChannelMessage { + op: "offer"; + sdp: string; +} + export interface ISfuAnswerDataChannelMessage extends ISfuBaseDataChannelMessage { op: "answer"; sdp: string; } +export interface ISfuPublishDataChannelMessage extends ISfuBaseDataChannelMessage { + op: "publish"; + sdp: string; +} + +export interface ISfuUnpublishDataChannelMessage extends ISfuBaseDataChannelMessage { + op: "unpublish"; + sdp: string; + stop: ISfuTrackDesc[]; +} + /* eslint-enable camelcase */ diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 61c54184039..1d5ed8b4e29 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -639,7 +639,9 @@ export class GroupCall extends TypedEventEmitter< // TODO: handle errors await Promise.all(this.calls.map(call => call.pushLocalFeed( - this.localScreenshareFeed.clone(), + this.client.getSFU().user_id + ? this.localScreenshareFeed + : this.localScreenshareFeed.clone(), ))); return true; @@ -653,7 +655,11 @@ export class GroupCall extends TypedEventEmitter< } else { await Promise.all(this.calls.map(call => call.removeLocalFeed(call.localScreensharingFeed))); this.client.getMediaHandler().stopScreensharingStream(this.localScreenshareFeed.stream); - this.removeScreenshareFeed(this.localScreenshareFeed); + // We have to remove the feed manually as MatrixCall has its clone, + // so it won't be removed automatically + if (!this.client.getSFU().user_id) { + this.removeScreenshareFeed(this.localScreenshareFeed); + } this.localScreenshareFeed = undefined; this.localDesktopCapturerSourceId = undefined; this.emit(GroupCallEvent.LocalScreenshareStateChanged, false, undefined, undefined); From 2dc7581e9a7f486df96112cabdc061ce4bca36b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 6 Aug 2022 15:32:26 +0200 Subject: [PATCH 051/137] Handle `ISfuInfo` better MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/client.ts | 8 +++++--- src/webrtc/call.ts | 23 ++++++++++++++-------- src/webrtc/groupCall.ts | 42 +++++++++++++++++------------------------ 3 files changed, 37 insertions(+), 36 deletions(-) diff --git a/src/client.ts b/src/client.ts index 192e8d9567e..be6110b7dfd 100644 --- a/src/client.ts +++ b/src/client.ts @@ -795,7 +795,7 @@ interface ITimestampToEventResponse { origin_server_ts: string; } -interface ISFUInfo { +export interface ISfuInfo { user_id: string; device_id: string; } @@ -1543,11 +1543,13 @@ export class MatrixClient extends TypedEventEmitter { this.peerConn.setLocalDescription(offer); @@ -836,7 +839,7 @@ export class MatrixCall extends TypedEventEmitter { this.peerConn.setLocalDescription(offer); @@ -916,7 +919,7 @@ export class MatrixCall extends TypedEventEmitter => { // Other than the initial offer, we handle negotiation manually when calling with an SFU - if (this.client.getSFU() && this.state !== CallState.CreateOffer) return; + if (this.isSfu && this.state !== CallState.CreateOffer) return; logger.info(`Call ${this.callId} Negotiation is needed!`); @@ -2796,9 +2799,12 @@ export class MatrixCall extends TypedEventEmitter { @@ -2904,6 +2910,7 @@ export function createNewMatrixCall(client: any, roomId: string, options?: CallO opponentSessionId: options?.opponentSessionId, groupCallId: options?.groupCallId, initialRemoteSDPStreamMetadata: options?.initialRemoteSDPStreamMetadata, + isSfu: options.isSfu, }; const call = new MatrixCall(opts); diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 1d5ed8b4e29..cc36a29ed16 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -1,6 +1,6 @@ import { TypedEventEmitter } from "../models/typed-event-emitter"; import { CallFeed, SPEAKING_THRESHOLD } from "./callFeed"; -import { MatrixClient } from "../client"; +import { ISfuInfo, MatrixClient } from "../client"; import { CallErrorCode, CallEvent, @@ -190,6 +190,7 @@ export class GroupCall extends TypedEventEmitter< private transmitTimer: ReturnType | null = null; private memberStateExpirationTimers: Map> = new Map(); private resendMemberStateTimer: ReturnType | null = null; + private sfu: ISfuInfo | null = null; constructor( private client: MatrixClient, @@ -205,11 +206,6 @@ export class GroupCall extends TypedEventEmitter< this.reEmitter = new ReEmitter(this); this.groupCallId = groupCallId || genCallID(); - if (this.client.getSFU().user_id) { - // we have to use DCs to talk to the SFU - this.dataChannelsEnabled = true; - } - for (const stateEvent of this.getMemberStateEvents()) { this.onMemberStateChanged(stateEvent); } @@ -351,9 +347,10 @@ export class GroupCall extends TypedEventEmitter< this.onActiveSpeakerLoop(); - if (this.client.getSFU().user_id) { + this.sfu = this.client.getSfu(); + if (this.sfu) { const opponentDevice = { - "device_id": this.client.getSFU().device_id, + "device_id": this.sfu.device_id, // XXX: the SFU might need to specify a session_id so that if it // restarts and starts sending invites to us, we know that it's // forgotten who we were? But then we need a way to communicate @@ -367,13 +364,12 @@ export class GroupCall extends TypedEventEmitter< this.client, this.room.roomId, { - invitee: this.client.getSFU().user_id, + invitee: this.sfu.user_id, opponentDeviceId: opponentDevice.device_id, opponentSessionId: opponentDevice.session_id, groupCallId: this.groupCallId, - initialRemoteSDPStreamMetadata: this.client.getSFU().user_id - ? this.getRemoteSDPStreamMetadataForCall() - : undefined, + initialRemoteSDPStreamMetadata: this.getRemoteSDPStreamMetadataForCall(), + isSfu: true, }, ); @@ -383,12 +379,12 @@ export class GroupCall extends TypedEventEmitter< await sfuCall.placeCallWithCallFeeds(this.getLocalFeeds()); // TODO: We should just setup the datachannel sfuCall.createDataChannel("datachannel", this.dataChannelOptions); } catch (e) { - logger.warn(`Failed to place call to ${this.client.getSFU().user_id}!`, e); + logger.warn(`Failed to place call to ${this.sfu.user_id}!`, e); this.emit( GroupCallEvent.Error, new GroupCallError( GroupCallErrorCode.PlaceCallFailed, - `Failed to place call to ${this.client.getSFU().user_id}.`, + `Failed to place call to ${this.sfu.user_id}.`, ), ); return; @@ -638,10 +634,9 @@ export class GroupCall extends TypedEventEmitter< ); // TODO: handle errors - await Promise.all(this.calls.map(call => call.pushLocalFeed( - this.client.getSFU().user_id - ? this.localScreenshareFeed - : this.localScreenshareFeed.clone(), + await Promise.all(this.calls.map(call => call.pushLocalFeed(call.isSfu + ? this.localScreenshareFeed + : this.localScreenshareFeed.clone(), ))); return true; @@ -657,7 +652,7 @@ export class GroupCall extends TypedEventEmitter< this.client.getMediaHandler().stopScreensharingStream(this.localScreenshareFeed.stream); // We have to remove the feed manually as MatrixCall has its clone, // so it won't be removed automatically - if (!this.client.getSFU().user_id) { + if (!this.sfu) { this.removeScreenshareFeed(this.localScreenshareFeed); } this.localScreenshareFeed = undefined; @@ -834,8 +829,8 @@ export class GroupCall extends TypedEventEmitter< const member = this.room.getMember(event.getStateKey()); if (!member) return; - if (this.client.getSFU().user_id) { - const sfuCall = this.getCallByUserId(this.client.getSFU().user_id); + if (this.sfu) { + const sfuCall = this.getCallByUserId(this.sfu.user_id); if (!sfuCall) return; sfuCall.updateRemoteSDPStreamMetadata(this.getRemoteSDPStreamMetadataForCall()); @@ -931,9 +926,6 @@ export class GroupCall extends TypedEventEmitter< opponentDeviceId: opponentDevice.device_id, opponentSessionId: opponentDevice.session_id, groupCallId: this.groupCallId, - initialRemoteSDPStreamMetadata: this.client.getSFU().user_id - ? this.getRemoteSDPStreamMetadataForCall() - : undefined, }, ); @@ -1166,7 +1158,7 @@ export class GroupCall extends TypedEventEmitter< this.retryCallCounts.delete(getCallUserId(call)); // if we're calling an SFU, subscribe to its feeds - if (call.getOpponentMember().userId === this.client.getSFU().user_id) { + if (call.getOpponentMember().userId === this.sfu.user_id) { call.subscribeToSFU(this.getRemoteFeedsFromState()); } } From 4d901bf9f00aa118cdbc9ec292695cb8dafa4368 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 6 Aug 2022 18:33:50 +0200 Subject: [PATCH 052/137] Make mute signalling work MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 3 +++ src/webrtc/callEventTypes.ts | 6 +++--- src/webrtc/groupCall.ts | 8 +++++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index d8bb62df273..5ae76eb918b 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -619,6 +619,9 @@ export class MatrixCall extends TypedEventEmitter ({ id: transceiver.sender.track.id, })), diff --git a/src/webrtc/callEventTypes.ts b/src/webrtc/callEventTypes.ts index a49591d3f25..46c134bef18 100644 --- a/src/webrtc/callEventTypes.ts +++ b/src/webrtc/callEventTypes.ts @@ -12,10 +12,10 @@ export enum SDPStreamMetadataPurpose { } export interface SDPStreamMetadataObject { - purpose: SDPStreamMetadataPurpose; userId?: string; - audio_muted?: boolean; - video_muted?: boolean; + purpose: SDPStreamMetadataPurpose; + audio_muted: boolean; + video_muted: boolean; } export interface SDPStreamMetadata { diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index cc36a29ed16..65e8f4577b0 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -112,7 +112,9 @@ export interface IGroupCallMemberTrack { export interface IGroupCallRoomMemberFeed { id: string; purpose: SDPStreamMetadataPurpose; - tracks: IGroupCallMemberTrack[]; + audio_muted?: boolean; + video_muted?: boolean; + tracks?: IGroupCallMemberTrack[]; } export interface IGroupCallRoomMemberDevice { @@ -560,6 +562,7 @@ export class GroupCall extends TypedEventEmitter< } this.emit(GroupCallEvent.LocalMuteStateChanged, muted, this.isLocalVideoMuted()); + this.sendMemberStateEvent(); return true; } @@ -588,6 +591,7 @@ export class GroupCall extends TypedEventEmitter< } this.emit(GroupCallEvent.LocalMuteStateChanged, this.isMicrophoneMuted(), muted); + this.sendMemberStateEvent(); return true; } @@ -814,6 +818,8 @@ export class GroupCall extends TypedEventEmitter< return recursivelyAssign(feedAcc, { [feed.id]: { purpose: feed.purpose, + audio_muted: feed.audio_muted, + video_muted: feed.video_muted, userId: event.getSender(), }, }, true); From e98ec6df1a87760ceb90bd875342d4aeab6c0c3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 8 Aug 2022 13:33:16 +0200 Subject: [PATCH 053/137] Update comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 65e8f4577b0..b627569967a 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -353,11 +353,7 @@ export class GroupCall extends TypedEventEmitter< if (this.sfu) { const opponentDevice = { "device_id": this.sfu.device_id, - // XXX: the SFU might need to specify a session_id so that if it - // restarts and starts sending invites to us, we know that it's - // forgotten who we were? But then we need a way to communicate - // the session_id to the clients, which is tough if the SFU is - // not in the right room. + // XXX: What if an SFU gets restarted? "session_id": "sfu", "feeds": [], }; From 836db69cdf8284f09b917fc35f8d5ae4ec704dd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 9 Aug 2022 12:54:29 +0200 Subject: [PATCH 054/137] Fix possible race condition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 5ae76eb918b..fc43938dc93 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -800,7 +800,7 @@ export class MatrixCall extends TypedEventEmitter { this.peerConn.setLocalDescription(offer); @@ -2304,7 +2304,7 @@ export class MatrixCall extends TypedEventEmitter => { // Other than the initial offer, we handle negotiation manually when calling with an SFU - if (this.isSfu && this.state !== CallState.CreateOffer) return; + if (this.isSfu && ![CallState.Fledgling, CallState.CreateOffer].includes(this.state)) return; logger.info(`Call ${this.callId} Negotiation is needed!`); From 17f4140be8f978596c85f492880953525bce0563 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 9 Aug 2022 13:39:08 +0200 Subject: [PATCH 055/137] Avoid race condition where we would call people full-mesh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index b627569967a..ca36d0e1473 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -327,6 +327,11 @@ export class GroupCall extends TypedEventEmitter< this.activeSpeaker = null; + // This needs to be done before we set the state to entered. With the + // state set to entered, we'll start calling other participants full-mesh + // which we don't want, if we have an SFU + this.sfu = this.client.getSfu(); + this.setState(GroupCallState.Entered); logger.log(`Entered group call ${this.groupCallId}`); @@ -349,7 +354,6 @@ export class GroupCall extends TypedEventEmitter< this.onActiveSpeakerLoop(); - this.sfu = this.client.getSfu(); if (this.sfu) { const opponentDevice = { "device_id": this.sfu.device_id, From ec7a6a2dd561e2c9434311072a092cc272121e74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 9 Aug 2022 14:21:01 +0200 Subject: [PATCH 056/137] Try to get the `trackId`s from SDP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index fc43938dc93..3bfa64ffa01 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -606,6 +606,7 @@ export class MatrixCall extends TypedEventEmitter ({ - id: transceiver.sender.track.id, - })), + tracks: transceivers.map((transceiver) => { + const media = sdp.media.find((m) => m.mid === transceiver.mid); + if (media) { + return { id: media.msid.split(" ")[1] }; + } else { + return { id: transceiver.sender.track.id }; + } + }), }); } return feeds; From 05e92e0561c329cebe94bec7089dd0381ab34360 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 9 Aug 2022 14:25:30 +0200 Subject: [PATCH 057/137] Don't be an idiot, Simon MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 3bfa64ffa01..bfbf14e3427 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -606,7 +606,7 @@ export class MatrixCall extends TypedEventEmitter Date: Tue, 9 Aug 2022 14:30:30 +0200 Subject: [PATCH 058/137] Be safe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index bfbf14e3427..5d40ed27df9 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -606,7 +606,7 @@ export class MatrixCall extends TypedEventEmitter { - const media = sdp.media.find((m) => m.mid === transceiver.mid); - if (media) { - return { id: media.msid.split(" ")[1] }; + const splitMsid = sdp?.media?.find((m) => m.mid === transceiver.mid)?.msid?.split(" "); + if (splitMsid?.[1]) { + return { id: splitMsid[1] }; } else { return { id: transceiver.sender.track.id }; } From ccd55b286f2fff1e0ceb8e734fe9c9e17e23c403 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 9 Aug 2022 15:48:26 +0200 Subject: [PATCH 059/137] Add some logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 5d40ed27df9..6d71a464acb 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -626,8 +626,10 @@ export class MatrixCall extends TypedEventEmitter { const splitMsid = sdp?.media?.find((m) => m.mid === transceiver.mid)?.msid?.split(" "); if (splitMsid?.[1]) { + logger.warn("Using msid to get trackId", splitMsid); return { id: splitMsid[1] }; } else { + logger.warn("Using transceiver to get trackId", splitMsid); return { id: transceiver.sender.track.id }; } }), From ba6f4e090e38a0b689dc34e5b30b55ccbbb409d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 9 Aug 2022 16:00:45 +0200 Subject: [PATCH 060/137] Hack to send state events when local description changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 6d71a464acb..83497ad3b7a 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2071,6 +2071,7 @@ export class MatrixCall extends TypedEventEmitter Date: Tue, 9 Aug 2022 16:06:45 +0200 Subject: [PATCH 061/137] More logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 83497ad3b7a..d4bc389031e 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -624,6 +624,7 @@ export class MatrixCall extends TypedEventEmitter { + logger.warn("Parsed SDP here:", sdp); const splitMsid = sdp?.media?.find((m) => m.mid === transceiver.mid)?.msid?.split(" "); if (splitMsid?.[1]) { logger.warn("Using msid to get trackId", splitMsid); From f0fd24022ab37ba3ea049006c9f31a936e164a38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 9 Aug 2022 16:07:26 +0200 Subject: [PATCH 062/137] Event more logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index d4bc389031e..08481e15736 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -624,6 +624,7 @@ export class MatrixCall extends TypedEventEmitter { + logger.warn("Original SDP here:", this.peerConn.localDescription.sdp); logger.warn("Parsed SDP here:", sdp); const splitMsid = sdp?.media?.find((m) => m.mid === transceiver.mid)?.msid?.split(" "); if (splitMsid?.[1]) { From 892201ff13f9f4ecfd8ae218e5c654cafb531686 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 9 Aug 2022 16:14:11 +0200 Subject: [PATCH 063/137] Smarter logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 08481e15736..266f0c94145 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -607,6 +607,10 @@ export class MatrixCall extends TypedEventEmitter { - logger.warn("Original SDP here:", this.peerConn.localDescription.sdp); - logger.warn("Parsed SDP here:", sdp); const splitMsid = sdp?.media?.find((m) => m.mid === transceiver.mid)?.msid?.split(" "); if (splitMsid?.[1]) { - logger.warn("Using msid to get trackId", splitMsid); + logger.warn("Using msid to get trackId", splitMsid, transceiver.mid); return { id: splitMsid[1] }; } else { - logger.warn("Using transceiver to get trackId", splitMsid); + logger.warn("Using transceiver to get trackId", splitMsid, transceiver.mid); return { id: transceiver.sender.track.id }; } }), From 55111b6753eec736e713e117eed6dd1b2ab45337 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 9 Aug 2022 16:27:55 +0200 Subject: [PATCH 064/137] Even more logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 266f0c94145..2a9aceb3431 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -628,7 +628,9 @@ export class MatrixCall extends TypedEventEmitter { - const splitMsid = sdp?.media?.find((m) => m.mid === transceiver.mid)?.msid?.split(" "); + const media = sdp?.media?.find((m) => m.mid === transceiver.mid); + const splitMsid = media?.msid?.split(" "); + logger.warn("Media with matching mid", media); if (splitMsid?.[1]) { logger.warn("Using msid to get trackId", splitMsid, transceiver.mid); return { id: splitMsid[1] }; From bf8bfbaecae9a90de3e27516160004d716171de9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 9 Aug 2022 16:32:54 +0200 Subject: [PATCH 065/137] Types? MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 2a9aceb3431..ea41ae58e70 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -628,7 +628,7 @@ export class MatrixCall extends TypedEventEmitter { - const media = sdp?.media?.find((m) => m.mid === transceiver.mid); + const media = sdp?.media?.find((m) => m.mid == transceiver.mid); const splitMsid = media?.msid?.split(" "); logger.warn("Media with matching mid", media); if (splitMsid?.[1]) { From 1537a2b120a161b3bc2b3ec292edac651ea0aa57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 9 Aug 2022 16:43:33 +0200 Subject: [PATCH 066/137] Tidy up code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index ea41ae58e70..961672510d5 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -607,10 +607,6 @@ export class MatrixCall extends TypedEventEmitter { - const media = sdp?.media?.find((m) => m.mid == transceiver.mid); - const splitMsid = media?.msid?.split(" "); - logger.warn("Media with matching mid", media); - if (splitMsid?.[1]) { - logger.warn("Using msid to get trackId", splitMsid, transceiver.mid); - return { id: splitMsid[1] }; + // XXX: We only use double equals because MediaDescription::mid is in fact a number + const trackId = sdp?.media?.find((m) => m.mid == transceiver.mid)?.msid?.split(" ")?.[1]; + if (trackId) { + return { id: trackId }; } else { - logger.warn("Using transceiver to get trackId", splitMsid, transceiver.mid); return { id: transceiver.sender.track.id }; } }), From 95051aba02c670c3457dbd2857206ded70aae32d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 11 Aug 2022 16:36:35 +0200 Subject: [PATCH 067/137] Implement SFU timeouts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 19 ++++++++++++++++++- src/webrtc/callEventTypes.ts | 5 +++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 961672510d5..d0646903163 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -51,6 +51,7 @@ import { ISfuPublishDataChannelMessage, ISfuUnpublishDataChannelMessage, ISfuOfferDataChannelMessage, + ISfuKeepAliveDataChannelMessage, } from './callEventTypes'; import { CallFeed } from './callFeed'; import { MatrixClient } from "../client"; @@ -277,6 +278,9 @@ const FALLBACK_ICE_SERVER = 'stun:turn.matrix.org'; /** The length of time a call can be ringing for. */ const CALL_TIMEOUT_MS = 60000; +const CALL_LENGTH_INTERVAL = 1000; // 1 second +const SFU_KEEP_ALIVE_INTERVAL = 30 * 1000; // 30 seconds + export class CallError extends Error { code: string; @@ -399,6 +403,7 @@ export class MatrixCall extends TypedEventEmitter; private callLengthInterval: ReturnType; private callLength = 0; @@ -2203,7 +2208,15 @@ export class MatrixCall extends TypedEventEmitter { this.callLength++; this.emit(CallEvent.LengthChanged, this.callLength); - }, 1000); + }, CALL_LENGTH_INTERVAL); + } + if (!this.sfuKeepAliveInterval) { + this.sfuKeepAliveInterval = setInterval(() => { + this.sendSFUDataChannelMessage({ + op: "alive", + ts: Date.now(), + } as ISfuKeepAliveDataChannelMessage); + }, SFU_KEEP_ALIVE_INTERVAL); } } else if (this.peerConn.iceConnectionState == 'failed') { // Firefox for Android does not yet have support for restartIce() @@ -2585,6 +2598,10 @@ export class MatrixCall extends TypedEventEmitter Date: Thu, 11 Aug 2022 21:21:18 +0200 Subject: [PATCH 068/137] Fix timeouts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 3 +-- src/webrtc/callEventTypes.ts | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index d0646903163..8c4f03f63e6 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2214,9 +2214,8 @@ export class MatrixCall extends TypedEventEmitter { this.sendSFUDataChannelMessage({ op: "alive", - ts: Date.now(), } as ISfuKeepAliveDataChannelMessage); - }, SFU_KEEP_ALIVE_INTERVAL); + }, SFU_KEEP_ALIVE_INTERVAL * 3 / 4); } } else if (this.peerConn.iceConnectionState == 'failed') { // Firefox for Android does not yet have support for restartIce() diff --git a/src/webrtc/callEventTypes.ts b/src/webrtc/callEventTypes.ts index 191fc687e25..857ed81b6d8 100644 --- a/src/webrtc/callEventTypes.ts +++ b/src/webrtc/callEventTypes.ts @@ -129,7 +129,6 @@ export interface ISfuUnpublishDataChannelMessage extends ISfuBaseDataChannelMess export interface ISfuKeepAliveDataChannelMessage extends ISfuBaseDataChannelMessage { op: "alive"; - ts: number; } /* eslint-enable camelcase */ From fae7bb4cc03da0a17332d38c4103e03a4e9725a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 12 Aug 2022 12:19:13 +0200 Subject: [PATCH 069/137] Don't spam SFUs with metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 8c4f03f63e6..0ba6117cfce 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1493,6 +1493,9 @@ export class MatrixCall extends TypedEventEmitter { + // We send metadata over state in SFU calls + if (this.isSfu) return; + await this.sendVoipEvent(EventType.CallSDPStreamMetadataChangedPrefix, { [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(), }); From cfa11e27849125ef47cd796eabedce58faef5c74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 12 Aug 2022 12:44:48 +0200 Subject: [PATCH 070/137] Delint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 17 ++++++++--------- src/webrtc/groupCall.ts | 1 - 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 8530c5a1de8..4a73d28a1c8 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2405,15 +2405,6 @@ export class MatrixCall extends TypedEventEmitter Date: Fri, 12 Aug 2022 16:00:10 +0200 Subject: [PATCH 071/137] Don't publish tracks that are not in SDP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 4a73d28a1c8..b613203f0e7 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -623,10 +623,13 @@ export class MatrixCall extends TypedEventEmitter { + // XXX: We only use double equals because MediaDescription::mid is in fact a number + const trackId = sdp?.media?.find((m) => m.mid == transceiver.mid)?.msid?.split(" ")?.[1]; + return trackId ? { id: trackId } : undefined; + }).filter((t) => Boolean(t)); feeds.push({ id: feed.stream.id, @@ -634,15 +637,7 @@ export class MatrixCall extends TypedEventEmitter { - // XXX: We only use double equals because MediaDescription::mid is in fact a number - const trackId = sdp?.media?.find((m) => m.mid == transceiver.mid)?.msid?.split(" ")?.[1]; - if (trackId) { - return { id: trackId }; - } else { - return { id: transceiver.sender.track.id }; - } - }), + tracks: tracks.length ? tracks : undefined, }); } return feeds; From b361f8df2923c2a673133279c4d57aeeed1b7888 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 12 Aug 2022 17:14:35 +0200 Subject: [PATCH 072/137] Properly skip track-less MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index b613203f0e7..96a18b77608 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1979,7 +1979,7 @@ export class MatrixCall extends TypedEventEmitter feed.tracks) // Skip trackless feeds + .filter((feed) => feed.tracks?.length) // Skip trackless feeds .reduce((tracks, f) => [...tracks, ...f.tracks.map((t) => ({ stream_id: f.id, track_id: t.id }))], []) // Get array of tracks from feeds .filter((track) => !this.subscribedTracks.find((subscribed) => utils.deepCompare(track, subscribed))); // Filter out already subscribed tracks From 5159b24812030bcc962ff1c982979e2f526f4570 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 13 Aug 2022 14:32:24 +0200 Subject: [PATCH 073/137] Switch to data-channel for sending metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 135 +++++++++++++++++------------------ src/webrtc/callEventTypes.ts | 39 +++++++--- src/webrtc/groupCall.ts | 62 +++------------- 3 files changed, 106 insertions(+), 130 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 96a18b77608..e49f7deb1eb 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -51,17 +51,16 @@ import { ISfuPublishDataChannelMessage, ISfuUnpublishDataChannelMessage, ISfuOfferDataChannelMessage, - ISfuKeepAliveDataChannelMessage, + SFUDataChannelMessageOp, + ISfuMetadataDataChannelMessage, + SDPStreamMetadataTracks, } from './callEventTypes'; import { CallFeed } from './callFeed'; import { MatrixClient } from "../client"; import { EventEmitterEvents, TypedEventEmitter } from "../models/typed-event-emitter"; import { DeviceInfo } from '../crypto/deviceinfo'; import { IScreensharingOpts } from "./mediaHandler"; -import { - GroupCallUnknownDeviceError, - IGroupCallRoomMemberFeed, -} from "./groupCall"; +import { GroupCallUnknownDeviceError } from "./groupCall"; // events: hangup, error(err), replaced(call), state(state, oldState) @@ -90,7 +89,6 @@ interface CallOpts { opponentDeviceId?: string; opponentSessionId?: string; groupCallId?: string; - initialRemoteSDPStreamMetadata?: SDPStreamMetadata; isSfu?: boolean; } @@ -422,7 +420,6 @@ export class MatrixCall extends TypedEventEmitter { + : this.screensharingTransceivers).reduce((tracks, transceiver) => { // XXX: We only use double equals because MediaDescription::mid is in fact a number const trackId = sdp?.media?.find((m) => m.mid == transceiver.mid)?.msid?.split(" ")?.[1]; - return trackId ? { id: trackId } : undefined; - }).filter((t) => Boolean(t)); + if (trackId) { + tracks[trackId] = {}; + } + return tracks; + }, {} as SDPStreamMetadataTracks); + if (!Object.keys(tracks).length) continue; - feeds.push({ - id: feed.stream.id, - purpose: feed.purpose, + metadata[localFeed.sdpMetadataStreamId] = { + // FIXME: This allows for impersonation - the SFU should be + // handling these + user_id: this.client.getUserId(), + device_id: this.client.getDeviceId(), + purpose: localFeed.purpose, // FIXME: This is very ineffective as state is slow, we should really be sending this over DC - audio_muted: feed.isAudioMuted(), - video_muted: feed.isVideoMuted(), - tracks: tracks.length ? tracks : undefined, - }); + audio_muted: localFeed.isAudioMuted(), + video_muted: localFeed.isVideoMuted(), + tracks: tracks, + }; } - return feeds; + return metadata; } /** @@ -660,7 +650,7 @@ export class MatrixCall extends TypedEventEmitter { - this.peerConn.setLocalDescription(offer); + this.peerConn.createOffer().then(async (offer) => { + await this.peerConn.setLocalDescription(offer); - this.sendSFUDataChannelMessage({ - op: "publish", + this.sendSFUDataChannelMessage(SFUDataChannelMessageOp.Publish, { sdp: offer.sdp, } as ISfuPublishDataChannelMessage); this.emit(CallEvent.FeedsChanged, this.feeds); @@ -852,11 +841,10 @@ export class MatrixCall extends TypedEventEmitter { - this.peerConn.setLocalDescription(offer); + this.peerConn.createOffer().then(async (offer) => { + await this.peerConn.setLocalDescription(offer); - this.sendSFUDataChannelMessage({ - op: "unpublish", + this.sendSFUDataChannelMessage(SFUDataChannelMessageOp.Unpublish, { sdp: offer.sdp, stop: tracksToUnpublish, } as ISfuUnpublishDataChannelMessage); @@ -1490,12 +1478,13 @@ export class MatrixCall extends TypedEventEmitter { - // We send metadata over state in SFU calls - if (this.isSfu) return; - - await this.sendVoipEvent(EventType.CallSDPStreamMetadataChangedPrefix, { - [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(), - }); + if (this.isSfu) { + this.sendSFUDataChannelMessage(SFUDataChannelMessageOp.Metadata); + } else { + await this.sendVoipEvent(EventType.CallSDPStreamMetadataChangedPrefix, { + [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(), + }); + } } private gotCallFeedsForInvite(callFeeds: CallFeed[], requestScreenshareFeed = false): void { @@ -1923,9 +1912,10 @@ export class MatrixCall extends TypedEventEmitter { + public async subscribeToSFU(): Promise { await this.waitForDatachannelToBeOpen(); - if (this.dataChannel.readyState !== "open") { - logger.warn("Can't sent to DC in state:", this.dataChannel.readyState); - return; - } + if (!this.remoteSDPStreamMetadata) return; + if (this.dataChannel.readyState !== "open") return; - const tracks: ISfuTrackDesc[] = feeds - .filter((feed) => feed.tracks?.length) // Skip trackless feeds - .reduce((tracks, f) => [...tracks, ...f.tracks.map((t) => ({ stream_id: f.id, track_id: t.id }))], []) // Get array of tracks from feeds + const tracks: ISfuTrackDesc[] = Object.entries(this.remoteSDPStreamMetadata) + .filter(([, info]) => Boolean(info.tracks)) // Skip trackless feeds + .reduce((a, [s, i]) => [...a, ...Object.keys(i.tracks).map((t) => ({ stream_id: s, track_id: t }))], []) // Get array of tracks from feeds .filter((track) => !this.subscribedTracks.find((subscribed) => utils.deepCompare(track, subscribed))); // Filter out already subscribed tracks if (tracks.length === 0) { @@ -1991,13 +1984,13 @@ export class MatrixCall extends TypedEventEmitter { - this.sendSFUDataChannelMessage({ - op: "alive", - } as ISfuKeepAliveDataChannelMessage); + this.sendSFUDataChannelMessage(SFUDataChannelMessageOp.Alive); }, SFU_KEEP_ALIVE_INTERVAL * 3 / 4); } } else if (this.peerConn.iceConnectionState == 'failed') { @@ -2445,12 +2440,17 @@ export class MatrixCall extends TypedEventEmitter): void { + private sendSFUDataChannelMessage(op: SFUDataChannelMessageOp, content: object = {}): void { const realContent: ISfuBaseDataChannelMessage = Object.assign(content, { + op: op, id: randomString(24), conf_id: this.groupCallId, }); + if (![SFUDataChannelMessageOp.Select, SFUDataChannelMessageOp.Alive].includes(op)) { + realContent.metadata = this.getLocalSDPStreamMetadata(); + } + // FIXME: RPC reliability over DC this.dataChannel.send(JSON.stringify(realContent)); logger.warn(`Sent ${realContent.op} over DC:`, realContent); @@ -2925,7 +2925,6 @@ export function createNewMatrixCall(client: any, roomId: string, options?: CallO opponentDeviceId: options?.opponentDeviceId, opponentSessionId: options?.opponentSessionId, groupCallId: options?.groupCallId, - initialRemoteSDPStreamMetadata: options?.initialRemoteSDPStreamMetadata, isSfu: options.isSfu, }; const call = new MatrixCall(opts); diff --git a/src/webrtc/callEventTypes.ts b/src/webrtc/callEventTypes.ts index 857ed81b6d8..a4a1dd32246 100644 --- a/src/webrtc/callEventTypes.ts +++ b/src/webrtc/callEventTypes.ts @@ -11,11 +11,19 @@ export enum SDPStreamMetadataPurpose { Screenshare = "m.screenshare", } +export interface SDPStreamMetadataTrack {} + +export interface SDPStreamMetadataTracks { + [key: string]: SDPStreamMetadataTrack; +} + export interface SDPStreamMetadataObject { - userId?: string; + user_id: string; + device_id: string; purpose: SDPStreamMetadataPurpose; audio_muted: boolean; video_muted: boolean; + tracks: SDPStreamMetadataTracks; } export interface SDPStreamMetadata { @@ -95,40 +103,55 @@ export interface ISfuTrackDesc { track_id: string; } +export enum SFUDataChannelMessageOp { + Select = "select", + Offer = "offer", + Answer = "answer", + Publish = "publish", + Unpublish = "unpublish", + Metadata = "metadata", + Alive = "alive", +} + export interface ISfuBaseDataChannelMessage { - op: string; + op: SFUDataChannelMessageOp; id: string; conf_id: string; + metadata?: SDPStreamMetadata; } export interface ISfuSelectDataChannelMessage extends ISfuBaseDataChannelMessage { - op: "select"; + op: SFUDataChannelMessageOp.Select; start: ISfuTrackDesc[]; } export interface ISfuOfferDataChannelMessage extends ISfuBaseDataChannelMessage { - op: "offer"; + op: SFUDataChannelMessageOp.Offer; sdp: string; } export interface ISfuAnswerDataChannelMessage extends ISfuBaseDataChannelMessage { - op: "answer"; + op: SFUDataChannelMessageOp.Answer; sdp: string; } export interface ISfuPublishDataChannelMessage extends ISfuBaseDataChannelMessage { - op: "publish"; + op: SFUDataChannelMessageOp.Publish; sdp: string; } export interface ISfuUnpublishDataChannelMessage extends ISfuBaseDataChannelMessage { - op: "unpublish"; + op: SFUDataChannelMessageOp.Unpublish; sdp: string; stop: ISfuTrackDesc[]; } +export interface ISfuMetadataDataChannelMessage extends ISfuBaseDataChannelMessage { + op: SFUDataChannelMessageOp.Metadata; +} + export interface ISfuKeepAliveDataChannelMessage extends ISfuBaseDataChannelMessage { - op: "alive"; + op: SFUDataChannelMessageOp.Alive; } /* eslint-enable camelcase */ diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 885db59ba92..62325f409b2 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -15,14 +15,13 @@ import { RoomMember } from "../models/room-member"; import { Room } from "../models/room"; import { logger } from "../logger"; import { ReEmitter } from "../ReEmitter"; -import { SDPStreamMetadata, SDPStreamMetadataPurpose } from "./callEventTypes"; +import { SDPStreamMetadataPurpose } from "./callEventTypes"; import { ISendEventResponse } from "../@types/requests"; import { MatrixEvent } from "../models/event"; import { EventType } from "../@types/event"; import { CallEventHandlerEvent } from "./callEventHandler"; import { GroupCallEventHandlerEvent } from "./groupCallEventHandler"; import { IScreensharingOpts } from "./mediaHandler"; -import { recursivelyAssign } from "../utils"; export enum GroupCallIntent { Ring = "m.ring", @@ -105,16 +104,8 @@ export interface IGroupCallDataChannelOptions { protocol: string; } -export interface IGroupCallMemberTrack { - id: string; -} - export interface IGroupCallRoomMemberFeed { - id: string; purpose: SDPStreamMetadataPurpose; - audio_muted?: boolean; - video_muted?: boolean; - tracks?: IGroupCallMemberTrack[]; } export interface IGroupCallRoomMemberDevice { @@ -370,7 +361,6 @@ export class GroupCall extends TypedEventEmitter< opponentDeviceId: opponentDevice.device_id, opponentSessionId: opponentDevice.session_id, groupCallId: this.groupCallId, - initialRemoteSDPStreamMetadata: this.getRemoteSDPStreamMetadataForCall(), isSfu: true, }, ); @@ -742,8 +732,9 @@ export class GroupCall extends TypedEventEmitter< { "device_id": this.client.getDeviceId(), "session_id": this.client.getSessionId(), - // We assume that all calls are sending the same feeds - "feeds": this.calls[0]?.getGroupCallRoomMemberFeeds(), + "feeds": this.getLocalFeeds().map((feed) => ({ + purpose: feed.purpose, + })), // TODO: Add data channels }, ], @@ -799,52 +790,15 @@ export class GroupCall extends TypedEventEmitter< return this.client.sendStateEvent(this.room.roomId, EventType.GroupCallMemberPrefix, content, localUserId); } - private getRemoteFeedsFromState(): IGroupCallRoomMemberFeed[] { - return this.getMemberStateEvents()?.reduce((feeds, event) => { - if (event.getSender() === this.client.getUserId()) return feeds; // Ignore local - const newFeeds = event.getContent()?.["m.calls"]?.[0]?.["m.devices"]?.[0]?.feeds; - if (!newFeeds) return feeds; - return [...feeds, ...newFeeds]; - }, []) ?? []; - } - - private getRemoteSDPStreamMetadataForCall(): SDPStreamMetadata { - return this.getMemberStateEvents().reduce((metaAcc: SDPStreamMetadata, event: MatrixEvent) => { - if (event.getSender() === this.client.getUserId()) return metaAcc; // Ignore local - const feeds = event.getContent()?.["m.calls"]?.[0]?.["m.devices"]?.[0]?.feeds; - if (!feeds) return metaAcc; - const metadata = feeds.reduce((feedAcc: SDPStreamMetadata, feed: IGroupCallRoomMemberFeed) => { - if (!feed?.id) return feedAcc; - return recursivelyAssign(feedAcc, { - [feed.id]: { - purpose: feed.purpose, - audio_muted: feed.audio_muted, - video_muted: feed.video_muted, - userId: event.getSender(), - }, - }, true); - }, {}); - return recursivelyAssign(metaAcc, metadata, true); - }, {}); - } - public onMemberStateChanged = async (event: MatrixEvent) => { + if (this.sfu) return; + // The member events may be received for another room, which we will ignore. if (event.getRoomId() !== this.room.roomId) return; const member = this.room.getMember(event.getStateKey()); if (!member) return; - if (this.sfu) { - const sfuCall = this.getCallByUserId(this.sfu.user_id); - if (!sfuCall) return; - - sfuCall.updateRemoteSDPStreamMetadata(this.getRemoteSDPStreamMetadataForCall()); - sfuCall.subscribeToSFU(this.getRemoteFeedsFromState()); - - return; - } - const ignore = () => { this.removeParticipant(member); clearTimeout(this.memberStateExpirationTimers.get(member.userId)); @@ -1164,8 +1118,8 @@ export class GroupCall extends TypedEventEmitter< this.retryCallCounts.delete(getCallUserId(call)); // if we're calling an SFU, subscribe to its feeds - if (call.getOpponentMember().userId === this.sfu.user_id) { - call.subscribeToSFU(this.getRemoteFeedsFromState()); + if (call.isSfu) { + call.subscribeToSFU(); } } }; From b837da2a0b080bf04e327a03c1e296609fb5af97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 13 Aug 2022 17:17:31 +0200 Subject: [PATCH 074/137] Remove unused field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 2 -- src/webrtc/callEventTypes.ts | 2 -- 2 files changed, 4 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index e49f7deb1eb..a322108369a 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2443,8 +2443,6 @@ export class MatrixCall extends TypedEventEmitter Date: Sat, 20 Aug 2022 10:32:46 +0200 Subject: [PATCH 075/137] Remove TODO we decided not to do MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 66d0c8ca476..8a9346ec476 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -368,7 +368,7 @@ export class GroupCall extends TypedEventEmitter< sfuCall.isPtt = this.isPtt; try { - await sfuCall.placeCallWithCallFeeds(this.getLocalFeeds()); // TODO: We should just setup the datachannel + await sfuCall.placeCallWithCallFeeds(this.getLocalFeeds()); sfuCall.createDataChannel("datachannel", this.dataChannelOptions); } catch (e) { logger.warn(`Failed to place call to ${this.sfu.user_id}!`, e); From bfdb974e71c0e646f3b9a0b0a1dd95269668f7cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 8 Nov 2022 11:26:03 +0100 Subject: [PATCH 076/137] Fixup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index eddc237165c..adfc6b35912 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -183,6 +183,9 @@ export class GroupCall extends TypedEventEmitter< private transmitTimer: ReturnType | null = null; private memberStateExpirationTimers: Map> = new Map(); private resendMemberStateTimer: ReturnType | null = null; + private initWithAudioMuted = false; + private initWithVideoMuted = false; + private sfu: ISfuInfo | null; constructor( private client: MatrixClient, @@ -1110,6 +1113,16 @@ export class GroupCall extends TypedEventEmitter< // Find removed feeds [...this.userMediaFeeds, ...this.screenshareFeeds].filter((gf) => gf.disposed).forEach((feed) => { + // Only remove the feed if the other feed array doesn't have a feed + // from the given user + if ( + !(feed.purpose === SDPStreamMetadataPurpose.Usermedia + ? this.screenshareFeeds + : this.userMediaFeeds).some((f) => f.userId === feed.getMember().userId) + ) { + this.removeParticipant(feed.getMember()); + } + if (feed.purpose === SDPStreamMetadataPurpose.Usermedia) this.removeUserMediaFeed(feed); else if (feed.purpose === SDPStreamMetadataPurpose.Screenshare) this.removeScreenshareFeed(feed); }); @@ -1118,6 +1131,8 @@ export class GroupCall extends TypedEventEmitter< call.getRemoteFeeds().filter((cf) => { return !this.userMediaFeeds.find((gf) => gf.stream.id === cf.stream.id); }).forEach((feed) => { + this.addParticipant(feed.getMember()); + if (feed.purpose === SDPStreamMetadataPurpose.Usermedia) this.addUserMediaFeed(feed); else if (feed.purpose === SDPStreamMetadataPurpose.Screenshare) this.addScreenshareFeed(feed); }); From c9319538ab9a9aa02540015f610bd7b3546e0bd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 12 Nov 2022 11:57:35 +0100 Subject: [PATCH 077/137] Improve TS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 7cf74ad9eb7..9b9c5ed8f9c 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -186,7 +186,7 @@ export class GroupCall extends TypedEventEmitter< private resendMemberStateTimer: ReturnType | null = null; private initWithAudioMuted = false; private initWithVideoMuted = false; - private sfu: ISfuInfo | null; + private sfu: ISfuInfo | null = null; constructor( private client: MatrixClient, @@ -367,6 +367,17 @@ export class GroupCall extends TypedEventEmitter< "feeds": [], }; + const onError = (e?): void => { + logger.warn(`Failed to place call to ${this.sfu!.user_id}!`, e); + this.emit( + GroupCallEvent.Error, + new GroupCallError( + GroupCallErrorCode.PlaceCallFailed, + `Failed to place call to ${this.sfu!.user_id}.`, + ), + ); + }; + const sfuCall = createNewMatrixCall( this.client, this.room.roomId, @@ -378,21 +389,17 @@ export class GroupCall extends TypedEventEmitter< isSfu: true, }, ); - - sfuCall.isPtt = this.isPtt; + if (!sfuCall) { + onError(); + return; + } try { + sfuCall.isPtt = this.isPtt; await sfuCall.placeCallWithCallFeeds(this.getLocalFeeds()); sfuCall.createDataChannel("datachannel", this.dataChannelOptions); } catch (e) { - logger.warn(`Failed to place call to ${this.sfu.user_id}!`, e); - this.emit( - GroupCallEvent.Error, - new GroupCallError( - GroupCallErrorCode.PlaceCallFailed, - `Failed to place call to ${this.sfu.user_id}.`, - ), - ); + onError(e); return; } @@ -656,7 +663,7 @@ export class GroupCall extends TypedEventEmitter< // TODO: handle errors await Promise.all(this.calls.map(call => call.pushLocalFeed(call.isSfu - ? this.localScreenshareFeed + ? this.localScreenshareFeed! : this.localScreenshareFeed!.clone(), ))); @@ -1146,9 +1153,9 @@ export class GroupCall extends TypedEventEmitter< if ( !(feed.purpose === SDPStreamMetadataPurpose.Usermedia ? this.screenshareFeeds - : this.userMediaFeeds).some((f) => f.userId === feed.getMember().userId) + : this.userMediaFeeds).some((f) => f.userId === feed.getMember()!.userId) ) { - this.removeParticipant(feed.getMember()); + this.removeParticipant(feed.getMember()!); } if (feed.purpose === SDPStreamMetadataPurpose.Usermedia) this.removeUserMediaFeed(feed); @@ -1159,7 +1166,7 @@ export class GroupCall extends TypedEventEmitter< call.getRemoteFeeds().filter((cf) => { return !this.userMediaFeeds.find((gf) => gf.stream.id === cf.stream.id); }).forEach((feed) => { - this.addParticipant(feed.getMember()); + this.addParticipant(feed.getMember()!); if (feed.purpose === SDPStreamMetadataPurpose.Usermedia) this.addUserMediaFeed(feed); else if (feed.purpose === SDPStreamMetadataPurpose.Screenshare) this.addScreenshareFeed(feed); From 3e4fcd9a55d25da97b6da1df047d59ebd350a155 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 12 Nov 2022 12:03:05 +0100 Subject: [PATCH 078/137] Improve TS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- spec/unit/webrtc/call.spec.ts | 3 +++ src/client.ts | 12 ++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 030e6a6ba68..6a2c23e8372 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -820,9 +820,12 @@ describe('Call', function() { const setupCall = (audio: boolean, video: boolean): SDPStreamMetadata => { const metadata = { stream: { + user_id: "user", + device_id: "device", purpose: SDPStreamMetadataPurpose.Usermedia, audio_muted: audio, video_muted: video, + tracks: {}, }, }; (call as any).pushRemoteFeed(new MockMediaStream("stream", [ diff --git a/src/client.ts b/src/client.ts index 260e163b488..e2a28e8d880 100644 --- a/src/client.ts +++ b/src/client.ts @@ -375,7 +375,7 @@ export interface ICreateClientOpts { /** * The user ID for the local SFU to use for group calling, if any */ - localSfu?: string; + localSfuUserId?: string; /** * The device ID for the local SFU to use for group calling, if any @@ -1021,8 +1021,8 @@ export class MatrixClient extends TypedEventEmitter>(); - private localSfu: string; - private localSfuDeviceId: string; + private localSfuUserId?: string; + private localSfuDeviceId?: string; private useE2eForGroupCall = true; private toDeviceMessageQueue: ToDeviceMessageQueue; @@ -1099,7 +1099,7 @@ export class MatrixClient extends TypedEventEmitter Date: Sun, 13 Nov 2022 17:11:31 +0100 Subject: [PATCH 079/137] Improve typing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 29 ++++++++++++++++++++--------- src/webrtc/groupCallEventHandler.ts | 3 ++- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 9b9c5ed8f9c..50fe428e55a 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -109,6 +109,15 @@ export interface IGroupCallRoomMemberFeed { purpose: SDPStreamMetadataPurpose; } +export interface IGroupCallRoomState { + "m.intent": GroupCallIntent; + "m.type": GroupCallType; + "io.element.ptt"?: boolean; + // TODO: Specify data-channels + "dataChannelsEnabled"?: boolean; + "dataChannelOptions"?: IGroupCallDataChannelOptions; +} + export interface IGroupCallRoomMemberDevice { "device_id": string; "session_id": string; @@ -195,7 +204,7 @@ export class GroupCall extends TypedEventEmitter< public isPtt: boolean, public intent: GroupCallIntent, groupCallId?: string, - private dataChannelsEnabled?: boolean, + private dataChannelsEnabled = false, private dataChannelOptions?: IGroupCallDataChannelOptions, ) { super(); @@ -210,17 +219,19 @@ export class GroupCall extends TypedEventEmitter< public async create() { this.client.groupCallEventHandler!.groupCalls.set(this.room.roomId, this); + const groupCallState: IGroupCallRoomState = { + "m.intent": this.intent, + "m.type": this.type, + "io.element.ptt": this.isPtt, + // TODO: Specify data-channels better + "dataChannelsEnabled": this.dataChannelsEnabled, + "dataChannelOptions": this.dataChannelsEnabled ? this.dataChannelOptions : undefined, + }; + await this.client.sendStateEvent( this.room.roomId, EventType.GroupCallPrefix, - { - "m.intent": this.intent, - "m.type": this.type, - "io.element.ptt": this.isPtt, - // TODO: Specify datachannels - "dataChannelsEnabled": this.dataChannelsEnabled, - "dataChannelOptions": this.dataChannelOptions, - }, + groupCallState, this.groupCallId, ); diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts index 86df722895d..4074cab5a9d 100644 --- a/src/webrtc/groupCallEventHandler.ts +++ b/src/webrtc/groupCallEventHandler.ts @@ -21,6 +21,7 @@ import { GroupCallIntent, GroupCallType, IGroupCallDataChannelOptions, + IGroupCallRoomState, } from "./groupCall"; import { Room } from "../models/room"; import { RoomState, RoomStateEvent } from "../models/room-state"; @@ -140,7 +141,7 @@ export class GroupCallEventHandler { private createGroupCallFromRoomStateEvent(event: MatrixEvent): GroupCall | undefined { const roomId = event.getRoomId(); - const content = event.getContent(); + const content = event.getContent(); const room = this.client.getRoom(roomId); From 23b3d537afc19fcd76a49079897ec31ae9f1e742 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 29 Nov 2022 04:44:02 +0100 Subject: [PATCH 080/137] Implement a very basic focus selection algorithm (#2875) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Implement a very basic focus choosing algorithm Signed-off-by: Šimon Brandner * Fix for loop Signed-off-by: Šimon Brandner Signed-off-by: Šimon Brandner --- src/client.ts | 12 +++---- src/webrtc/call.ts | 27 ++++++++-------- src/webrtc/groupCall.ts | 72 ++++++++++++++++++++++++++++++----------- 3 files changed, 73 insertions(+), 38 deletions(-) diff --git a/src/client.ts b/src/client.ts index f5715971cd7..52e8cdade39 100644 --- a/src/client.ts +++ b/src/client.ts @@ -829,7 +829,7 @@ interface ITimestampToEventResponse { origin_server_ts: string; } -export interface ISfuInfo { +export interface IFocusInfo { user_id: string; device_id: string; } @@ -1575,18 +1575,16 @@ export class MatrixClient extends TypedEventEmitter { await this.peerConn!.setLocalDescription(offer); @@ -889,7 +890,7 @@ export class MatrixCall extends TypedEventEmitter { await this.peerConn!.setLocalDescription(offer); @@ -968,7 +969,7 @@ export class MatrixCall extends TypedEventEmitter { - if (this.isSfu) { + if (this.isFocus) { this.sendSFUDataChannelMessage(SFUDataChannelMessageOp.Metadata); } else { await this.sendVoipEvent(EventType.CallSDPStreamMetadataChangedPrefix, { @@ -1841,7 +1842,7 @@ export class MatrixCall extends TypedEventEmitter => { // Other than the initial offer, we handle negotiation manually when calling with an SFU - if (this.isSfu && ![CallState.Fledgling, CallState.CreateOffer].includes(this.state)) return; + if (this.isFocus && ![CallState.Fledgling, CallState.CreateOffer].includes(this.state)) return; logger.info(`Call ${this.callId} Negotiation is needed!`); @@ -2475,7 +2476,7 @@ export class MatrixCall extends TypedEventEmitter, ): MatrixCall | null { if (!supportsMatrixCall()) return null; @@ -3005,7 +3006,7 @@ export function createNewMatrixCall( opponentDeviceId: options?.opponentDeviceId, opponentSessionId: options?.opponentSessionId, groupCallId: options?.groupCallId, - isSfu: options?.isSfu, + isFocus: options?.isFocus, }; const call = new MatrixCall(opts); diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index e512f3a227e..634b63ca933 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -1,6 +1,6 @@ import { TypedEventEmitter } from "../models/typed-event-emitter"; import { CallFeed, SPEAKING_THRESHOLD } from "./callFeed"; -import { ISfuInfo, MatrixClient } from "../client"; +import { IFocusInfo, MatrixClient } from "../client"; import { CallErrorCode, CallEvent, @@ -122,11 +122,12 @@ export interface IGroupCallRoomMemberDevice { "device_id": string; "session_id": string; "feeds": IGroupCallRoomMemberFeed[]; + "m.foci.active"?: IFocusInfo[]; + "m.foci.preferred"?: IFocusInfo[]; } export interface IGroupCallRoomMemberCallState { "m.call_id": string; - "m.foci"?: string[]; "m.devices": IGroupCallRoomMemberDevice[]; } @@ -195,7 +196,7 @@ export class GroupCall extends TypedEventEmitter< private resendMemberStateTimer: ReturnType | null = null; private initWithAudioMuted = false; private initWithVideoMuted = false; - private sfu: ISfuInfo | null = null; + private foci: IFocusInfo[] = []; constructor( private client: MatrixClient, @@ -244,6 +245,15 @@ export class GroupCall extends TypedEventEmitter< this.emit(GroupCallEvent.GroupCallStateChanged, newState, oldState); } + private getPreferredFoci(): IFocusInfo[] { + const preferredFoci = this.client.getFoci(); + const isUsingPreferredFocus = Boolean(preferredFoci.find(pf => ( + this.foci.find(f => pf.user_id === f.user_id && pf.device_id === pf.device_id) + ))); + + return isUsingPreferredFocus ? [] : preferredFoci; + } + public getLocalFeeds(): CallFeed[] { const feeds: CallFeed[] = []; @@ -339,14 +349,16 @@ export class GroupCall extends TypedEventEmitter< this.addParticipant(this.room.getMember(this.client.getUserId()!)!); - await this.sendMemberStateEvent(); - - this.activeSpeaker = undefined; + // TODO: Call preferred foci // This needs to be done before we set the state to entered. With the // state set to entered, we'll start calling other participants full-mesh - // which we don't want, if we have an SFU - this.sfu = this.client.getSfu(); + // which we don't want, if we have a focus + this.chooseFocus(); + + await this.sendMemberStateEvent(); + + this.activeSpeaker = undefined; this.setState(GroupCallState.Entered); @@ -370,21 +382,21 @@ export class GroupCall extends TypedEventEmitter< this.onActiveSpeakerLoop(); - if (this.sfu) { + if (this.foci[0]) { const opponentDevice = { - "device_id": this.sfu.device_id, + "device_id": this.foci[0].device_id, // XXX: What if an SFU gets restarted? "session_id": "sfu", "feeds": [], }; const onError = (e?): void => { - logger.warn(`Failed to place call to ${this.sfu!.user_id}!`, e); + logger.warn(`Failed to place call to ${this.foci[0].user_id}!`, e); this.emit( GroupCallEvent.Error, new GroupCallError( GroupCallErrorCode.PlaceCallFailed, - `Failed to place call to ${this.sfu!.user_id}.`, + `Failed to place call to ${this.foci[0].user_id}.`, ), ); }; @@ -393,11 +405,11 @@ export class GroupCall extends TypedEventEmitter< this.client, this.room.roomId, { - invitee: this.sfu.user_id, + invitee: this.foci[0].user_id, opponentDeviceId: opponentDevice.device_id, opponentSessionId: opponentDevice.session_id, groupCallId: this.groupCallId, - isSfu: true, + isFocus: true, }, ); if (!sfuCall) { @@ -418,6 +430,25 @@ export class GroupCall extends TypedEventEmitter< } } + private chooseFocus(): void { + // TODO: Go through all state and find best focus and try to use that + + // Try to find a focus of another user to use + let focusOfAnotherMember: IFocusInfo | undefined; + for (const event of this.getMemberStateEvents()) { + const focus = event.getContent() + ?.["m.calls"] ?.[0] + ?.["m.devices"]?.[0] + ?.["m.foci.active"]?.[0]; + if (focus) { + focusOfAnotherMember = focus; + break; + } + } + + this.foci = [focusOfAnotherMember ?? this.client.getFoci()[0]]; + } + private dispose() { if (this.localCallFeed) { this.removeUserMediaFeed(this.localCallFeed); @@ -673,7 +704,7 @@ export class GroupCall extends TypedEventEmitter< ); // TODO: handle errors - await Promise.all(this.calls.map(call => call.pushLocalFeed(call.isSfu + await Promise.all(this.calls.map(call => call.pushLocalFeed(call.isFocus ? this.localScreenshareFeed! : this.localScreenshareFeed!.clone(), ))); @@ -697,7 +728,7 @@ export class GroupCall extends TypedEventEmitter< this.client.getMediaHandler().stopScreensharingStream(this.localScreenshareFeed!.stream); // We have to remove the feed manually as MatrixCall has its clone, // so it won't be removed automatically - if (!this.sfu) { + if (!this.foci[0]) { this.removeScreenshareFeed(this.localScreenshareFeed!); } this.localScreenshareFeed = undefined; @@ -786,6 +817,8 @@ export class GroupCall extends TypedEventEmitter< "feeds": this.getLocalFeeds().map((feed) => ({ purpose: feed.purpose, })), + "m.foci.active": this.foci, + "m.foci.preferred": this.getPreferredFoci(), // TODO: Add data channels }, ], @@ -849,7 +882,10 @@ export class GroupCall extends TypedEventEmitter< } public onMemberStateChanged = async (event: MatrixEvent) => { - if (this.sfu) return; + if (this.foci[0]) { + // TODO: Check if someone switched to one of our preferred foci + return; + } // If we haven't entered the call yet, we don't care if (this.state !== GroupCallState.Entered) { @@ -1207,7 +1243,7 @@ export class GroupCall extends TypedEventEmitter< this.retryCallCounts.delete(getCallUserId(call)!); // if we're calling an SFU, subscribe to its feeds - if (call.isSfu) { + if (call.isFocus) { call.subscribeToSFU(); } } From a504a7c8409a259b9def9805c390a357927292f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 30 Nov 2022 19:21:39 +0100 Subject: [PATCH 081/137] Don't send member events until we're participating MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 634b63ca933..9da1e2477e9 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -807,8 +807,10 @@ export class GroupCall extends TypedEventEmitter< } } - private async sendMemberStateEvent(): Promise { - const send = () => this.updateMemberCallState({ + private async sendMemberStateEvent(): Promise { + if (!this.participants.includes(this.room.getMember(this.client.getUserId()!)!)) return null; + + const send = (): Promise => this.updateMemberCallState({ "m.call_id": this.groupCallId, "m.devices": [ { From 47c14c055dd5f5913e38f66aa2e906cc1d5ea446 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sun, 4 Dec 2022 12:50:43 +0100 Subject: [PATCH 082/137] Merge develop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- .eslintrc.js | 9 + .github/CODEOWNERS | 10 +- .github/workflows/docs-pr-netlify.yaml | 2 +- .github/workflows/pull_request.yaml | 2 +- .github/workflows/static_analysis.yml | 37 +- .github/workflows/tests.yml | 9 + CHANGELOG.md | 59 + README.md | 10 +- package.json | 14 +- spec/integ/matrix-client-crypto.spec.ts | 41 + spec/integ/matrix-client-methods.spec.ts | 51 +- spec/integ/matrix-client-syncing.spec.ts | 6 +- spec/integ/megolm-integ.spec.ts | 110 +- spec/integ/sliding-sync-sdk.spec.ts | 205 +++ spec/slowReporter.js | 100 ++ spec/test-utils/test-utils.ts | 24 +- spec/test-utils/webrtc.ts | 3 +- spec/unit/crypto.spec.ts | 45 + spec/unit/embedded.spec.ts | 4 +- spec/unit/notifications.spec.ts | 24 +- spec/unit/read-receipt.spec.ts | 17 + spec/unit/room-state.spec.ts | 2 +- spec/unit/webrtc/call.spec.ts | 33 +- spec/unit/webrtc/groupCall.spec.ts | 233 ++-- .../unit/webrtc/groupCallEventHandler.spec.ts | 65 +- src/ReEmitter.ts | 6 +- src/ToDeviceMessageQueue.ts | 2 +- src/client.ts | 138 +- src/content-helpers.ts | 13 +- src/crypto/CrossSigning.ts | 23 +- src/crypto/DeviceList.ts | 8 +- src/crypto/EncryptionSetup.ts | 10 +- src/crypto/OlmDevice.ts | 24 +- src/crypto/OutgoingRoomKeyRequestManager.ts | 4 +- src/crypto/RoomList.ts | 2 +- src/crypto/SecretStorage.ts | 4 +- src/crypto/algorithms/base.ts | 8 +- src/crypto/algorithms/megolm.ts | 46 +- src/crypto/backup.ts | 10 +- src/crypto/dehydration.ts | 4 +- src/crypto/deviceinfo.ts | 2 +- src/crypto/index.ts | 150 ++- src/crypto/olmlib.ts | 10 +- .../store/indexeddb-crypto-store-backend.ts | 96 +- src/crypto/store/indexeddb-crypto-store.ts | 18 +- src/crypto/store/localStorage-crypto-store.ts | 6 +- src/crypto/store/memory-crypto-store.ts | 2 +- src/crypto/verification/Base.ts | 8 +- src/crypto/verification/QRCode.ts | 10 +- src/crypto/verification/SAS.ts | 18 +- .../verification/request/InRoomChannel.ts | 2 +- .../verification/request/ToDeviceChannel.ts | 2 +- .../request/VerificationRequest.ts | 8 +- src/embedded.ts | 16 +- src/event-mapper.ts | 2 +- src/filter-component.ts | 2 +- src/filter.ts | 12 +- src/http-api/errors.ts | 8 +- src/http-api/fetch.ts | 2 +- src/http-api/index.ts | 6 +- src/http-api/utils.ts | 4 +- src/indexeddb-helpers.ts | 8 +- src/interactive-auth.ts | 6 +- src/logger.ts | 4 +- src/matrix.ts | 5 +- src/models/MSC3089TreeSpace.ts | 4 +- src/models/beacon.ts | 4 +- src/models/event-context.ts | 2 +- src/models/event-timeline-set.ts | 6 +- src/models/event-timeline.ts | 4 +- src/models/event.ts | 22 +- src/models/invites-ignorer.ts | 6 +- src/models/read-receipt.ts | 2 +- src/models/related-relations.ts | 4 +- src/models/relations-container.ts | 4 +- src/models/relations.ts | 12 +- src/models/room-member.ts | 4 +- src/models/room-state.ts | 8 +- src/models/room-summary.ts | 2 +- src/models/room.ts | 46 +- src/models/search-result.ts | 2 +- src/models/thread.ts | 26 +- src/models/user.ts | 2 +- src/pushprocessor.ts | 2 +- src/realtime-callbacks.ts | 2 +- src/rendezvous/MSC3906Rendezvous.ts | 4 +- src/rendezvous/RendezvousError.ts | 2 +- src/room-hierarchy.ts | 2 +- src/scheduler.ts | 8 +- src/sliding-sync-sdk.ts | 101 +- src/sliding-sync.ts | 24 +- src/store/indexeddb-local-backend.ts | 48 +- src/store/indexeddb-remote-backend.ts | 2 +- src/store/indexeddb-store-worker.ts | 2 +- src/store/indexeddb.ts | 4 +- src/store/memory.ts | 12 +- src/store/stub.ts | 18 +- src/sync-accumulator.ts | 2 +- src/sync.ts | 46 +- src/timeline-window.ts | 8 +- src/utils.ts | 21 +- src/webrtc/audioContext.ts | 2 +- src/webrtc/call.ts | 22 +- src/webrtc/callEventHandler.ts | 16 +- src/webrtc/callFeed.ts | 13 +- src/webrtc/groupCall.ts | 1184 +++++++++-------- src/webrtc/groupCallEventHandler.ts | 16 +- src/webrtc/mediaHandler.ts | 10 +- yarn.lock | 1122 ++++++++-------- 109 files changed, 2875 insertions(+), 1802 deletions(-) create mode 100644 spec/slowReporter.js diff --git a/.eslintrc.js b/.eslintrc.js index 918ae4365e2..3059df17f4d 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -85,5 +85,14 @@ module.exports = { // We use a `logger` intermediary module "no-console": "error", }, + }, { + files: [ + "spec/**/*.ts", + ], + rules: { + // We don't need super strict typing in test utilities + "@typescript-eslint/explicit-function-return-type": "off", + "@typescript-eslint/explicit-member-accessibility": "off", + }, }], }; diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f28b852a788..dd0bfc9ebfa 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,4 +1,6 @@ -* @matrix-org/element-web - -/src/webrtc @matrix-org/element-call-reviewers -/spec/*/webrtc @matrix-org/element-call-reviewers +* @matrix-org/element-web +/.github/workflows/** @matrix-org/element-web-app-team +/package.json @matrix-org/element-web-app-team +/yarn.lock @matrix-org/element-web-app-team +/src/webrtc @matrix-org/element-call-reviewers +/spec/*/webrtc @matrix-org/element-call-reviewers diff --git a/.github/workflows/docs-pr-netlify.yaml b/.github/workflows/docs-pr-netlify.yaml index 297986ac9e0..4e1b09c5b0f 100644 --- a/.github/workflows/docs-pr-netlify.yaml +++ b/.github/workflows/docs-pr-netlify.yaml @@ -14,7 +14,7 @@ jobs: # There's a 'download artifact' action, but it hasn't been updated for the workflow_run action # (https://github.com/actions/download-artifact/issues/60) so instead we get this mess: - name: 📥 Download artifact - uses: dawidd6/action-download-artifact@b12b127cf24433d14b4f93cee62f5465076ba82a # v2.24.1 + uses: dawidd6/action-download-artifact@e6e25ac3a2b93187502a8be1ef9e9603afc34925 # v2.24.2 with: workflow: static_analysis.yml run_id: ${{ github.event.workflow_run.id }} diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 940aa0ea239..d2a8857aa5f 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -42,7 +42,7 @@ jobs: if: github.event.action == 'opened' steps: - name: Check membership - uses: tspascoal/get-user-teams-membership@v1 + uses: tspascoal/get-user-teams-membership@v2 id: teams with: username: ${{ github.event.pull_request.user.login }} diff --git a/.github/workflows/static_analysis.yml b/.github/workflows/static_analysis.yml index 0054561f47d..5f612727815 100644 --- a/.github/workflows/static_analysis.yml +++ b/.github/workflows/static_analysis.yml @@ -66,9 +66,44 @@ jobs: run: "yarn run gendoc" - name: Upload Artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: docs path: _docs # We'll only use this in a workflow_run, then we're done with it retention-days: 1 + + tsc-strict: + name: Typescript Strict Error Checker + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + permissions: + pull-requests: read + checks: write + steps: + - uses: actions/checkout@v3 + + - name: Get diff lines + id: diff + uses: Equip-Collaboration/diff-line-numbers@v1.0.0 + with: + include: '["\\.tsx?$"]' + + - name: Detecting files changed + id: files + uses: futuratrepadeira/changed-files@v4.0.0 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + pattern: '^.*\.tsx?$' + + - uses: t3chguy/typescript-check-action@main + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + use-check: false + check-fail-mode: added + output-behaviour: annotate + ts-extra-args: '--noImplicitAny' + files-changed: ${{ steps.files.outputs.files_updated }} + files-added: ${{ steps.files.outputs.files_created }} + files-deleted: ${{ steps.files.outputs.files_deleted }} + line-numbers: ${{ steps.diff.outputs.lineNumbers }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 13b289e1ce7..c7c7da7e730 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,7 +36,16 @@ jobs: id: cpu-cores uses: SimenB/github-actions-cpu-cores@v1 + - name: Run tests with coverage and metrics + if: github.ref == 'refs/heads/develop' + run: | + yarn coverage --ci --reporters github-actions '--reporters=/spec/slowReporter.js' --max-workers ${{ steps.cpu-cores.outputs.count }} ./spec/${{ matrix.specs }} + mv coverage/lcov.info coverage/${{ matrix.node }}-${{ matrix.specs }}.lcov.info + env: + JEST_SONAR_UNIQUE_OUTPUT_NAME: true + - name: Run tests with coverage + if: github.ref != 'refs/heads/develop' run: | yarn coverage --ci --reporters github-actions --max-workers ${{ steps.cpu-cores.outputs.count }} ./spec/${{ matrix.specs }} mv coverage/lcov.info coverage/${{ matrix.node }}-${{ matrix.specs }}.lcov.info diff --git a/CHANGELOG.md b/CHANGELOG.md index 9085928c690..c835a0105c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,62 @@ +Changes in [21.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v21.2.0) (2022-11-22) +================================================================================================== + +## ✨ Features + * Make calls go back to 'connecting' state when media lost ([\#2880](https://github.com/matrix-org/matrix-js-sdk/pull/2880)). + * Add ability to send unthreaded receipt ([\#2878](https://github.com/matrix-org/matrix-js-sdk/pull/2878)). + * Add way to abort search requests ([\#2877](https://github.com/matrix-org/matrix-js-sdk/pull/2877)). + * sliding sync: add custom room subscriptions support ([\#2834](https://github.com/matrix-org/matrix-js-sdk/pull/2834)). + * webrtc: add advanced audio settings ([\#2434](https://github.com/matrix-org/matrix-js-sdk/pull/2434)). Contributed by @MrAnno. + * Add support for group calls using MSC3401 ([\#2553](https://github.com/matrix-org/matrix-js-sdk/pull/2553)). + * Make the js-sdk conform to tsc --strict ([\#2835](https://github.com/matrix-org/matrix-js-sdk/pull/2835)). Fixes #2112 #2116 and #2124. + * Let leave requests outlive the window ([\#2815](https://github.com/matrix-org/matrix-js-sdk/pull/2815)). Fixes vector-im/element-call#639. + * Add event and message capabilities to RoomWidgetClient ([\#2797](https://github.com/matrix-org/matrix-js-sdk/pull/2797)). + * Misc fixes for group call widgets ([\#2657](https://github.com/matrix-org/matrix-js-sdk/pull/2657)). + * Support nested Matrix clients via the widget API ([\#2473](https://github.com/matrix-org/matrix-js-sdk/pull/2473)). + * Set max average bitrate on PTT calls ([\#2499](https://github.com/matrix-org/matrix-js-sdk/pull/2499)). Fixes vector-im/element-call#440. + * Add config option for e2e group call signalling ([\#2492](https://github.com/matrix-org/matrix-js-sdk/pull/2492)). + * Enable DTX on audio tracks in calls ([\#2482](https://github.com/matrix-org/matrix-js-sdk/pull/2482)). + * Don't ignore call member events with a distant future expiration date ([\#2466](https://github.com/matrix-org/matrix-js-sdk/pull/2466)). + * Expire call member state events after 1 hour ([\#2446](https://github.com/matrix-org/matrix-js-sdk/pull/2446)). + * Emit unknown device errors for group call participants without e2e ([\#2447](https://github.com/matrix-org/matrix-js-sdk/pull/2447)). + * Mute disconnected peers in PTT mode ([\#2421](https://github.com/matrix-org/matrix-js-sdk/pull/2421)). + * Add support for sending encrypted to-device events with OLM ([\#2322](https://github.com/matrix-org/matrix-js-sdk/pull/2322)). Contributed by @robertlong. + * Support for PTT group call mode ([\#2338](https://github.com/matrix-org/matrix-js-sdk/pull/2338)). + +## 🐛 Bug Fixes + * Fix registration add phone number not working ([\#2876](https://github.com/matrix-org/matrix-js-sdk/pull/2876)). Contributed by @bagvand. + * Use an underride rule for Element Call notifications ([\#2873](https://github.com/matrix-org/matrix-js-sdk/pull/2873)). Fixes vector-im/element-web#23691. + * Fixes unwanted highlight notifications with encrypted threads ([\#2862](https://github.com/matrix-org/matrix-js-sdk/pull/2862)). + * Extra insurance that we don't mix events in the wrong timelines - v2 ([\#2856](https://github.com/matrix-org/matrix-js-sdk/pull/2856)). Contributed by @MadLittleMods. + * Hide pending events in thread timelines ([\#2843](https://github.com/matrix-org/matrix-js-sdk/pull/2843)). Fixes vector-im/element-web#23684. + * Fix pagination token tracking for mixed room timelines ([\#2855](https://github.com/matrix-org/matrix-js-sdk/pull/2855)). Fixes vector-im/element-web#23695. + * Extra insurance that we don't mix events in the wrong timelines ([\#2848](https://github.com/matrix-org/matrix-js-sdk/pull/2848)). Contributed by @MadLittleMods. + * Do not freeze state in `initialiseState()` ([\#2846](https://github.com/matrix-org/matrix-js-sdk/pull/2846)). + * Don't remove our own member for a split second when entering a call ([\#2844](https://github.com/matrix-org/matrix-js-sdk/pull/2844)). + * Resolve races between `initLocalCallFeed` and `leave` ([\#2826](https://github.com/matrix-org/matrix-js-sdk/pull/2826)). + * Add throwOnFail to groupCall.setScreensharingEnabled ([\#2787](https://github.com/matrix-org/matrix-js-sdk/pull/2787)). + * Fix connectivity regressions ([\#2780](https://github.com/matrix-org/matrix-js-sdk/pull/2780)). + * Fix screenshare failing after several attempts ([\#2771](https://github.com/matrix-org/matrix-js-sdk/pull/2771)). Fixes vector-im/element-call#625. + * Don't block muting/unmuting on network requests ([\#2754](https://github.com/matrix-org/matrix-js-sdk/pull/2754)). Fixes vector-im/element-call#592. + * Fix ICE restarts ([\#2702](https://github.com/matrix-org/matrix-js-sdk/pull/2702)). + * Target widget actions at a specific room ([\#2670](https://github.com/matrix-org/matrix-js-sdk/pull/2670)). + * Add tests for ice candidate sending ([\#2674](https://github.com/matrix-org/matrix-js-sdk/pull/2674)). + * Prevent exception when muting ([\#2667](https://github.com/matrix-org/matrix-js-sdk/pull/2667)). Fixes vector-im/element-call#578. + * Fix race in creating calls ([\#2662](https://github.com/matrix-org/matrix-js-sdk/pull/2662)). + * Add client.waitUntilRoomReadyForGroupCalls() ([\#2641](https://github.com/matrix-org/matrix-js-sdk/pull/2641)). + * Wait for client to start syncing before making group calls ([\#2632](https://github.com/matrix-org/matrix-js-sdk/pull/2632)). Fixes #2589. + * Add GroupCallEventHandlerEvent.Room ([\#2631](https://github.com/matrix-org/matrix-js-sdk/pull/2631)). + * Add missing events from reemitter to GroupCall ([\#2527](https://github.com/matrix-org/matrix-js-sdk/pull/2527)). Contributed by @toger5. + * Prevent double mute status changed events ([\#2502](https://github.com/matrix-org/matrix-js-sdk/pull/2502)). + * Don't mute the remote side immediately in PTT calls ([\#2487](https://github.com/matrix-org/matrix-js-sdk/pull/2487)). Fixes vector-im/element-call#425. + * Fix some MatrixCall leaks and use a shared AudioContext ([\#2484](https://github.com/matrix-org/matrix-js-sdk/pull/2484)). Fixes vector-im/element-call#412. + * Don't block muting on determining whether the device exists ([\#2461](https://github.com/matrix-org/matrix-js-sdk/pull/2461)). + * Only clone streams on Safari ([\#2450](https://github.com/matrix-org/matrix-js-sdk/pull/2450)). Fixes vector-im/element-call#267. + * Set PTT mode on call correctly ([\#2445](https://github.com/matrix-org/matrix-js-sdk/pull/2445)). Fixes vector-im/element-call#382. + * Wait for mute event to send in PTT mode ([\#2401](https://github.com/matrix-org/matrix-js-sdk/pull/2401)). + * Handle other members having no e2e keys ([\#2383](https://github.com/matrix-org/matrix-js-sdk/pull/2383)). Fixes vector-im/element-call#338. + * Fix races when muting/unmuting ([\#2370](https://github.com/matrix-org/matrix-js-sdk/pull/2370)). + Changes in [21.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v21.1.0) (2022-11-08) ================================================================================================== diff --git a/README.md b/README.md index 5e7de61a17f..19592b0ea8f 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,14 @@ Matrix Javascript SDK ===================== -This is the [Matrix](https://matrix.org) Client-Server r0 SDK for -JavaScript. This SDK can be run in a browser or in Node.js. +This is the [Matrix](https://matrix.org) Client-Server SDK for JavaScript and TypeScript. This SDK can be run in a +browser or in Node.js. + +The Matrix specification is constantly evolving - while this SDK aims for maximum backwards compatibility, it only +guarantees that a feature will be supported for at least 4 spec releases. For example, if a feature the js-sdk supports +is removed in v1.4 then the feature is *eligible* for removal from the SDK when v1.8 is released. This SDK has no +guarantee on implementing all features of any particular spec release, currently. This can mean that the SDK will call +endpoints from before Matrix 1.1, for example. Quickstart ========== diff --git a/package.json b/package.json index 17d6ef45008..969c71cba4a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "matrix-js-sdk", - "version": "21.1.0", + "version": "21.2.0", "description": "Matrix Client-Server SDK for Javascript", "engines": { "node": ">=16.0.0" @@ -85,7 +85,7 @@ "@types/content-type": "^1.1.5", "@types/domexception": "^4.0.0", "@types/jest": "^29.0.0", - "@types/node": "16", + "@types/node": "18", "@typescript-eslint/eslint-plugin": "^5.6.0", "@typescript-eslint/parser": "^5.6.0", "allchange": "^1.0.6", @@ -93,18 +93,18 @@ "babelify": "^10.0.0", "better-docs": "^2.4.0-beta.9", "browserify": "^17.0.0", - "docdash": "^1.2.0", + "docdash": "^2.0.0", "domexception": "^4.0.0", - "eslint": "8.26.0", + "eslint": "8.28.0", "eslint-config-google": "^0.14.0", "eslint-import-resolver-typescript": "^3.5.1", "eslint-plugin-import": "^2.26.0", - "eslint-plugin-matrix-org": "^0.7.0", - "eslint-plugin-unicorn": "^44.0.2", + "eslint-plugin-matrix-org": "^0.8.0", + "eslint-plugin-unicorn": "^45.0.0", "exorcist": "^2.0.0", "fake-indexeddb": "^4.0.0", "jest": "^29.0.0", - "jest-environment-jsdom": "^28.1.3", + "jest-environment-jsdom": "^29.0.0", "jest-localstorage-mock": "^2.4.6", "jest-mock": "^29.0.0", "matrix-mock-request": "^2.5.0", diff --git a/spec/integ/matrix-client-crypto.spec.ts b/spec/integ/matrix-client-crypto.spec.ts index 38de34aa59d..847bb8528ae 100644 --- a/spec/integ/matrix-client-crypto.spec.ts +++ b/spec/integ/matrix-client-crypto.spec.ts @@ -679,4 +679,45 @@ describe("MatrixClient crypto", () => { }); await httpBackend.flushAllExpected(); }); + + it("Checks for outgoing room key requests for a given event's session", async () => { + const eventA0 = new MatrixEvent({ + sender: "@bob:example.com", + room_id: "!someroom", + content: { + algorithm: 'm.megolm.v1.aes-sha2', + session_id: "sessionid", + sender_key: "senderkey", + }, + }); + const eventA1 = new MatrixEvent({ + sender: "@bob:example.com", + room_id: "!someroom", + content: { + algorithm: 'm.megolm.v1.aes-sha2', + session_id: "sessionid", + sender_key: "senderkey", + }, + }); + const eventB = new MatrixEvent({ + sender: "@bob:example.com", + room_id: "!someroom", + content: { + algorithm: 'm.megolm.v1.aes-sha2', + session_id: "othersessionid", + sender_key: "senderkey", + }, + }); + const nonEncryptedEvent = new MatrixEvent({ + sender: "@bob:example.com", + room_id: "!someroom", + content: {}, + }); + + aliTestClient.client.crypto?.onSyncCompleted({}); + await aliTestClient.client.cancelAndResendEventRoomKeyRequest(eventA0); + expect(await aliTestClient.client.getOutgoingRoomKeyRequest(eventA1)).not.toBeNull(); + expect(await aliTestClient.client.getOutgoingRoomKeyRequest(eventB)).toBeNull(); + expect(await aliTestClient.client.getOutgoingRoomKeyRequest(nonEncryptedEvent)).toBeNull(); + }); }); diff --git a/spec/integ/matrix-client-methods.spec.ts b/spec/integ/matrix-client-methods.spec.ts index 8e721193726..f2ff9e6df77 100644 --- a/spec/integ/matrix-client-methods.spec.ts +++ b/spec/integ/matrix-client-methods.spec.ts @@ -173,7 +173,9 @@ describe("MatrixClient", function() { signatures: {}, }; - httpBackend!.when("POST", inviteSignUrl).respond(200, signature); + httpBackend!.when("POST", inviteSignUrl).check(request => { + expect(request.queryParams?.mxid).toEqual(client!.getUserId()); + }).respond(200, signature); httpBackend!.when("POST", "/join/" + encodeURIComponent(roomId)).check(request => { expect(request.data.third_party_signed).toEqual(signature); }).respond(200, { room_id: roomId }); @@ -1335,27 +1337,38 @@ describe("MatrixClient", function() { }); }); - describe("registerWithIdentityServer", () => { - it("should pass data to POST request", async () => { - const token = { - access_token: "access_token", - token_type: "Bearer", - matrix_server_name: "server_name", - expires_in: 12345, - }; + describe("setPowerLevel", () => { + it.each([ + { + userId: "alice@localhost", + expectation: { + "alice@localhost": 100, + }, + }, + { + userId: ["alice@localhost", "bob@localhost"], + expectation: { + "alice@localhost": 100, + "bob@localhost": 100, + }, + }, + ])("should modify power levels of $userId correctly", async ({ userId, expectation }) => { + const event = { + getType: () => "m.room.power_levels", + getContent: () => ({ + users: { + "alice@localhost": 50, + }, + }), + } as MatrixEvent; - httpBackend!.when("POST", "/account/register").check(req => { - expect(req.data).toStrictEqual(token); - }).respond(200, { - access_token: "at", - token: "tt", - }); + httpBackend!.when("PUT", "/state/m.room.power_levels").check(req => { + expect(req.data.users).toStrictEqual(expectation); + }).respond(200, {}); - const prom = client!.registerWithIdentityServer(token); + const prom = client!.setPowerLevel("!room_id:server", userId, 100, event); await httpBackend!.flushAllExpected(); - const resp = await prom; - expect(resp.access_token).toBe("at"); - expect(resp.token).toBe("tt"); + await prom; }); }); }); diff --git a/spec/integ/matrix-client-syncing.spec.ts b/spec/integ/matrix-client-syncing.spec.ts index 216fbdbb1f6..8c3969bee7d 100644 --- a/spec/integ/matrix-client-syncing.spec.ts +++ b/spec/integ/matrix-client-syncing.spec.ts @@ -709,11 +709,11 @@ describe("MatrixClient syncing", () => { const room = client!.getRoom(roomOne)!; const stateAtStart = room.getLiveTimeline().getState(EventTimeline.BACKWARDS)!; const startRoomNameEvent = stateAtStart.getStateEvents('m.room.name', ''); - expect(startRoomNameEvent.getContent().name).toEqual('Old room name'); + expect(startRoomNameEvent!.getContent().name).toEqual('Old room name'); const stateAtEnd = room.getLiveTimeline().getState(EventTimeline.FORWARDS)!; const endRoomNameEvent = stateAtEnd.getStateEvents('m.room.name', ''); - expect(endRoomNameEvent.getContent().name).toEqual('A new room name'); + expect(endRoomNameEvent!.getContent().name).toEqual('A new room name'); }); }); @@ -1599,7 +1599,7 @@ describe("MatrixClient syncing", () => { expect(room.roomId).toBe(roomOne); expect(room.getMyMembership()).toBe("leave"); expect(room.name).toBe("Room Name"); - expect(room.currentState.getStateEvents("m.room.name", "").getId()).toBe("$eventId"); + expect(room.currentState.getStateEvents("m.room.name", "")?.getId()).toBe("$eventId"); expect(room.timeline[0].getContent().body).toBe("Message 1"); expect(room.timeline[1].getContent().body).toBe("Message 2"); client?.stopPeeking(); diff --git a/spec/integ/megolm-integ.spec.ts b/spec/integ/megolm-integ.spec.ts index a4891b702f6..89c06426617 100644 --- a/spec/integ/megolm-integ.spec.ts +++ b/spec/integ/megolm-integ.spec.ts @@ -21,16 +21,19 @@ import * as testUtils from "../test-utils/test-utils"; import { TestClient } from "../TestClient"; import { logger } from "../../src/logger"; import { + IClaimOTKsResult, IContent, + IDownloadKeyResult, IEvent, - IClaimOTKsResult, IJoinedRoom, + IndexedDBCryptoStore, ISyncResponse, - IDownloadKeyResult, + IUploadKeysRequest, MatrixEvent, MatrixEventEvent, - IndexedDBCryptoStore, Room, + RoomMember, + RoomStateEvent, } from "../../src/matrix"; import { IDeviceKeys } from "../../src/crypto/dehydration"; import { DeviceInfo } from "../../src/crypto/deviceinfo"; @@ -327,7 +330,9 @@ describe("megolm", () => { const room = aliceTestClient.client.getRoom(ROOM_ID)!; const event = room.getLiveTimeline().getEvents()[0]; expect(event.isEncrypted()).toBe(true); - const decryptedEvent = await testUtils.awaitDecryption(event); + + // it probably won't be decrypted yet, because it takes a while to process the olm keys + const decryptedEvent = await testUtils.awaitDecryption(event, { waitOnDecryptionFailure: true }); expect(decryptedEvent.getContent().body).toEqual('42'); }); @@ -873,7 +878,12 @@ describe("megolm", () => { const room = aliceTestClient.client.getRoom(ROOM_ID)!; await room.decryptCriticalEvents(); - expect(room.getLiveTimeline().getEvents()[0].getContent().body).toEqual('42'); + + // it probably won't be decrypted yet, because it takes a while to process the olm keys + const decryptedEvent = await testUtils.awaitDecryption( + room.getLiveTimeline().getEvents()[0], { waitOnDecryptionFailure: true }, + ); + expect(decryptedEvent.getContent().body).toEqual('42'); const exported = await aliceTestClient.client.exportRoomKeys(); @@ -1012,7 +1022,9 @@ describe("megolm", () => { const room = aliceTestClient.client.getRoom(ROOM_ID)!; const event = room.getLiveTimeline().getEvents()[0]; expect(event.isEncrypted()).toBe(true); - const decryptedEvent = await testUtils.awaitDecryption(event); + + // it probably won't be decrypted yet, because it takes a while to process the olm keys + const decryptedEvent = await testUtils.awaitDecryption(event, { waitOnDecryptionFailure: true }); expect(decryptedEvent.getRoomId()).toEqual(ROOM_ID); expect(decryptedEvent.getContent()).toEqual({}); expect(decryptedEvent.getClearContent()).toBeUndefined(); @@ -1364,4 +1376,90 @@ describe("megolm", () => { await beccaTestClient.stop(); }); + + it("allows sending an encrypted event as soon as room state arrives", async () => { + /* Empirically, clients expect to be able to send encrypted events as soon as the + * RoomStateEvent.NewMember notification is emitted, so test that works correctly. + */ + const testRoomId = "!testRoom:id"; + await aliceTestClient.start(); + + aliceTestClient.httpBackend.when("POST", "/keys/query") + .respond(200, function(_path, content: IUploadKeysRequest) { + return { device_keys: {} }; + }); + + /* Alice makes the /createRoom call */ + aliceTestClient.httpBackend.when("POST", "/createRoom") + .respond(200, { room_id: testRoomId }); + await Promise.all([ + aliceTestClient.client.createRoom({ + initial_state: [{ + type: 'm.room.encryption', + state_key: '', + content: { algorithm: 'm.megolm.v1.aes-sha2' }, + }], + }), + aliceTestClient.httpBackend.flushAllExpected(), + ]); + + /* The sync arrives in two parts; first the m.room.create... */ + aliceTestClient.httpBackend.when("GET", "/sync").respond(200, { + rooms: { join: { + [testRoomId]: { + timeline: { events: [ + { + type: 'm.room.create', + state_key: '', + event_id: "$create", + }, + { + type: 'm.room.member', + state_key: aliceTestClient.getUserId(), + content: { membership: "join" }, + event_id: "$alijoin", + }, + ] }, + }, + } }, + }); + await aliceTestClient.flushSync(); + + // ... and then the e2e event and an invite ... + aliceTestClient.httpBackend.when("GET", "/sync").respond(200, { + rooms: { join: { + [testRoomId]: { + timeline: { events: [ + { + type: 'm.room.encryption', + state_key: '', + content: { algorithm: 'm.megolm.v1.aes-sha2' }, + event_id: "$e2e", + }, + { + type: 'm.room.member', + state_key: "@other:user", + content: { membership: "invite" }, + event_id: "$otherinvite", + }, + ] }, + }, + } }, + }); + + // as soon as the roomMember arrives, try to send a message + aliceTestClient.client.on(RoomStateEvent.NewMember, (_e, _s, member: RoomMember) => { + if (member.userId == "@other:user") { + aliceTestClient.client.sendMessage(testRoomId, { msgtype: "m.text", body: "Hello, World" }); + } + }); + + // flush the sync and wait for the /send/ request. + aliceTestClient.httpBackend.when("PUT", "/send/m.room.encrypted/") + .respond(200, (_path, _content) => ({ event_id: "asdfgh" })); + await Promise.all([ + aliceTestClient.flushSync(), + aliceTestClient.httpBackend.flush("/send/m.room.encrypted/", 1), + ]); + }); }); diff --git a/spec/integ/sliding-sync-sdk.spec.ts b/spec/integ/sliding-sync-sdk.spec.ts index f09a9a3316c..72a7eeaa2e9 100644 --- a/spec/integ/sliding-sync-sdk.spec.ts +++ b/spec/integ/sliding-sync-sdk.spec.ts @@ -542,6 +542,7 @@ describe("SlidingSyncSdk", () => { describe("ExtensionE2EE", () => { let ext: Extension; + beforeAll(async () => { await setupClient({ withCrypto: true, @@ -551,18 +552,21 @@ describe("SlidingSyncSdk", () => { await hasSynced; ext = findExtension("e2ee"); }); + afterAll(async () => { // needed else we do some async operations in the background which can cause Jest to whine: // "Cannot log after tests are done. Did you forget to wait for something async in your test?" // Attempted to log "Saving device tracking data null"." client!.crypto!.stop(); }); + it("gets enabled on the initial request only", () => { expect(ext.onRequest(true)).toEqual({ enabled: true, }); expect(ext.onRequest(false)).toEqual(undefined); }); + it("can update device lists", () => { ext.onResponse({ device_lists: { @@ -572,6 +576,7 @@ describe("SlidingSyncSdk", () => { }); // TODO: more assertions? }); + it("can update OTK counts", () => { client!.crypto!.updateOneTimeKeyCount = jest.fn(); ext.onResponse({ @@ -588,6 +593,7 @@ describe("SlidingSyncSdk", () => { }); expect(client!.crypto!.updateOneTimeKeyCount).toHaveBeenCalledWith(0); }); + it("can update fallback keys", () => { ext.onResponse({ device_unused_fallback_key_types: ["signed_curve25519"], @@ -599,8 +605,10 @@ describe("SlidingSyncSdk", () => { expect(client!.crypto!.getNeedsNewFallback()).toEqual(true); }); }); + describe("ExtensionAccountData", () => { let ext: Extension; + beforeAll(async () => { await setupClient(); const hasSynced = sdk!.sync(); @@ -608,12 +616,14 @@ describe("SlidingSyncSdk", () => { await hasSynced; ext = findExtension("account_data"); }); + it("gets enabled on the initial request only", () => { expect(ext.onRequest(true)).toEqual({ enabled: true, }); expect(ext.onRequest(false)).toEqual(undefined); }); + it("processes global account data", async () => { const globalType = "global_test"; const globalContent = { @@ -633,6 +643,7 @@ describe("SlidingSyncSdk", () => { expect(globalData).toBeDefined(); expect(globalData.getContent()).toEqual(globalContent); }); + it("processes rooms account data", async () => { const roomId = "!room:id"; mockSlidingSync!.emit(SlidingSyncEvent.RoomData, roomId, { @@ -667,6 +678,7 @@ describe("SlidingSyncSdk", () => { expect(event).toBeDefined(); expect(event.getContent()).toEqual(roomContent); }); + it("doesn't crash for unknown room account data", async () => { const unknownRoomId = "!unknown:id"; const roomType = "tester"; @@ -686,6 +698,7 @@ describe("SlidingSyncSdk", () => { expect(room).toBeNull(); expect(client!.getAccountData(roomType)).toBeUndefined(); }); + it("can update push rules via account data", async () => { const roomId = "!foo:bar"; const pushRulesContent: IPushRules = { @@ -718,8 +731,10 @@ describe("SlidingSyncSdk", () => { expect(pushRule).toEqual(pushRulesContent.global[PushRuleKind.RoomSpecific]![0]); }); }); + describe("ExtensionToDevice", () => { let ext: Extension; + beforeAll(async () => { await setupClient(); const hasSynced = sdk!.sync(); @@ -727,12 +742,14 @@ describe("SlidingSyncSdk", () => { await hasSynced; ext = findExtension("to_device"); }); + it("gets enabled with a limit on the initial request only", () => { const reqJson: any = ext.onRequest(true); expect(reqJson.enabled).toEqual(true); expect(reqJson.limit).toBeGreaterThan(0); expect(reqJson.since).toBeUndefined(); }); + it("updates the since value", async () => { ext.onResponse({ next_batch: "12345", @@ -742,12 +759,14 @@ describe("SlidingSyncSdk", () => { since: "12345", }); }); + it("can handle missing fields", async () => { ext.onResponse({ next_batch: "23456", // no events array }); }); + it("emits to-device events on the client", async () => { const toDeviceType = "custom_test"; const toDeviceContent = { @@ -770,6 +789,7 @@ describe("SlidingSyncSdk", () => { }); expect(called).toBe(true); }); + it("can cancel key verification requests", async () => { const seen: Record = {}; client!.on(ClientEvent.ToDeviceEvent, (ev) => { @@ -809,4 +829,189 @@ describe("SlidingSyncSdk", () => { }); }); }); + + describe("ExtensionTyping", () => { + let ext: Extension; + + beforeAll(async () => { + await setupClient(); + const hasSynced = sdk!.sync(); + await httpBackend!.flushAllExpected(); + await hasSynced; + ext = findExtension("typing"); + }); + + it("gets enabled on the initial request only", () => { + expect(ext.onRequest(true)).toEqual({ + enabled: true, + }); + expect(ext.onRequest(false)).toEqual(undefined); + }); + + it("processes typing notifications", async () => { + const roomId = "!room:id"; + mockSlidingSync!.emit(SlidingSyncEvent.RoomData, roomId, { + name: "Room with typing", + required_state: [], + timeline: [ + mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""), + mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId), + mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""), + mkOwnEvent(EventType.RoomMessage, { body: "hello" }), + ], + initial: true, + }); + const room = client!.getRoom(roomId)!; + expect(room).toBeDefined(); + expect(room.getMember(selfUserId)?.typing).toEqual(false); + ext.onResponse({ + rooms: { + [roomId]: { + type: EventType.Typing, + content: { + user_ids: [selfUserId], + }, + }, + }, + }); + expect(room.getMember(selfUserId)?.typing).toEqual(true); + ext.onResponse({ + rooms: { + [roomId]: { + type: EventType.Typing, + content: { + user_ids: [], + }, + }, + }, + }); + expect(room.getMember(selfUserId)?.typing).toEqual(false); + }); + + it("gracefully handles missing rooms and members when typing", async () => { + const roomId = "!room:id"; + mockSlidingSync!.emit(SlidingSyncEvent.RoomData, roomId, { + name: "Room with typing", + required_state: [], + timeline: [ + mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""), + mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId), + mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""), + mkOwnEvent(EventType.RoomMessage, { body: "hello" }), + ], + initial: true, + }); + const room = client!.getRoom(roomId)!; + expect(room).toBeDefined(); + expect(room.getMember(selfUserId)?.typing).toEqual(false); + ext.onResponse({ + rooms: { + [roomId]: { + type: EventType.Typing, + content: { + user_ids: ["@someone:else"], + }, + }, + }, + }); + expect(room.getMember(selfUserId)?.typing).toEqual(false); + ext.onResponse({ + rooms: { + "!something:else": { + type: EventType.Typing, + content: { + user_ids: [selfUserId], + }, + }, + }, + }); + expect(room.getMember(selfUserId)?.typing).toEqual(false); + }); + }); + + describe("ExtensionReceipts", () => { + let ext: Extension; + + const generateReceiptResponse = ( + userId: string, roomId: string, eventId: string, recType: string, ts: number, + ) => { + return { + rooms: { + [roomId]: { + type: EventType.Receipt, + content: { + [eventId]: { + [recType]: { + [userId]: { + ts: ts, + }, + }, + }, + }, + }, + }, + }; + }; + + beforeAll(async () => { + await setupClient(); + const hasSynced = sdk!.sync(); + await httpBackend!.flushAllExpected(); + await hasSynced; + ext = findExtension("receipts"); + }); + + it("gets enabled on the initial request only", () => { + expect(ext.onRequest(true)).toEqual({ + enabled: true, + }); + expect(ext.onRequest(false)).toEqual(undefined); + }); + + it("processes receipts", async () => { + const roomId = "!room:id"; + const alice = "@alice:alice"; + const lastEvent = mkOwnEvent(EventType.RoomMessage, { body: "hello" }); + mockSlidingSync!.emit(SlidingSyncEvent.RoomData, roomId, { + name: "Room with receipts", + required_state: [], + timeline: [ + mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""), + mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId), + mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""), + { + type: EventType.RoomMember, + state_key: alice, + content: { membership: "join" }, + sender: alice, + origin_server_ts: Date.now(), + event_id: "$alice", + }, + lastEvent, + ], + initial: true, + }); + const room = client!.getRoom(roomId)!; + expect(room).toBeDefined(); + expect(room.getReadReceiptForUserId(alice, true)).toBeNull(); + ext.onResponse( + generateReceiptResponse(alice, roomId, lastEvent.event_id, "m.read", 1234567), + ); + const receipt = room.getReadReceiptForUserId(alice); + expect(receipt).toBeDefined(); + expect(receipt?.eventId).toEqual(lastEvent.event_id); + expect(receipt?.data.ts).toEqual(1234567); + expect(receipt?.data.thread_id).toBeFalsy(); + }); + + it("gracefully handles missing rooms when receiving receipts", async () => { + const roomId = "!room:id"; + const alice = "@alice:alice"; + const eventId = "$something"; + ext.onResponse( + generateReceiptResponse(alice, roomId, eventId, "m.read", 1234567), + ); + // we expect it not to crash + }); + }); }); diff --git a/spec/slowReporter.js b/spec/slowReporter.js new file mode 100644 index 00000000000..3a653ffda28 --- /dev/null +++ b/spec/slowReporter.js @@ -0,0 +1,100 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* eslint-disable no-console */ + +class JestSlowTestReporter { + constructor(globalConfig, options) { + this._globalConfig = globalConfig; + this._options = options; + this._slowTests = []; + this._slowTestSuites = []; + } + + onRunComplete() { + const displayResult = (result, isTestSuite) => { + if (!isTestSuite) console.log(); + + result.sort((a, b) => b.duration - a.duration); + const rootPathRegex = new RegExp(`^${process.cwd()}`); + const slowestTests = result.slice(0, this._options.numTests || 10); + const slowTestTime = this._slowTestTime(slowestTests); + const allTestTime = this._allTestTime(result); + const percentTime = (slowTestTime / allTestTime) * 100; + + if (isTestSuite) { + console.log( + `Top ${slowestTests.length} slowest test suites (${slowTestTime / 1000} seconds,` + + ` ${percentTime.toFixed(1)}% of total time):`, + ); + } else { + console.log( + `Top ${slowestTests.length} slowest tests (${slowTestTime / 1000} seconds,` + + ` ${percentTime.toFixed(1)}% of total time):`, + ); + } + + for (let i = 0; i < slowestTests.length; i++) { + const duration = slowestTests[i].duration; + const filePath = slowestTests[i].filePath.replace(rootPathRegex, '.'); + + if (isTestSuite) { + console.log(` ${duration / 1000} seconds ${filePath}`); + } else { + const fullName = slowestTests[i].fullName; + console.log(` ${fullName}`); + console.log(` ${duration / 1000} seconds ${filePath}`); + } + } + console.log(); + }; + + displayResult(this._slowTests); + displayResult(this._slowTestSuites, true); + } + + onTestResult(test, testResult) { + this._slowTestSuites.push({ + duration: testResult.perfStats.runtime, + filePath: testResult.testFilePath, + }); + for (let i = 0; i < testResult.testResults.length; i++) { + this._slowTests.push({ + duration: testResult.testResults[i].duration, + fullName: testResult.testResults[i].fullName, + filePath: testResult.testFilePath, + }); + } + } + + _slowTestTime(slowestTests) { + let slowTestTime = 0; + for (let i = 0; i < slowestTests.length; i++) { + slowTestTime += slowestTests[i].duration; + } + return slowTestTime; + } + + _allTestTime(result) { + let allTestTime = 0; + for (let i = 0; i < result.length; i++) { + allTestTime += result[i].duration; + } + return allTestTime; + } +} + +module.exports = JestSlowTestReporter; diff --git a/spec/test-utils/test-utils.ts b/spec/test-utils/test-utils.ts index af87ebbe64f..21ae4d18bad 100644 --- a/spec/test-utils/test-utils.ts +++ b/spec/test-utils/test-utils.ts @@ -362,22 +362,28 @@ export class MockStorageApi { * @param {MatrixEvent} event * @returns {Promise} promise which resolves (to `event`) when the event has been decrypted */ -export async function awaitDecryption(event: MatrixEvent): Promise { +export async function awaitDecryption( + event: MatrixEvent, { waitOnDecryptionFailure = false } = {}, +): Promise { // An event is not always decrypted ahead of time // getClearContent is a good signal to know whether an event has been decrypted // already if (event.getClearContent() !== null) { - return event; + if (waitOnDecryptionFailure && event.isDecryptionFailure()) { + logger.log(`${Date.now()} event ${event.getId()} got decryption error; waiting`); + } else { + return event; + } } else { - logger.log(`${Date.now()} event ${event.getId()} is being decrypted; waiting`); + logger.log(`${Date.now()} event ${event.getId()} is not yet decrypted; waiting`); + } - return new Promise((resolve) => { - event.once(MatrixEventEvent.Decrypted, (ev) => { - logger.log(`${Date.now()} event ${event.getId()} now decrypted`); - resolve(ev); - }); + return new Promise((resolve) => { + event.once(MatrixEventEvent.Decrypted, (ev) => { + logger.log(`${Date.now()} event ${event.getId()} now decrypted`); + resolve(ev); }); - } + }); } export const emitPromise = (e: EventEmitter, k: string): Promise => new Promise(r => e.once(k, r)); diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index 66110701139..8d9423796a3 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -360,7 +360,7 @@ export class MockMediaDevices { Promise.resolve(new MockMediaStream("local_stream").typed()), ); - public getDisplayMedia = jest.fn, [DisplayMediaStreamConstraints]>().mockReturnValue( + public getDisplayMedia = jest.fn, [MediaStreamConstraints]>().mockReturnValue( Promise.resolve(new MockMediaStream("local_display_stream").typed()), ); @@ -431,6 +431,7 @@ export class MockCallMatrixClient extends TypedEventEmitter { + let mockClient: MatrixClient; + let mockRoomList: RoomList; + let clientStore: IStore; + let crypto: Crypto; + + beforeEach(async function() { + mockClient = {} as MatrixClient; + const mockStorage = new MockStorageApi() as unknown as Storage; + clientStore = new MemoryStore({ localStorage: mockStorage }) as unknown as IStore; + const cryptoStore = new MemoryCryptoStore(); + + mockRoomList = { + getRoomEncryption: jest.fn().mockReturnValue(null), + setRoomEncryption: jest.fn().mockResolvedValue(undefined), + } as unknown as RoomList; + + crypto = new Crypto( + mockClient, + "@alice:home.server", + "FLIBBLE", + clientStore, + cryptoStore, + mockRoomList, + [], + ); + }); + + it("should set the algorithm if called for a known room", async () => { + const room = new Room("!room:id", mockClient, "@my.user:id"); + await clientStore.storeRoom(room); + await crypto.setRoomEncryption("!room:id", { algorithm: "m.megolm.v1.aes-sha2" } as IRoomEncryption); + expect(mockRoomList!.setRoomEncryption).toHaveBeenCalledTimes(1); + expect(jest.mocked(mockRoomList!.setRoomEncryption).mock.calls[0][0]).toEqual("!room:id"); + }); + + it("should raise if called for an unknown room", async () => { + await expect(async () => { + await crypto.setRoomEncryption("!room:id", { algorithm: "m.megolm.v1.aes-sha2" } as IRoomEncryption); + }).rejects.toThrow(/unknown room/); + expect(mockRoomList!.setRoomEncryption).not.toHaveBeenCalled(); + }); + }); }); diff --git a/spec/unit/embedded.spec.ts b/spec/unit/embedded.spec.ts index 436909be73c..3e0ead87169 100644 --- a/spec/unit/embedded.spec.ts +++ b/spec/unit/embedded.spec.ts @@ -179,7 +179,7 @@ describe("RoomWidgetClient", () => { // It should've also inserted the event into the room object const room = client.getRoom("!1:example.org"); expect(room).not.toBeNull(); - expect(room!.currentState.getStateEvents("org.example.foo", "bar").getEffectiveEvent()).toEqual(event); + expect(room!.currentState.getStateEvents("org.example.foo", "bar")?.getEffectiveEvent()).toEqual(event); }); it("backfills", async () => { @@ -195,7 +195,7 @@ describe("RoomWidgetClient", () => { const room = client.getRoom("!1:example.org"); expect(room).not.toBeNull(); - expect(room!.currentState.getStateEvents("org.example.foo", "bar").getEffectiveEvent()).toEqual(event); + expect(room!.currentState.getStateEvents("org.example.foo", "bar")?.getEffectiveEvent()).toEqual(event); }); }); diff --git a/spec/unit/notifications.spec.ts b/spec/unit/notifications.spec.ts index def7ef820cd..d7936014b7d 100644 --- a/spec/unit/notifications.spec.ts +++ b/spec/unit/notifications.spec.ts @@ -37,7 +37,7 @@ let event: MatrixEvent; let threadEvent: MatrixEvent; const ROOM_ID = "!roomId:example.org"; -let THREAD_ID; +let THREAD_ID: string; function mkPushAction(notify, highlight): IActionsObject { return { @@ -76,7 +76,7 @@ describe("fixNotificationCountOnDecryption", () => { event: true, }, mockClient); - THREAD_ID = event.getId(); + THREAD_ID = event.getId()!; threadEvent = mkEvent({ type: EventType.RoomMessage, content: { @@ -108,6 +108,16 @@ describe("fixNotificationCountOnDecryption", () => { expect(room.getUnreadNotificationCount(NotificationCountType.Highlight)).toBe(1); }); + it("does not change the room count when there's no unread count", () => { + room.setUnreadNotificationCount(NotificationCountType.Total, 0); + room.setUnreadNotificationCount(NotificationCountType.Highlight, 0); + + fixNotificationCountOnDecryption(mockClient, event); + + expect(room.getRoomUnreadNotificationCount(NotificationCountType.Total)).toBe(0); + expect(room.getRoomUnreadNotificationCount(NotificationCountType.Highlight)).toBe(0); + }); + it("changes the thread count to highlight on decryption", () => { expect(room.getThreadUnreadNotificationCount(THREAD_ID, NotificationCountType.Total)).toBe(1); expect(room.getThreadUnreadNotificationCount(THREAD_ID, NotificationCountType.Highlight)).toBe(0); @@ -118,6 +128,16 @@ describe("fixNotificationCountOnDecryption", () => { expect(room.getThreadUnreadNotificationCount(THREAD_ID, NotificationCountType.Highlight)).toBe(1); }); + it("does not change the room count when there's no unread count", () => { + room.setThreadUnreadNotificationCount(THREAD_ID, NotificationCountType.Total, 0); + room.setThreadUnreadNotificationCount(THREAD_ID, NotificationCountType.Highlight, 0); + + fixNotificationCountOnDecryption(mockClient, event); + + expect(room.getThreadUnreadNotificationCount(THREAD_ID, NotificationCountType.Total)).toBe(0); + expect(room.getThreadUnreadNotificationCount(THREAD_ID, NotificationCountType.Highlight)).toBe(0); + }); + it("emits events", () => { const cb = jest.fn(); room.on(RoomEvent.UnreadNotifications, cb); diff --git a/spec/unit/read-receipt.spec.ts b/spec/unit/read-receipt.spec.ts index 2a3fbd87bcc..4443c25befc 100644 --- a/spec/unit/read-receipt.spec.ts +++ b/spec/unit/read-receipt.spec.ts @@ -20,6 +20,7 @@ import { MAIN_ROOM_TIMELINE, ReceiptType } from '../../src/@types/read_receipts' import { MatrixClient } from "../../src/client"; import { Feature, ServerSupport } from '../../src/feature'; import { EventType } from '../../src/matrix'; +import { synthesizeReceipt } from '../../src/models/read-receipt'; import { encodeUri } from '../../src/utils'; import * as utils from "../test-utils/test-utils"; @@ -175,4 +176,20 @@ describe("Read receipt", () => { await flushPromises(); }); }); + + describe("synthesizeReceipt", () => { + it.each([ + { event: roomEvent, destinationId: MAIN_ROOM_TIMELINE }, + { event: threadEvent, destinationId: threadEvent.threadRootId! }, + ])("adds the receipt to $destinationId", ({ event, destinationId }) => { + const userId = "@bob:example.org"; + const receiptType = ReceiptType.Read; + + const fakeReadReceipt = synthesizeReceipt(userId, event, receiptType); + + const content = fakeReadReceipt.getContent()[event.getId()!][receiptType][userId]; + + expect(content.thread_id).toEqual(destinationId); + }); + }); }); diff --git a/spec/unit/room-state.spec.ts b/spec/unit/room-state.spec.ts index 5ee44f6116f..1ac3721a121 100644 --- a/spec/unit/room-state.spec.ts +++ b/spec/unit/room-state.spec.ts @@ -152,7 +152,7 @@ describe("RoomState", function() { it("should return a single MatrixEvent if a state_key was specified", function() { const event = state.getStateEvents("m.room.member", userA); - expect(event.getContent()).toMatchObject({ + expect(event?.getContent()).toMatchObject({ membership: "join", }); }); diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 6a2c23e8372..d16ce952006 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -81,10 +81,13 @@ const fakeIncomingCall = async (client: TestClient, call: MatrixCall, version: s call.getFeeds().push(new CallFeed({ client: client.client, userId: "remote_user_id", - // @ts-ignore Mock - stream: new MockMediaStream("remote_stream_id", [new MockMediaStreamTrack("remote_tack_id")]), - id: "remote_feed_id", + deviceId: undefined, + stream: new MockMediaStream( + "remote_stream_id", [new MockMediaStreamTrack("remote_tack_id", "audio")], + ) as unknown as MediaStream, purpose: SDPStreamMetadataPurpose.Usermedia, + audioMuted: false, + videoMuted: false, })); await callPromise; }; @@ -447,7 +450,7 @@ describe('Call', function() { client.client.getRoom = () => { return { - getMember: (userId) => { + getMember: (userId: string) => { if (userId === opponentMember.userId) { return opponentMember; } @@ -521,10 +524,12 @@ describe('Call', function() { it("should correctly generate local SDPStreamMetadata", async () => { const callPromise = call.placeCallWithCallFeeds([new CallFeed({ client: client.client, - // @ts-ignore Mock - stream: new MockMediaStream("local_stream1", [new MockMediaStreamTrack("track_id", "audio")]), + stream: new MockMediaStream( + "local_stream1", [new MockMediaStreamTrack("track_id", "audio")], + ) as unknown as MediaStream, roomId: call.roomId, userId: client.getUserId(), + deviceId: undefined, purpose: SDPStreamMetadataPurpose.Usermedia, audioMuted: false, videoMuted: false, @@ -534,8 +539,10 @@ describe('Call', function() { call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); (call as any).pushNewLocalFeed( - new MockMediaStream("local_stream2", [new MockMediaStreamTrack("track_id", "video")]), - SDPStreamMetadataPurpose.Screenshare, "feed_id2", + new MockMediaStream( + "local_stream2", [new MockMediaStreamTrack("track_id", "video")], + ) as unknown as MediaStream, + SDPStreamMetadataPurpose.Screenshare, ); await call.setMicrophoneMuted(true); @@ -563,20 +570,18 @@ describe('Call', function() { new CallFeed({ client: client.client, userId: client.getUserId(), - // @ts-ignore Mock - stream: localUsermediaStream, + deviceId: undefined, + stream: localUsermediaStream as unknown as MediaStream, purpose: SDPStreamMetadataPurpose.Usermedia, - id: "local_usermedia_feed_id", audioMuted: false, videoMuted: false, }), new CallFeed({ client: client.client, userId: client.getUserId(), - // @ts-ignore Mock - stream: localScreensharingStream, + deviceId: undefined, + stream: localScreensharingStream as unknown as MediaStream, purpose: SDPStreamMetadataPurpose.Screenshare, - id: "local_screensharing_feed_id", audioMuted: false, videoMuted: false, }), diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index fa84490c146..047474f8e12 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -23,6 +23,7 @@ import { Room, RoomMember, } from '../../../src'; +import { RoomStateEvent } from "../../../src/models/room-state"; import { GroupCall, GroupCallEvent, GroupCallState } from "../../../src/webrtc/groupCall"; import { MatrixClient } from "../../../src/client"; import { @@ -53,40 +54,57 @@ const FAKE_USER_ID_3 = "@charlie:test.dummy"; const FAKE_STATE_EVENTS = [ { getContent: () => ({ - ["m.expires_ts"]: Date.now() + ONE_HOUR, + "m.calls": [], }), getStateKey: () => FAKE_USER_ID_1, getRoomId: () => FAKE_ROOM_ID, + getTs: () => 0, }, { getContent: () => ({ - ["m.expires_ts"]: Date.now() + ONE_HOUR, - ["m.calls"]: [{ - ["m.call_id"]: FAKE_CONF_ID, - ["m.devices"]: [{ + "m.calls": [{ + "m.call_id": FAKE_CONF_ID, + "m.devices": [{ device_id: FAKE_DEVICE_ID_2, + session_id: FAKE_SESSION_ID_2, + expires_ts: Date.now() + ONE_HOUR, feeds: [], }], }], }), getStateKey: () => FAKE_USER_ID_2, getRoomId: () => FAKE_ROOM_ID, + getTs: () => 0, }, { getContent: () => ({ - ["m.expires_ts"]: Date.now() + ONE_HOUR, - ["m.calls"]: [{ - ["m.call_id"]: FAKE_CONF_ID, - ["m.devices"]: [{ + "m.expires_ts": Date.now() + ONE_HOUR, + "m.calls": [{ + "m.call_id": FAKE_CONF_ID, + "m.devices": [{ device_id: "user3_device", + session_id: "user3_session", + expires_ts: Date.now() + ONE_HOUR, feeds: [], }], }], }), getStateKey: () => "user3", getRoomId: () => FAKE_ROOM_ID, + getTs: () => 0, }, ]; +const mockGetStateEvents = (type: EventType, userId?: string): MatrixEvent[] | MatrixEvent | null => { + if (type === EventType.GroupCallMemberPrefix) { + return userId === undefined + ? FAKE_STATE_EVENTS as MatrixEvent[] + : FAKE_STATE_EVENTS.find(e => e.getStateKey() === userId) as MatrixEvent; + } else { + const fakeEvent = { getContent: () => ({}), getTs: () => 0 } as MatrixEvent; + return userId === undefined ? [fakeEvent] : fakeEvent; + } +}; + const ONE_HOUR = 1000 * 60 * 60; const createAndEnterGroupCall = async (cli: MatrixClient, room: Room): Promise => { @@ -111,6 +129,8 @@ class MockCall { public state = CallState.Ringing; public opponentUserId = FAKE_USER_ID_1; + public opponentDeviceId = FAKE_DEVICE_ID_1; + public opponentMember = { userId: this.opponentUserId }; public callId = "1"; public localUsermediaFeed = { setAudioVideoMuted: jest.fn(), @@ -129,9 +149,11 @@ class MockCall { public removeListener = jest.fn(); public getOpponentMember(): Partial { - return { - userId: this.opponentUserId, - }; + return this.opponentMember; + } + + public getOpponentDeviceId(): string { + return this.opponentDeviceId; } public typed(): MatrixCall { return this as unknown as MatrixCall; } @@ -326,8 +348,8 @@ describe('Group Call', function() { describe("call feeds changing", () => { let call: MockCall; - const currentFeed = new MockCallFeed(FAKE_USER_ID_1, new MockMediaStream("current")); - const newFeed = new MockCallFeed(FAKE_USER_ID_1, new MockMediaStream("new")); + const currentFeed = new MockCallFeed(FAKE_USER_ID_1, FAKE_DEVICE_ID_1, new MockMediaStream("current")); + const newFeed = new MockCallFeed(FAKE_USER_ID_1, FAKE_DEVICE_ID_1, new MockMediaStream("new")); beforeEach(async () => { jest.spyOn(currentFeed, "dispose"); @@ -358,7 +380,7 @@ describe('Group Call', function() { }); it("replaces usermedia feed", async () => { - groupCall.userMediaFeeds = [currentFeed.typed()]; + groupCall.userMediaFeeds.push(currentFeed.typed()); call.remoteUsermediaFeed = newFeed.typed(); // @ts-ignore Mock @@ -368,7 +390,7 @@ describe('Group Call', function() { }); it("removes usermedia feed", async () => { - groupCall.userMediaFeeds = [currentFeed.typed()]; + groupCall.userMediaFeeds.push(currentFeed.typed()); // @ts-ignore Mock groupCall.onCallFeedsChanged(call); @@ -387,7 +409,7 @@ describe('Group Call', function() { }); it("replaces screenshare feed", async () => { - groupCall.screenshareFeeds = [currentFeed.typed()]; + groupCall.screenshareFeeds.push(currentFeed.typed()); call.remoteScreensharingFeed = newFeed.typed(); // @ts-ignore Mock @@ -397,7 +419,7 @@ describe('Group Call', function() { }); it("removes screenshare feed", async () => { - groupCall.screenshareFeeds = [currentFeed.typed()]; + groupCall.screenshareFeeds.push(currentFeed.typed()); // @ts-ignore Mock groupCall.onCallFeedsChanged(call); @@ -408,7 +430,7 @@ describe('Group Call', function() { describe("feed replacing", () => { it("replaces usermedia feed", async () => { - groupCall.userMediaFeeds = [currentFeed.typed()]; + groupCall.userMediaFeeds.push(currentFeed.typed()); // @ts-ignore Mock groupCall.replaceUserMediaFeed(currentFeed, newFeed); @@ -422,7 +444,7 @@ describe('Group Call', function() { }); it("replaces screenshare feed", async () => { - groupCall.screenshareFeeds = [currentFeed.typed()]; + groupCall.screenshareFeeds.push(currentFeed.typed()); // @ts-ignore Mock groupCall.replaceScreenshareFeed(currentFeed, newFeed); @@ -489,7 +511,10 @@ describe('Group Call', function() { it("sends metadata updates before unmuting in PTT mode", async () => { const mockCall = new MockCall(FAKE_ROOM_ID, groupCall.groupCallId); - groupCall.calls.push(mockCall as unknown as MatrixCall); + groupCall.calls.set( + mockCall.getOpponentMember() as RoomMember, + new Map([[mockCall.getOpponentDeviceId(), mockCall.typed()]]), + ); let metadataUpdateResolve: () => void; const metadataUpdatePromise = new Promise(resolve => { @@ -511,7 +536,10 @@ describe('Group Call', function() { it("sends metadata updates after muting in PTT mode", async () => { const mockCall = new MockCall(FAKE_ROOM_ID, groupCall.groupCallId); - groupCall.calls.push(mockCall as unknown as MatrixCall); + groupCall.calls.set( + mockCall.getOpponentMember() as RoomMember, + new Map([[mockCall.getOpponentDeviceId(), mockCall.typed()]]), + ); // the call starts muted, so unmute to get in the right state to test await groupCall.setMicrophoneMuted(false); @@ -560,7 +588,7 @@ describe('Group Call', function() { if (eventType === EventType.GroupCallMemberPrefix) { const fakeEvent = { getContent: () => content, - getRoomId: () => FAKE_ROOM_ID, + getRoomId: () => roomId, getStateKey: () => statekey, } as unknown as MatrixEvent; @@ -574,8 +602,8 @@ describe('Group Call', function() { // just add it once. subMap.set(statekey, fakeEvent); - groupCall1.onMemberStateChanged(fakeEvent); - groupCall2.onMemberStateChanged(fakeEvent); + client1Room.currentState.emit(RoomStateEvent.Update, client1Room.currentState); + client2Room.currentState.emit(RoomStateEvent.Update, client2Room.currentState); } return Promise.resolve({ "event_id": "foo" }); }; @@ -584,9 +612,17 @@ describe('Group Call', function() { client2.sendStateEvent.mockImplementation(fakeSendStateEvents); const client1Room = new Room(FAKE_ROOM_ID, client1.typed(), FAKE_USER_ID_1); - const client2Room = new Room(FAKE_ROOM_ID, client2.typed(), FAKE_USER_ID_2); + client1Room.currentState.members[FAKE_USER_ID_1] = client2Room.currentState.members[FAKE_USER_ID_1] = { + userId: FAKE_USER_ID_1, + membership: "join", + } as unknown as RoomMember; + client1Room.currentState.members[FAKE_USER_ID_2] = client2Room.currentState.members[FAKE_USER_ID_2] = { + userId: FAKE_USER_ID_2, + membership: "join", + } as unknown as RoomMember; + groupCall1 = new GroupCall( client1.typed(), client1Room, GroupCallType.Video, false, GroupCallIntent.Prompt, FAKE_CONF_ID, ); @@ -594,20 +630,6 @@ describe('Group Call', function() { groupCall2 = new GroupCall( client2.typed(), client2Room, GroupCallType.Video, false, GroupCallIntent.Prompt, FAKE_CONF_ID, ); - - client1Room.currentState.members[FAKE_USER_ID_1] = { - userId: FAKE_USER_ID_1, - } as unknown as RoomMember; - client1Room.currentState.members[FAKE_USER_ID_2] = { - userId: FAKE_USER_ID_2, - } as unknown as RoomMember; - - client2Room.currentState.members[FAKE_USER_ID_1] = { - userId: FAKE_USER_ID_1, - } as unknown as RoomMember; - client2Room.currentState.members[FAKE_USER_ID_2] = { - userId: FAKE_USER_ID_2, - } as unknown as RoomMember; }); afterEach(function() { @@ -672,8 +694,10 @@ describe('Group Call', function() { expect(client1.sendToDevice).toHaveBeenCalled(); - const oldCall = groupCall1.getCallByUserId(client2.userId); - oldCall!.emit(CallEvent.Hangup, oldCall!); + const oldCall = groupCall1.calls.get( + groupCall1.room.getMember(client2.userId)!, + )!.get(client2.deviceId)!; + oldCall.emit(CallEvent.Hangup, oldCall!); client1.sendToDevice.mockClear(); @@ -691,9 +715,11 @@ describe('Group Call', function() { // to even be created... let newCall: MatrixCall | undefined; while ( - (newCall = groupCall1.getCallByUserId(client2.userId)) === undefined || - newCall.peerConn === undefined || - newCall.callId == oldCall!.callId + (newCall = groupCall1.calls.get( + groupCall1.room.getMember(client2.userId)!, + )?.get(client2.deviceId)) === undefined + || newCall.peerConn === undefined + || newCall.callId == oldCall.callId ) { await flushPromises(); } @@ -733,7 +759,9 @@ describe('Group Call', function() { groupCall1.setMicrophoneMuted(false); groupCall1.setLocalVideoMuted(false); - const call = groupCall1.getCallByUserId(client2.userId)!; + const call = groupCall1.calls.get( + groupCall1.room.getMember(client2.userId)!, + )!.get(client2.deviceId)!; call.isMicrophoneMuted = jest.fn().mockReturnValue(true); call.setMicrophoneMuted = jest.fn(); call.isLocalVideoMuted = jest.fn().mockReturnValue(true); @@ -760,12 +788,15 @@ describe('Group Call', function() { mockClient = typedMockClient as unknown as MatrixClient; room = new Room(FAKE_ROOM_ID, mockClient, FAKE_USER_ID_1); - room.currentState.getStateEvents = jest.fn().mockImplementation((type: EventType, userId: string) => { - return type === EventType.GroupCallMemberPrefix - ? FAKE_STATE_EVENTS.find(e => e.getStateKey() === userId) || FAKE_STATE_EVENTS - : { getContent: () => ([]) }; - }); - room.getMember = jest.fn().mockImplementation((userId) => ({ userId })); + room.currentState.getStateEvents = jest.fn().mockImplementation(mockGetStateEvents); + room.currentState.members[FAKE_USER_ID_1] = { + userId: FAKE_USER_ID_1, + membership: "join", + } as unknown as RoomMember; + room.currentState.members[FAKE_USER_ID_2] = { + userId: FAKE_USER_ID_2, + membership: "join", + } as unknown as RoomMember; }); describe("local muting", () => { @@ -773,17 +804,13 @@ describe('Group Call', function() { const groupCall = await createAndEnterGroupCall(mockClient, room); groupCall.localCallFeed!.setAudioVideoMuted = jest.fn(); - const setAVMutedArray = groupCall.calls.map(call => { - call.localUsermediaFeed!.setAudioVideoMuted = jest.fn(); - return call.localUsermediaFeed!.setAudioVideoMuted; - }); - const tracksArray = groupCall.calls.reduce((acc: MediaStreamTrack[], call: MatrixCall) => { - acc.push(...call.localUsermediaStream!.getAudioTracks()); - return acc; - }, []); - const sendMetadataUpdateArray = groupCall.calls.map(call => { - call.sendMetadataUpdate = jest.fn(); - return call.sendMetadataUpdate; + const setAVMutedArray: ((audioMuted: boolean | null, videoMuted: boolean | null) => void)[] = []; + const tracksArray: MediaStreamTrack[] = []; + const sendMetadataUpdateArray: (() => Promise)[] = []; + groupCall.forEachCall(call => { + setAVMutedArray.push(call.localUsermediaFeed!.setAudioVideoMuted = jest.fn()); + tracksArray.push(...call.localUsermediaStream!.getAudioTracks()); + sendMetadataUpdateArray.push(call.sendMetadataUpdate = jest.fn()); }); await groupCall.setMicrophoneMuted(true); @@ -801,18 +828,14 @@ describe('Group Call', function() { const groupCall = await createAndEnterGroupCall(mockClient, room); groupCall.localCallFeed!.setAudioVideoMuted = jest.fn(); - const setAVMutedArray = groupCall.calls.map(call => { - call.localUsermediaFeed!.setAudioVideoMuted = jest.fn(); + const setAVMutedArray: ((audioMuted: boolean | null, videoMuted: boolean | null) => void)[] = []; + const tracksArray: MediaStreamTrack[] = []; + const sendMetadataUpdateArray: (() => Promise)[] = []; + groupCall.forEachCall(call => { call.localUsermediaFeed!.isVideoMuted = jest.fn().mockReturnValue(true); - return call.localUsermediaFeed!.setAudioVideoMuted; - }); - const tracksArray = groupCall.calls.reduce((acc: MediaStreamTrack[], call: MatrixCall) => { - acc.push(...call.localUsermediaStream!.getVideoTracks()); - return acc; - }, []); - const sendMetadataUpdateArray = groupCall.calls.map(call => { - call.sendMetadataUpdate = jest.fn(); - return call.sendMetadataUpdate; + setAVMutedArray.push(call.localUsermediaFeed!.setAudioVideoMuted = jest.fn()); + tracksArray.push(...call.localUsermediaStream!.getVideoTracks()); + sendMetadataUpdateArray.push(call.sendMetadataUpdate = jest.fn()); }); await groupCall.setLocalVideoMuted(true); @@ -847,7 +870,7 @@ describe('Group Call', function() { // It takes a bit of time for the calls to get created await sleep(10); - const call = groupCall.calls[0]; + const call = groupCall.calls.get(groupCall.room.getMember(FAKE_USER_ID_2)!)!.get(FAKE_DEVICE_ID_2)!; call.getOpponentMember = () => ({ userId: call.invitee }) as RoomMember; // @ts-ignore Mock call.pushRemoteFeed(new MockMediaStream("stream", [ @@ -856,7 +879,7 @@ describe('Group Call', function() { ])); call.onSDPStreamMetadataChangedReceived(metadataEvent); - const feed = groupCall.getUserMediaFeedByUserId(call.invitee!); + const feed = groupCall.getUserMediaFeed(call.invitee!, call.getOpponentDeviceId()!); expect(feed!.isAudioMuted()).toBe(true); expect(feed!.isVideoMuted()).toBe(false); @@ -870,7 +893,7 @@ describe('Group Call', function() { // It takes a bit of time for the calls to get created await sleep(10); - const call = groupCall.calls[0]; + const call = groupCall.calls.get(groupCall.room.getMember(FAKE_USER_ID_2)!)!.get(FAKE_DEVICE_ID_2)!; call.getOpponentMember = () => ({ userId: call.invitee }) as RoomMember; // @ts-ignore Mock call.pushRemoteFeed(new MockMediaStream("stream", [ @@ -879,7 +902,7 @@ describe('Group Call', function() { ])); call.onSDPStreamMetadataChangedReceived(metadataEvent); - const feed = groupCall.getUserMediaFeedByUserId(call.invitee!); + const feed = groupCall.getUserMediaFeed(call.invitee!, call.getOpponentDeviceId()!); expect(feed!.isAudioMuted()).toBe(false); expect(feed!.isVideoMuted()).toBe(true); @@ -945,12 +968,16 @@ describe('Group Call', function() { expect(mockCall.reject).not.toHaveBeenCalled(); expect(mockCall.answerWithCallFeeds).toHaveBeenCalled(); - expect(groupCall.calls).toEqual([mockCall]); + expect(groupCall.calls).toEqual(new Map([[ + groupCall.room.getMember(FAKE_USER_ID_1)!, + new Map([[FAKE_DEVICE_ID_1, mockCall]]), + ]])); }); it("replaces calls if it already has one with the same user", async () => { const oldMockCall = new MockCall(room.roomId, groupCall.groupCallId); const newMockCall = new MockCall(room.roomId, groupCall.groupCallId); + newMockCall.opponentMember = oldMockCall.opponentMember; // Ensure referential equality newMockCall.callId = "not " + oldMockCall.callId; mockClient.emit(CallEventHandlerEvent.Incoming, oldMockCall as unknown as MatrixCall); @@ -958,7 +985,10 @@ describe('Group Call', function() { expect(oldMockCall.hangup).toHaveBeenCalled(); expect(newMockCall.answerWithCallFeeds).toHaveBeenCalled(); - expect(groupCall.calls).toEqual([newMockCall]); + expect(groupCall.calls).toEqual(new Map([[ + groupCall.room.getMember(FAKE_USER_ID_1)!, + new Map([[FAKE_DEVICE_ID_1, newMockCall]]), + ]])); }); it("starts to process incoming calls when we've entered", async () => { @@ -988,32 +1018,34 @@ describe('Group Call', function() { mockClient = typedMockClient.typed(); room = new Room(FAKE_ROOM_ID, mockClient, FAKE_USER_ID_1); - room.getMember = jest.fn().mockImplementation((userId) => ({ userId })); - room.currentState.getStateEvents = jest.fn().mockImplementation((type: EventType, userId: string) => { - return type === EventType.GroupCallMemberPrefix - ? FAKE_STATE_EVENTS.find(e => e.getStateKey() === userId) || FAKE_STATE_EVENTS - : { getContent: () => ([]) }; - }); + room.currentState.members[FAKE_USER_ID_1] = { + userId: FAKE_USER_ID_1, + membership: "join", + } as unknown as RoomMember; + room.currentState.members[FAKE_USER_ID_2] = { + userId: FAKE_USER_ID_2, + membership: "join", + } as unknown as RoomMember; + room.currentState.getStateEvents = jest.fn().mockImplementation(mockGetStateEvents); groupCall = await createAndEnterGroupCall(mockClient, room); }); it("sending screensharing stream", async () => { - const onNegotiationNeededArray = groupCall.calls.map(call => { - // @ts-ignore Mock - call.gotLocalOffer = jest.fn(); + const onNegotiationNeededArray: (() => Promise)[] = []; + groupCall.forEachCall(call => { // @ts-ignore Mock - return call.gotLocalOffer; + onNegotiationNeededArray.push(call.gotLocalOffer = jest.fn()); }); - let enabledResult; + let enabledResult: boolean; enabledResult = await groupCall.setScreensharingEnabled(true); expect(enabledResult).toEqual(true); expect(typedMockClient.mediaHandler.getScreensharingStream).toHaveBeenCalled(); MockRTCPeerConnection.triggerAllNegotiations(); expect(groupCall.screenshareFeeds).toHaveLength(1); - groupCall.calls.forEach(c => { + groupCall.forEachCall(c => { expect(c.getLocalFeeds().find(f => f.purpose === SDPStreamMetadataPurpose.Screenshare)).toBeDefined(); }); onNegotiationNeededArray.forEach(f => expect(f).toHaveBeenCalled()); @@ -1036,7 +1068,7 @@ describe('Group Call', function() { // It takes a bit of time for the calls to get created await sleep(10); - const call = groupCall.calls[0]; + const call = groupCall.calls.get(groupCall.room.getMember(FAKE_USER_ID_2)!)!.get(FAKE_DEVICE_ID_2)!; call.getOpponentMember = () => ({ userId: call.invitee }) as RoomMember; call.onNegotiateReceived({ getContent: () => ({ @@ -1057,7 +1089,7 @@ describe('Group Call', function() { ])); expect(groupCall.screenshareFeeds).toHaveLength(1); - expect(groupCall.getScreenshareFeedByUserId(call.invitee!)).toBeDefined(); + expect(groupCall.getScreenshareFeed(call.invitee!, call.getOpponentDeviceId()!)).toBeDefined(); groupCall.terminate(); }); @@ -1097,12 +1129,16 @@ describe('Group Call', function() { ); room = new Room(FAKE_ROOM_ID, mockClient.typed(), FAKE_USER_ID_1); + room.currentState.members[FAKE_USER_ID_1] = { + userId: FAKE_USER_ID_1, + } as unknown as RoomMember; groupCall = await createAndEnterGroupCall(mockClient.typed(), room); mediaFeed1 = new CallFeed({ client: mockClient.typed(), roomId: FAKE_ROOM_ID, userId: FAKE_USER_ID_2, + deviceId: FAKE_DEVICE_ID_1, stream: (new MockMediaStream("foo", [])).typed(), purpose: SDPStreamMetadataPurpose.Usermedia, audioMuted: false, @@ -1114,6 +1150,7 @@ describe('Group Call', function() { client: mockClient.typed(), roomId: FAKE_ROOM_ID, userId: FAKE_USER_ID_3, + deviceId: FAKE_DEVICE_ID_1, stream: (new MockMediaStream("foo", [])).typed(), purpose: SDPStreamMetadataPurpose.Usermedia, audioMuted: false, @@ -1136,15 +1173,15 @@ describe('Group Call', function() { mediaFeed2.speakingVolumeSamples = [0, 0]; jest.runOnlyPendingTimers(); - expect(groupCall.activeSpeaker).toEqual(FAKE_USER_ID_2); - expect(onActiveSpeakerEvent).toHaveBeenCalledWith(FAKE_USER_ID_2); + expect(groupCall.activeSpeaker).toEqual(mediaFeed1); + expect(onActiveSpeakerEvent).toHaveBeenCalledWith(mediaFeed1); mediaFeed1.speakingVolumeSamples = [0, 0]; mediaFeed2.speakingVolumeSamples = [100, 100]; jest.runOnlyPendingTimers(); - expect(groupCall.activeSpeaker).toEqual(FAKE_USER_ID_3); - expect(onActiveSpeakerEvent).toHaveBeenCalledWith(FAKE_USER_ID_3); + expect(groupCall.activeSpeaker).toEqual(mediaFeed2); + expect(onActiveSpeakerEvent).toHaveBeenCalledWith(mediaFeed2); }); }); diff --git a/spec/unit/webrtc/groupCallEventHandler.spec.ts b/spec/unit/webrtc/groupCallEventHandler.spec.ts index 6712b0f09f0..de70e42085b 100644 --- a/spec/unit/webrtc/groupCallEventHandler.spec.ts +++ b/spec/unit/webrtc/groupCallEventHandler.spec.ts @@ -16,26 +16,21 @@ limitations under the License. import { mocked } from "jest-mock"; +import { ClientEvent } from "../../../src/client"; +import { RoomMember } from "../../../src/models/room-member"; +import { SyncState } from "../../../src/sync"; import { - ClientEvent, - GroupCall, GroupCallIntent, GroupCallState, GroupCallType, - IContent, - MatrixEvent, - Room, - RoomState, -} from "../../../src"; -import { SyncState } from "../../../src/sync"; -import { GroupCallTerminationReason } from "../../../src/webrtc/groupCall"; + GroupCallTerminationReason, +} from "../../../src/webrtc/groupCall"; +import { IContent, MatrixEvent } from "../../../src/models/event"; +import { Room } from "../../../src/models/room"; +import { RoomState } from "../../../src/models/room-state"; import { GroupCallEventHandler, GroupCallEventHandlerEvent } from "../../../src/webrtc/groupCallEventHandler"; import { flushPromises } from "../../test-utils/flushPromises"; -import { - makeMockGroupCallMemberStateEvent, - makeMockGroupCallStateEvent, - MockCallMatrixClient, -} from "../../test-utils/webrtc"; +import { makeMockGroupCallStateEvent, MockCallMatrixClient } from "../../test-utils/webrtc"; const FAKE_USER_ID = "@alice:test.dummy"; const FAKE_DEVICE_ID = "AAAAAAA"; @@ -47,6 +42,7 @@ describe('Group Call Event Handler', function() { let groupCallEventHandler: GroupCallEventHandler; let mockClient: MockCallMatrixClient; let mockRoom: Room; + let mockMember: RoomMember; beforeEach(() => { mockClient = new MockCallMatrixClient( @@ -54,13 +50,27 @@ describe('Group Call Event Handler', function() { ); groupCallEventHandler = new GroupCallEventHandler(mockClient.typed()); + mockMember = { + userId: FAKE_USER_ID, + membership: "join", + } as unknown as RoomMember; + + const mockEvent = makeMockGroupCallStateEvent(FAKE_ROOM_ID, FAKE_GROUP_CALL_ID); + mockRoom = { + on: () => {}, + off: () => {}, roomId: FAKE_ROOM_ID, currentState: { - getStateEvents: jest.fn().mockReturnValue([makeMockGroupCallStateEvent( - FAKE_ROOM_ID, FAKE_GROUP_CALL_ID, - )]), + getStateEvents: jest.fn((type, key) => { + if (type === mockEvent.getType()) { + return key === undefined ? [mockEvent] : mockEvent; + } else { + return key === undefined ? [] : null; + } + }), }, + getMember: (userId: string) => userId === FAKE_USER_ID ? mockMember : null, } as unknown as Room; (mockClient as any).getRoom = jest.fn().mockReturnValue(mockRoom); @@ -211,27 +221,6 @@ describe('Group Call Event Handler', function() { ); }); - it("sends member events to group calls", async () => { - await groupCallEventHandler.start(); - - const mockGroupCall = { - onMemberStateChanged: jest.fn(), - }; - - groupCallEventHandler.groupCalls.set(FAKE_ROOM_ID, mockGroupCall as unknown as GroupCall); - - const mockStateEvent = makeMockGroupCallMemberStateEvent(FAKE_ROOM_ID, FAKE_GROUP_CALL_ID); - - mockClient.emitRoomState( - mockStateEvent, - { - roomId: FAKE_ROOM_ID, - } as unknown as RoomState, - ); - - expect(mockGroupCall.onMemberStateChanged).toHaveBeenCalledWith(mockStateEvent); - }); - describe("ignoring invalid group call state events", () => { let mockClientEmit: jest.Func; diff --git a/src/ReEmitter.ts b/src/ReEmitter.ts index 6822e3d1e98..2fa6eface1e 100644 --- a/src/ReEmitter.ts +++ b/src/ReEmitter.ts @@ -22,7 +22,7 @@ import { EventEmitter } from "events"; import { ListenerMap, TypedEventEmitter } from "./models/typed-event-emitter"; export class ReEmitter { - constructor(private readonly target: EventEmitter) {} + public constructor(private readonly target: EventEmitter) {} // Map from emitter to event name to re-emitter private reEmitters = new Map void>>(); @@ -38,7 +38,7 @@ export class ReEmitter { // We include the source as the last argument for event handlers which may need it, // such as read receipt listeners on the client class which won't have the context // of the room. - const forSource = (...args: any[]) => { + const forSource = (...args: any[]): void => { // EventEmitter special cases 'error' to make the emit function throw if no // handler is attached, which sort of makes sense for making sure that something // handles an error, but for re-emitting, there could be a listener on the original @@ -74,7 +74,7 @@ export class TypedReEmitter< Events extends string, Arguments extends ListenerMap, > extends ReEmitter { - constructor(target: TypedEventEmitter) { + public constructor(target: TypedEventEmitter) { super(target); } diff --git a/src/ToDeviceMessageQueue.ts b/src/ToDeviceMessageQueue.ts index 815927679a4..0b5b1786a63 100644 --- a/src/ToDeviceMessageQueue.ts +++ b/src/ToDeviceMessageQueue.ts @@ -31,7 +31,7 @@ export class ToDeviceMessageQueue { private retryTimeout: ReturnType | null = null; private retryAttempts = 0; - constructor(private client: MatrixClient) { + public constructor(private client: MatrixClient) { } public start(): void { diff --git a/src/client.ts b/src/client.ts index 52e8cdade39..d0758632b23 100644 --- a/src/client.ts +++ b/src/client.ts @@ -78,6 +78,7 @@ import { IMegolmSessionData, isCryptoAvailable, VerificationMethod, + IRoomKeyRequestBody, } from './crypto'; import { DeviceInfo, IDevice } from "./crypto/deviceinfo"; import { decodeRecoveryKey } from './crypto/recoverykey'; @@ -184,7 +185,7 @@ import { RuleId, } from "./@types/PushRules"; import { IThreepid } from "./@types/threepids"; -import { CryptoStore } from "./crypto/store/base"; +import { CryptoStore, OutgoingRoomKeyRequest } from "./crypto/store/base"; import { GroupCall, IGroupCallDataChannelOptions, @@ -912,6 +913,7 @@ export type EmittedEvents = ClientEvent | CallEvent // re-emitted by call.ts using Object.values | CallEventHandlerEvent.Incoming | GroupCallEventHandlerEvent.Incoming + | GroupCallEventHandlerEvent.Outgoing | GroupCallEventHandlerEvent.Ended | GroupCallEventHandlerEvent.Participants | HttpApiEvent.SessionLoggedOut @@ -1028,7 +1030,7 @@ export class MatrixClient extends TypedEventEmitter { + this.clientOpts.canResetEntireTimeline = (roomId): boolean => { if (!this.canResetTimelineCallback) { return false; } @@ -1280,7 +1282,7 @@ export class MatrixClient extends TypedEventEmitter * @param {boolean} guest True if this is a guest account. */ - public setGuest(guest: boolean) { + public setGuest(guest: boolean): void { // EXPERIMENTAL: // If the token is a macaroon, it should be encoded in it that it is a 'guest' // access token, which means that the SDK can determine this entirely without @@ -1751,7 +1753,7 @@ export class MatrixClient extends TypedEventEmitter { if (!this.crypto) { throw new Error("End-to-end encryption disabled"); } @@ -2382,11 +2384,11 @@ export class MatrixClient extends TypedEventEmitter { if (!this.crypto) { throw new Error("End-to-end encryption disabled"); } @@ -2605,7 +2607,7 @@ export class MatrixClient extends TypedEventEmitter { if (!this.crypto) { throw new Error("End-to-end encryption disabled"); } @@ -2660,6 +2662,32 @@ export class MatrixClient extends TypedEventEmitter { + if (!this.crypto) { + throw new Error("End-to-End encryption disabled"); + } + const wireContent = event.getWireContent(); + const requestBody: IRoomKeyRequestBody = { + session_id: wireContent.session_id, + sender_key: wireContent.sender_key, + algorithm: wireContent.algorithm, + room_id: event.getRoomId()!, + }; + if ( + !requestBody.session_id + || !requestBody.sender_key + || !requestBody.algorithm + || !requestBody.room_id + ) return Promise.resolve(null); + return this.crypto.cryptoStore.getOutgoingRoomKeyRequest(requestBody); + } + /** * Cancel a room key request for this event if one is ongoing and resend the * request. @@ -2746,7 +2774,7 @@ export class MatrixClient extends TypedEventEmitter { if (!this.crypto) { throw new Error("End-to-end encryption disabled"); } @@ -3431,7 +3459,7 @@ export class MatrixClient extends TypedEventEmitter { if (!this.crypto) { throw new Error("End-to-end encryption disabled"); } @@ -3659,10 +3687,9 @@ export class MatrixClient extends TypedEventEmitter = Promise.resolve(); if (opts.inviteSignUrl) { - signPromise = this.http.requestOtherUrl( - Method.Post, - new URL(opts.inviteSignUrl), { mxid: this.credentials.userId }, - ); + const url = new URL(opts.inviteSignUrl); + url.searchParams.set("mxid", this.credentials.userId!); + signPromise = this.http.requestOtherUrl(Method.Post, url); } const queryString: Record = {}; @@ -3715,7 +3742,7 @@ export class MatrixClient extends TypedEventEmitter { let content = { - users: {}, + users: {} as Record, }; - if (event?.getType() === EventType.RoomPowerLevels) { + if (event.getType() === EventType.RoomPowerLevels) { // take a copy of the content to ensure we don't corrupt // existing client state with a failed power level change content = utils.deepCopy(event.getContent()); } - content.users[userId] = powerLevel; + if (Array.isArray(userId)) { + for (const user of userId) { + content.users[user] = powerLevel; + } + } else { + content.users[userId] = powerLevel; + } const path = utils.encodeUri("/rooms/$roomId/state/m.room.power_levels", { $roomId: roomId, }); @@ -3864,7 +3897,7 @@ export class MatrixClient extends TypedEventEmitter { return this.unstable_setLiveBeacon(roomId, beaconInfoContent); } @@ -3879,7 +3912,7 @@ export class MatrixClient extends TypedEventEmitter { return this.sendStateEvent(roomId, M_BEACON_INFO.name, beaconInfoContent, this.getUserId()!); } @@ -4151,7 +4184,7 @@ export class MatrixClient extends TypedEventEmitter[] = []; - const doLeave = (roomId: string) => { + const doLeave = (roomId: string): Promise => { return this.leave(roomId).then(() => { delete populationResults[roomId]; }).catch((err) => { + // suppress error populationResults[roomId] = err; - return null; // suppress error }); }; @@ -5860,8 +5893,14 @@ export class MatrixClient extends TypedEventEmitter { const mapper = this.getEventMapper(); const matrixEvents = res.chunk.map(mapper); - for (const event of matrixEvents) { - await eventTimeline.getTimelineSet()?.thread?.processEvent(event); + + // Process latest events first + for (const event of matrixEvents.slice().reverse()) { + await thread?.processEvent(event); + const sender = event.getSender()!; + if (!backwards || thread?.getEventReadUpTo(sender) === null) { + room.addLocalEchoReceipt(sender, event, ReceiptType.Read); + } } const newToken = res.next_batch; @@ -5934,7 +5973,7 @@ export class MatrixClient extends TypedEventEmitter 0) { + if ((oldHighlight !== newHighlight || currentCount > 0) && totalCount > 0) { // TODO: Handle mentions received while the client is offline // See also https://github.com/vector-im/element-web/issues/9069 const hasReadEvent = isThreadEvent @@ -9405,7 +9441,7 @@ export function fixNotificationCountOnDecryption(cli: MatrixClient, event: Matri // Fix 'Mentions Only' rooms from not having the right badge count const totalCount = (isThreadEvent ? room.getThreadUnreadNotificationCount(event.threadRootId, NotificationCountType.Total) - : room.getUnreadNotificationCount(NotificationCountType.Total)) ?? 0; + : room.getRoomUnreadNotificationCount(NotificationCountType.Total)) ?? 0; if (totalCount < newCount) { if (isThreadEvent) { diff --git a/src/content-helpers.ts b/src/content-helpers.ts index 4dbb37b212a..6fa4b684b5b 100644 --- a/src/content-helpers.ts +++ b/src/content-helpers.ts @@ -33,6 +33,7 @@ import { LegacyLocationEventContent, } from "./@types/location"; import { MRoomTopicEventContent, MTopicContent, M_TOPIC } from "./@types/topic"; +import { IContent } from "./models/event"; /** * Generates the content for a HTML Message event @@ -40,7 +41,7 @@ import { MRoomTopicEventContent, MTopicContent, M_TOPIC } from "./@types/topic"; * @param {string} htmlBody the HTML representation of the message * @returns {{msgtype: string, format: string, body: string, formatted_body: string}} */ -export function makeHtmlMessage(body: string, htmlBody: string) { +export function makeHtmlMessage(body: string, htmlBody: string): IContent { return { msgtype: MsgType.Text, format: "org.matrix.custom.html", @@ -55,7 +56,7 @@ export function makeHtmlMessage(body: string, htmlBody: string) { * @param {string} htmlBody the HTML representation of the notice * @returns {{msgtype: string, format: string, body: string, formatted_body: string}} */ -export function makeHtmlNotice(body: string, htmlBody: string) { +export function makeHtmlNotice(body: string, htmlBody: string): IContent { return { msgtype: MsgType.Notice, format: "org.matrix.custom.html", @@ -70,7 +71,7 @@ export function makeHtmlNotice(body: string, htmlBody: string) { * @param {string} htmlBody the HTML representation of the emote * @returns {{msgtype: string, format: string, body: string, formatted_body: string}} */ -export function makeHtmlEmote(body: string, htmlBody: string) { +export function makeHtmlEmote(body: string, htmlBody: string): IContent { return { msgtype: MsgType.Emote, format: "org.matrix.custom.html", @@ -84,7 +85,7 @@ export function makeHtmlEmote(body: string, htmlBody: string) { * @param {string} body the plaintext body of the emote * @returns {{msgtype: string, body: string}} */ -export function makeTextMessage(body: string) { +export function makeTextMessage(body: string): IContent { return { msgtype: MsgType.Text, body: body, @@ -96,7 +97,7 @@ export function makeTextMessage(body: string) { * @param {string} body the plaintext body of the notice * @returns {{msgtype: string, body: string}} */ -export function makeNotice(body: string) { +export function makeNotice(body: string): IContent { return { msgtype: MsgType.Notice, body: body, @@ -108,7 +109,7 @@ export function makeNotice(body: string) { * @param {string} body the plaintext body of the emote * @returns {{msgtype: string, body: string}} */ -export function makeEmoteMessage(body: string) { +export function makeEmoteMessage(body: string): IContent { return { msgtype: MsgType.Emote, body: body, diff --git a/src/crypto/CrossSigning.ts b/src/crypto/CrossSigning.ts index 59907bdbf78..e776b93ad94 100644 --- a/src/crypto/CrossSigning.ts +++ b/src/crypto/CrossSigning.ts @@ -21,7 +21,7 @@ limitations under the License. import { PkSigning } from "@matrix-org/olm"; -import { decodeBase64, encodeBase64, pkSign, pkVerify } from './olmlib'; +import { decodeBase64, encodeBase64, IObject, pkSign, pkVerify } from './olmlib'; import { logger } from '../logger'; import { IndexedDBCryptoStore } from '../crypto/store/indexeddb-crypto-store'; import { decryptAES, encryptAES } from './aes'; @@ -74,7 +74,7 @@ export class CrossSigningInfo { * Requires getCrossSigningKey and saveCrossSigningKeys * @param {object} cacheCallbacks Callbacks used to interact with the cache */ - constructor( + public constructor( public readonly userId: string, private callbacks: ICryptoCallbacks = {}, private cacheCallbacks: ICacheCallbacks = {}, @@ -175,7 +175,7 @@ export class CrossSigningInfo { // check what SSSS keys have encrypted the master key (if any) const stored = await secretStorage.isStored("m.cross_signing.master") || {}; // then check which of those SSSS keys have also encrypted the SSK and USK - function intersect(s: Record) { + function intersect(s: Record): void { for (const k of Object.keys(stored)) { if (!s[k]) { delete stored[k]; @@ -586,7 +586,14 @@ export class CrossSigningInfo { } } -function deviceToObject(device: DeviceInfo, userId: string) { +interface DeviceObject extends IObject { + algorithms: string[]; + keys: Record; + device_id: string; + user_id: string; +} + +function deviceToObject(device: DeviceInfo, userId: string): DeviceObject { return { algorithms: device.algorithms, keys: device.keys, @@ -606,7 +613,7 @@ export enum CrossSigningLevel { * Represents the ways in which we trust a user */ export class UserTrustLevel { - constructor( + public constructor( private readonly crossSigningVerified: boolean, private readonly crossSigningVerifiedBefore: boolean, private readonly tofu: boolean, @@ -646,7 +653,7 @@ export class UserTrustLevel { * Represents the ways in which we trust a device */ export class DeviceTrustLevel { - constructor( + public constructor( public readonly crossSigningVerified: boolean, public readonly tofu: boolean, private readonly localVerified: boolean, @@ -775,7 +782,7 @@ export async function requestKeysDuringVerification( // CrossSigningInfo.getCrossSigningKey() to validate/cache const crossSigning = new CrossSigningInfo( original.userId, - { getCrossSigningKey: async (type) => { + { getCrossSigningKey: async (type): Promise => { logger.debug("Cross-signing: requesting secret", type, deviceId); const { promise } = client.requestSecret( `m.cross_signing.${type}`, [deviceId], @@ -801,7 +808,7 @@ export async function requestKeysDuringVerification( }); // also request and cache the key backup key - const backupKeyPromise = (async () => { + const backupKeyPromise = (async (): Promise => { const cachedKey = await client.crypto!.getSessionBackupPrivateKey(); if (!cachedKey) { logger.info("No cached backup key found. Requesting..."); diff --git a/src/crypto/DeviceList.ts b/src/crypto/DeviceList.ts index c4345437c3e..da40a03f231 100644 --- a/src/crypto/DeviceList.ts +++ b/src/crypto/DeviceList.ts @@ -102,7 +102,7 @@ export class DeviceList extends TypedEventEmitter { await this.cryptoStore.doTxn( 'readonly', [IndexedDBCryptoStore.STORE_DEVICE_DATA], (txn) => { this.cryptoStore.getEndToEndDeviceData(txn, (deviceData) => { @@ -150,7 +150,7 @@ export class DeviceList extends TypedEventEmitter} accountData pre-existing account data, will only be read, not written. * @param {CryptoCallbacks} delegateCryptoCallbacks crypto callbacks to delegate to if the key isn't in cache yet */ - constructor(accountData: Record, delegateCryptoCallbacks?: ICryptoCallbacks) { + public constructor(accountData: Record, delegateCryptoCallbacks?: ICryptoCallbacks) { this.accountDataClientAdapter = new AccountDataClientAdapter(accountData); this.crossSigningCallbacks = new CrossSigningCallbacks(); this.ssssCryptoCallbacks = new SSSSCryptoCallbacks(delegateCryptoCallbacks); @@ -192,7 +192,7 @@ export class EncryptionSetupOperation { * @param {Object} keyBackupInfo * @param {Object} keySignatures */ - constructor( + public constructor( private readonly accountData: Map, private readonly crossSigningKeys?: ICrossSigningKeys, private readonly keyBackupInfo?: IKeyBackupInfo, @@ -272,7 +272,7 @@ class AccountDataClientAdapter /** * @param {Object.} existingValues existing account data */ - constructor(private readonly existingValues: Record) { + public constructor(private readonly existingValues: Record) { super(); } @@ -342,7 +342,7 @@ class CrossSigningCallbacks implements ICryptoCallbacks, ICacheCallbacks { return Promise.resolve(this.privateKeys.get(type) ?? null); } - public saveCrossSigningKeys(privateKeys: Record) { + public saveCrossSigningKeys(privateKeys: Record): void { for (const [type, privateKey] of Object.entries(privateKeys)) { this.privateKeys.set(type, privateKey); } @@ -356,7 +356,7 @@ class CrossSigningCallbacks implements ICryptoCallbacks, ICacheCallbacks { class SSSSCryptoCallbacks { private readonly privateKeys = new Map(); - constructor(private readonly delegateCryptoCallbacks?: ICryptoCallbacks) {} + public constructor(private readonly delegateCryptoCallbacks?: ICryptoCallbacks) {} public async getSecretStorageKey( { keys }: { keys: Record }, diff --git a/src/crypto/OlmDevice.ts b/src/crypto/OlmDevice.ts index c60d19c1a71..efa34ad4159 100644 --- a/src/crypto/OlmDevice.ts +++ b/src/crypto/OlmDevice.ts @@ -177,13 +177,13 @@ export class OlmDevice { // Used by olm to serialise prekey message decryptions public olmPrekeyPromise: Promise = Promise.resolve(); // set by consumers - constructor(private readonly cryptoStore: CryptoStore) { + public constructor(private readonly cryptoStore: CryptoStore) { } /** * @return {array} The version of Olm. */ - static getOlmVersion(): [number, number, number] { + public static getOlmVersion(): [number, number, number] { return global.Olm.get_library_version(); } @@ -916,6 +916,7 @@ export class OlmDevice { } public async recordSessionProblem(deviceKey: string, type: string, fixed: boolean): Promise { + logger.info(`Recording problem on olm session with ${deviceKey} of type ${type}. Recreating: ${fixed}`); await this.cryptoStore.storeEndToEndSessionProblem(deviceKey, type, fixed); } @@ -1139,17 +1140,14 @@ export class OlmDevice { } if (existingSession) { - logger.log( - "Update for megolm session " - + senderKey + "/" + sessionId, - ); + logger.log(`Update for megolm session ${senderKey}|${sessionId}`); if (existingSession.first_known_index() <= session.first_known_index()) { if (!existingSessionData!.untrusted || extraSessionData.untrusted) { // existing session has less-than-or-equal index // (i.e. can decrypt at least as much), and the // new session's trust does not win over the old // session's trust, so keep it - logger.log(`Keeping existing megolm session ${sessionId}`); + logger.log(`Keeping existing megolm session ${senderKey}|${sessionId}`); return; } if (existingSession.first_known_index() < session.first_known_index()) { @@ -1164,7 +1162,7 @@ export class OlmDevice { ) { logger.info( "Upgrading trust of existing megolm session " + - sessionId + " based on newly-received trusted session", + `${senderKey}|${sessionId} based on newly-received trusted session`, ); existingSessionData!.untrusted = false; this.cryptoStore.storeEndToEndInboundGroupSession( @@ -1172,7 +1170,7 @@ export class OlmDevice { ); } else { logger.warn( - "Newly-received megolm session " + sessionId + + `Newly-received megolm session ${senderKey}|$sessionId}` + " does not match existing session! Keeping existing session", ); } @@ -1183,8 +1181,8 @@ export class OlmDevice { } logger.info( - "Storing megolm session " + senderKey + "/" + sessionId + - " with first index " + session.first_known_index(), + `Storing megolm session ${senderKey}|${sessionId} with first index `+ + session.first_known_index(), ); const sessionData = Object.assign({}, extraSessionData, { @@ -1517,7 +1515,9 @@ export class OlmDevice { }); } - async getSharedHistoryInboundGroupSessions(roomId: string): Promise<[senderKey: string, sessionId: string][]> { + public async getSharedHistoryInboundGroupSessions( + roomId: string, + ): Promise<[senderKey: string, sessionId: string][]> { let result: Promise<[senderKey: string, sessionId: string][]>; await this.cryptoStore.doTxn( 'readonly', [ diff --git a/src/crypto/OutgoingRoomKeyRequestManager.ts b/src/crypto/OutgoingRoomKeyRequestManager.ts index d5d1c3003f2..42723434778 100644 --- a/src/crypto/OutgoingRoomKeyRequestManager.ts +++ b/src/crypto/OutgoingRoomKeyRequestManager.ts @@ -102,7 +102,7 @@ export class OutgoingRoomKeyRequestManager { private clientRunning = true; - constructor( + public constructor( private readonly baseApis: MatrixClient, private readonly deviceId: string, private readonly cryptoStore: CryptoStore, @@ -352,7 +352,7 @@ export class OutgoingRoomKeyRequestManager { return; } - const startSendingOutgoingRoomKeyRequests = () => { + const startSendingOutgoingRoomKeyRequests = (): void => { if (this.sendOutgoingRoomKeyRequestsRunning) { throw new Error("RoomKeyRequestSend already in progress!"); } diff --git a/src/crypto/RoomList.ts b/src/crypto/RoomList.ts index f0e09b657a0..672ef473c05 100644 --- a/src/crypto/RoomList.ts +++ b/src/crypto/RoomList.ts @@ -38,7 +38,7 @@ export class RoomList { // Object of roomId -> room e2e info object (body of the m.room.encryption event) private roomEncryption: Record = {}; - constructor(private readonly cryptoStore?: CryptoStore) {} + public constructor(private readonly cryptoStore?: CryptoStore) {} public async init(): Promise { await this.cryptoStore!.doTxn( diff --git a/src/crypto/SecretStorage.ts b/src/crypto/SecretStorage.ts index d1d6285fcce..5c13ba4b0bb 100644 --- a/src/crypto/SecretStorage.ts +++ b/src/crypto/SecretStorage.ts @@ -78,7 +78,7 @@ export class SecretStorage { // as you don't request any secrets. // A better solution would probably be to split this class up into secret storage and // secret sharing which are really two separate things, even though they share an MSC. - constructor( + public constructor( private readonly accountDataAdapter: IAccountDataClient, private readonly cryptoCallbacks: ICryptoCallbacks, private readonly baseApis: B, @@ -381,7 +381,7 @@ export class SecretStorage { const deferred = defer(); this.requests.set(requestId, { name, devices, deferred }); - const cancel = (reason: string) => { + const cancel = (reason: string): void => { // send cancellation event const cancelData = { action: "request_cancellation", diff --git a/src/crypto/algorithms/base.ts b/src/crypto/algorithms/base.ts index 190bfa437ab..d6c70bc1067 100644 --- a/src/crypto/algorithms/base.ts +++ b/src/crypto/algorithms/base.ts @@ -78,7 +78,7 @@ export abstract class EncryptionAlgorithm { protected readonly baseApis: MatrixClient; protected readonly roomId?: string; - constructor(params: IParams) { + public constructor(params: IParams) { this.userId = params.userId; this.deviceId = params.deviceId; this.crypto = params.crypto; @@ -150,7 +150,7 @@ export abstract class DecryptionAlgorithm { protected readonly baseApis: MatrixClient; protected readonly roomId?: string; - constructor(params: DecryptionClassParams) { + public constructor(params: DecryptionClassParams) { this.userId = params.userId; this.crypto = params.crypto; this.olmDevice = params.olmDevice; @@ -242,7 +242,7 @@ export abstract class DecryptionAlgorithm { export class DecryptionError extends Error { public readonly detailedString: string; - constructor(public readonly code: string, msg: string, details?: Record) { + public constructor(public readonly code: string, msg: string, details?: Record) { super(msg); this.code = code; this.name = 'DecryptionError'; @@ -272,7 +272,7 @@ function detailedStringForDecryptionError(err: DecryptionError, details?: Record * @extends Error */ export class UnknownDeviceError extends Error { - constructor( + public constructor( msg: string, public readonly devices: Record>, public event?: MatrixEvent, diff --git a/src/crypto/algorithms/megolm.ts b/src/crypto/algorithms/megolm.ts index 3302bbc7795..cbad327a608 100644 --- a/src/crypto/algorithms/megolm.ts +++ b/src/crypto/algorithms/megolm.ts @@ -136,7 +136,7 @@ class OutboundSessionInfo { public sharedWithDevices: Record> = {}; public blockedDevicesNotified: Record> = {}; - constructor(public readonly sessionId: string, public readonly sharedHistory = false) { + public constructor(public readonly sessionId: string, public readonly sharedHistory = false) { this.creationTime = new Date().getTime(); } @@ -248,7 +248,7 @@ class MegolmEncryption extends EncryptionAlgorithm { protected readonly roomId: string; - constructor(params: IParams & Required>) { + public constructor(params: IParams & Required>) { super(params); this.roomId = params.roomId; @@ -347,7 +347,7 @@ class MegolmEncryption extends EncryptionAlgorithm { singleOlmCreationPhase: boolean, blocked: IBlockedMap, session: OutboundSessionInfo, - ) { + ): Promise { // now check if we need to share with any devices const shareMap: Record = {}; @@ -386,13 +386,13 @@ class MegolmEncryption extends EncryptionAlgorithm { ); await Promise.all([ - (async () => { + (async (): Promise => { // share keys with devices that we already have a session for logger.debug(`Sharing keys with existing Olm sessions in ${this.roomId}`, olmSessions); await this.shareKeyWithOlmSessions(session, key, payload, olmSessions); logger.debug(`Shared keys with existing Olm sessions in ${this.roomId}`); })(), - (async () => { + (async (): Promise => { logger.debug( `Sharing keys (start phase 1) with new Olm sessions in ${this.roomId}`, devicesWithoutSession, @@ -415,7 +415,7 @@ class MegolmEncryption extends EncryptionAlgorithm { if (!singleOlmCreationPhase && (Date.now() - start < 10000)) { // perform the second phase of olm session creation if requested, // and if the first phase didn't take too long - (async () => { + (async (): Promise => { // Retry sending keys to devices that we were unable to establish // an olm session for. This time, we use a longer timeout, but we // do this in the background and don't block anything else while we @@ -452,7 +452,7 @@ class MegolmEncryption extends EncryptionAlgorithm { } logger.debug(`Shared keys (all phases done) with new Olm sessions in ${this.roomId}`); })(), - (async () => { + (async (): Promise => { logger.debug(`There are ${Object.entries(blocked).length} blocked devices in ${this.roomId}`, Object.entries(blocked)); @@ -696,28 +696,27 @@ class MegolmEncryption extends EncryptionAlgorithm { ): Promise { const obSessionInfo = this.outboundSessions[sessionId]; if (!obSessionInfo) { - logger.debug(`megolm session ${sessionId} not found: not re-sharing keys`); + logger.debug(`megolm session ${senderKey}|${sessionId} not found: not re-sharing keys`); return; } // The chain index of the key we previously sent this device if (obSessionInfo.sharedWithDevices[userId] === undefined) { - logger.debug(`megolm session ${sessionId} never shared with user ${userId}`); + logger.debug(`megolm session ${senderKey}|${sessionId} never shared with user ${userId}`); return; } const sessionSharedData = obSessionInfo.sharedWithDevices[userId][device.deviceId]; if (sessionSharedData === undefined) { logger.debug( - "megolm session ID " + sessionId + " never shared with device " + - userId + ":" + device.deviceId, + `megolm session ${senderKey}|${sessionId} never shared with device ${userId}:${device.deviceId}`, ); return; } if (sessionSharedData.deviceKey !== device.getIdentityKey()) { logger.warn( - `Session has been shared with device ${device.deviceId} but with identity ` + - `key ${sessionSharedData.deviceKey}. Key is now ${device.getIdentityKey()}!`, + `Megolm session ${senderKey}|${sessionId} has been shared with device ${device.deviceId} but ` + + `with identity key ${sessionSharedData.deviceKey}. Key is now ${device.getIdentityKey()}!`, ); return; } @@ -730,7 +729,7 @@ class MegolmEncryption extends EncryptionAlgorithm { if (!key) { logger.warn( - `No inbound session key found for megolm ${sessionId}: not re-sharing keys`, + `No inbound session key found for megolm session ${senderKey}|${sessionId}: not re-sharing keys`, ); return; } @@ -776,7 +775,7 @@ class MegolmEncryption extends EncryptionAlgorithm { [device.deviceId]: encryptedContent, }, }); - logger.debug(`Re-shared key for megolm session ${sessionId} with ${userId}:${device.deviceId}`); + logger.debug(`Re-shared key for megolm session ${senderKey}|${sessionId} with ${userId}:${device.deviceId}`); } /** @@ -810,7 +809,7 @@ class MegolmEncryption extends EncryptionAlgorithm { errorDevices: IOlmDevice[], otkTimeout: number, failedServers?: string[], - ) { + ): Promise { logger.debug(`Ensuring Olm sessions for devices in ${this.roomId}`); const devicemap = await olmlib.ensureOlmSessionsForDevices( this.olmDevice, this.baseApis, devicesByUser, false, otkTimeout, failedServers, @@ -970,7 +969,7 @@ class MegolmEncryption extends EncryptionAlgorithm { this.encryptionPreparation = { startTime: Date.now(), - promise: (async () => { + promise: (async (): Promise => { try { logger.debug(`Getting devices in ${this.roomId}`); const [devicesInRoom, blocked] = await this.getDevicesInRoom(room); @@ -1232,7 +1231,7 @@ class MegolmDecryption extends DecryptionAlgorithm { protected readonly roomId: string; - constructor(params: DecryptionClassParams>>) { + public constructor(params: DecryptionClassParams>>) { super(params); this.roomId = params.roomId; } @@ -1312,6 +1311,10 @@ class MegolmDecryption extends DecryptionAlgorithm { content.sender_key, event.getTs() - 120000, ); if (problem) { + logger.info( + `When handling UISI from ${event.getSender()} (sender key ${content.sender_key}): ` + + `recent session problem with that sender: ${problem}`, + ); let problemDescription = PROBLEM_DESCRIPTIONS[problem.type as "no_olm"] || PROBLEM_DESCRIPTIONS.unknown; if (problem.fixed) { problemDescription += @@ -1902,10 +1905,11 @@ class MegolmDecryption extends DecryptionAlgorithm { public async sendSharedHistoryInboundSessions(devicesByUser: Record): Promise { await olmlib.ensureOlmSessionsForDevices(this.olmDevice, this.baseApis, devicesByUser); - logger.log("sendSharedHistoryInboundSessions to users", Object.keys(devicesByUser)); - const sharedHistorySessions = await this.olmDevice.getSharedHistoryInboundGroupSessions(this.roomId); - logger.log("shared-history sessions", sharedHistorySessions); + logger.log( + `Sharing history in ${this.roomId} with users ${Object.keys(devicesByUser)}`, + sharedHistorySessions.map(([senderKey, sessionId]) => `${senderKey}|${sessionId}`), + ); for (const [senderKey, sessionId] of sharedHistorySessions) { const payload = await this.buildKeyForwardingMessage(this.roomId, senderKey, sessionId); diff --git a/src/crypto/backup.ts b/src/crypto/backup.ts index d9c1ba4587c..f2160165bd8 100644 --- a/src/crypto/backup.ts +++ b/src/crypto/backup.ts @@ -120,7 +120,7 @@ export class BackupManager { private sendingBackups: boolean; // Are we currently sending backups? private sessionLastCheckAttemptedTime: Record = {}; // When did we last try to check the server for a given session id? - constructor(private readonly baseApis: MatrixClient, public readonly getKey: GetKey) { + public constructor(private readonly baseApis: MatrixClient, public readonly getKey: GetKey) { this.checkedForBackup = false; this.sendingBackups = false; } @@ -609,7 +609,7 @@ export class BackupManager { export class Curve25519 implements BackupAlgorithm { public static algorithmName = "m.megolm_backup.v1.curve25519-aes-sha2"; - constructor( + public constructor( public authData: ICurve25519AuthData, private publicKey: any, // FIXME: PkEncryption private getKey: () => Promise, @@ -661,7 +661,7 @@ export class Curve25519 implements BackupAlgorithm { } } - public get untrusted() { return true; } + public get untrusted(): boolean { return true; } public async encryptSession(data: Record): Promise { const plainText: Record = Object.assign({}, data); @@ -735,7 +735,7 @@ const UNSTABLE_MSC3270_NAME = new UnstableValue( export class Aes256 implements BackupAlgorithm { public static algorithmName = UNSTABLE_MSC3270_NAME.name; - constructor( + public constructor( public readonly authData: IAes256AuthData, private readonly key: Uint8Array, ) {} @@ -786,7 +786,7 @@ export class Aes256 implements BackupAlgorithm { } } - public get untrusted() { return false; } + public get untrusted(): boolean { return false; } public encryptSession(data: Record): Promise { const plainText: Record = Object.assign({}, data); diff --git a/src/crypto/dehydration.ts b/src/crypto/dehydration.ts index 5b12159ae5d..cc6b252da59 100644 --- a/src/crypto/dehydration.ts +++ b/src/crypto/dehydration.ts @@ -62,7 +62,7 @@ export class DehydrationManager { private keyInfo?: {[props: string]: any}; private deviceDisplayName?: string; - constructor(private readonly crypto: Crypto) { + public constructor(private readonly crypto: Crypto) { this.getDehydrationKeyFromCache(); } @@ -294,7 +294,7 @@ export class DehydrationManager { } } - public stop() { + public stop(): void { if (this.timeoutId) { global.clearTimeout(this.timeoutId); this.timeoutId = undefined; diff --git a/src/crypto/deviceinfo.ts b/src/crypto/deviceinfo.ts index 00ebf7c389f..3b4d53f6813 100644 --- a/src/crypto/deviceinfo.ts +++ b/src/crypto/deviceinfo.ts @@ -94,7 +94,7 @@ export class DeviceInfo { public unsigned: Record = {}; public signatures: ISignatures = {}; - constructor(public readonly deviceId: string) {} + public constructor(public readonly deviceId: string) {} /** * Prepare a DeviceInfo for JSON serialisation in the session store diff --git a/src/crypto/index.ts b/src/crypto/index.ts index 2a55d75fee8..0f203bc25cc 100644 --- a/src/crypto/index.ts +++ b/src/crypto/index.ts @@ -86,6 +86,7 @@ import { ISyncStateData } from "../sync"; import { CryptoStore } from "./store/base"; import { IVerificationChannel } from "./verification/request/Channel"; import { TypedEventEmitter } from "../models/typed-event-emitter"; +import { IContent } from "../models/event"; const DeviceVerification = DeviceInfo.DeviceVerification; @@ -278,7 +279,7 @@ export class Crypto extends TypedEventEmitter { + cryptoCallbacks.getCrossSigningKey = async (type): Promise => { return CrossSigningInfo.getFromSecretStorage(type, this.secretStorage); }; } @@ -709,7 +710,7 @@ export class Crypto extends TypedEventEmitter { + const resetCrossSigning = async (): Promise => { crossSigningInfo.resetKeys(); // Sign master key with device key await this.signObject(crossSigningInfo.keys.master); @@ -846,12 +847,12 @@ export class Crypto extends TypedEventEmitter ({} as IRecoveryKey), + createSecretStorageKey = async (): Promise => ({} as IRecoveryKey), keyBackupInfo, setupNewKeyBackup, setupNewSecretStorage, getKeyBackupPassphrase, - }: ICreateSecretStorageOpts = {}) { + }: ICreateSecretStorageOpts = {}): Promise { logger.log("Bootstrapping Secure Secret Storage"); const delegateCryptoCallbacks = this.baseApis.cryptoCallbacks; const builder = new EncryptionSetupBuilder( @@ -868,7 +869,7 @@ export class Crypto extends TypedEventEmitter { + const createSSSS = async (opts: IAddSecretStorageKeyOpts, privateKey?: Uint8Array): Promise => { if (privateKey) { opts.key = privateKey; } @@ -884,7 +885,7 @@ export class Crypto extends TypedEventEmitter { + const ensureCanCheckPassphrase = async (keyId: string, keyInfo: ISecretStorageKeyInfo): Promise => { if (!keyInfo.mac) { const key = await this.baseApis.cryptoCallbacks.getSecretStorageKey?.( { keys: { [keyId]: keyInfo } }, "", @@ -903,7 +904,7 @@ export class Crypto extends TypedEventEmitter { + const signKeyBackupWithCrossSigning = async (keyBackupAuthData: IKeyBackupInfo["auth_data"]): Promise => { if ( this.crossSigningInfo.getId() && await this.crossSigningInfo.isStoredInKeyCache("master") @@ -1241,7 +1242,7 @@ export class Crypto extends TypedEventEmitter { + const upload = ({ shouldEmit = false }): Promise => { return this.baseApis.uploadKeySignatures({ [this.userId]: { [this.deviceId]: signedDevice!, @@ -1475,7 +1476,7 @@ export class Crypto extends TypedEventEmitter { + private onDeviceListUserCrossSigningUpdated = async (userId: string): Promise => { if (userId === this.userId) { // An update to our own cross-signing key. // Get the new key first: @@ -1657,7 +1658,7 @@ export class Crypto extends TypedEventEmitter { + const upload = ({ shouldEmit = false }): Promise => { logger.info(`Starting background key sig upload for ${keysToUpload}`); return this.baseApis.uploadKeySignatures({ [this.userId]: keySignatures }) .then((response) => { @@ -1864,7 +1865,7 @@ export class Crypto extends TypedEventEmitter { + const uploadLoop = async (keyCount: number): Promise => { while (keyLimit > keyCount || this.getNeedsNewFallback()) { // Ask olm to generate new one time keys, then upload them to synapse. if (keyLimit > keyCount) { @@ -2155,7 +2156,7 @@ export class Crypto extends TypedEventEmitter { + const upload = async ({ shouldEmit = false }): Promise => { logger.info("Uploading signature for " + userId + "..."); const response = await this.baseApis.uploadKeySignatures({ [userId]: { @@ -2246,7 +2247,7 @@ export class Crypto extends TypedEventEmitter { + const upload = async ({ shouldEmit = false }): Promise => { logger.info("Uploading signature for " + deviceId); const response = await this.baseApis.uploadKeySignatures({ [userId]: { @@ -2552,12 +2553,45 @@ export class Crypto extends TypedEventEmitter { + const room = this.clientStore.getRoom(roomId); + if (!room) { + throw new Error(`Unable to enable encryption tracking devices in unknown room ${roomId}`); + } + await this.setRoomEncryptionImpl(room, config); + if (!this.lazyLoadMembers && !inhibitDeviceQuery) { + this.deviceList.refreshOutdatedDeviceLists(); + } + } + + /** + * Set up encryption for a room. + * + * This is called when an m.room.encryption event is received. It saves a flag + * for the room in the cryptoStore (if it wasn't already set), sets up an "encryptor" for + * the room, and enables device-list tracking for the room. + * + * It does not initiate a device list query for the room. That is normally + * done once we finish processing the sync, in onSyncCompleted. + * + * @param room The room to enable encryption in. + * @param config The encryption config for the room. + */ + private async setRoomEncryptionImpl( + room: Room, + config: IRoomEncryption, + ): Promise { + const roomId = room.roomId; + // ignore crypto events with no algorithm defined // This will happen if a crypto event is redacted before we fetch the room state // It would otherwise just throw later as an unknown algorithm would, but we may @@ -2625,14 +2659,7 @@ export class Crypto extends TypedEventEmitter { - const trackMembers = async () => { + const room = this.clientStore.getRoom(roomId); + if (!room) { + throw new Error(`Unable to start tracking devices in unknown room ${roomId}`); + } + return this.trackRoomDevicesImpl(room); + } + + /** + * Make sure we are tracking the device lists for all users in this room. + * + * This is normally called when we are about to send an encrypted event, to make sure + * we have all the devices in the room; but it is also called when processing an + * m.room.encryption state event (if lazy-loading is disabled), or when members are + * loaded (if lazy-loading is enabled), to prepare the device list. + * + * @param room Room to enable device-list tracking in + */ + private trackRoomDevicesImpl(room: Room): Promise { + const roomId = room.roomId; + const trackMembers = async (): Promise => { // not an encrypted room if (!this.roomEncryptors.has(roomId)) { return; } - const room = this.clientStore.getRoom(roomId); - if (!room) { - throw new Error(`Unable to start tracking devices in unknown room ${roomId}`); - } logger.log(`Starting to track devices for room ${roomId} ...`); const members = await room.getEncryptionTargetMembers(); members.forEach((m) => { @@ -2747,7 +2789,7 @@ export class Crypto extends TypedEventEmitter { - const roomId = event.getRoomId()!; + public async onCryptoEvent(room: Room, event: MatrixEvent): Promise { const content = event.getContent(); - - try { - // inhibit the device list refresh for now - it will happen once we've - // finished processing the sync, in onSyncCompleted. - await this.setRoomEncryption(roomId, content, true); - } catch (e) { - logger.error(`Error configuring encryption in room ${roomId}`, e); - } + await this.setRoomEncryptionImpl(room, content); } /** @@ -3187,7 +3219,7 @@ export class Crypto extends TypedEventEmitter { + private onMembership = (event: MatrixEvent, member: RoomMember, oldMembership?: string): void => { try { this.onRoomMembership(event, member, oldMembership); } catch (e) { @@ -3269,9 +3301,9 @@ export class Crypto extends TypedEventEmitter { + const createRequest = (event: MatrixEvent): VerificationRequest => { const channel = new InRoomChannel(this.baseApis, event.getRoomId()!); return new VerificationRequest( channel, this.verificationMethods, this.baseApis); @@ -3361,7 +3393,7 @@ export class Crypto extends TypedEventEmitter((resolve, reject) => { eventIdListener = resolve; - statusListener = () => { + statusListener = (): void => { if (event.status == EventStatus.CANCELLED) { reject(new Error("Event status set to CANCELLED.")); } @@ -3420,7 +3452,7 @@ export class Crypto extends TypedEventEmitter { + const retryDecryption = (): void => { const roomDecryptors = this.getRoomDecryptors(olmlib.MEGOLM_ALGORITHM); for (const decryptor of roomDecryptors) { decryptor.retryDecryptionFromSender(deviceKey); @@ -3692,7 +3724,7 @@ export class Crypto extends TypedEventEmitter { + req.share = (): void => { decryptor.shareKeysWithDevice(req); }; @@ -3863,14 +3895,14 @@ export class IncomingRoomKeyRequest { public readonly requestBody: IRoomKeyRequestBody; public share: () => void; - constructor(event: MatrixEvent) { + public constructor(event: MatrixEvent) { const content = event.getContent(); this.userId = event.getSender()!; this.deviceId = content.requesting_device_id; this.requestId = content.request_id; this.requestBody = content.body || {}; - this.share = () => { + this.share = (): void => { throw new Error("don't know how to share keys for this request yet"); }; } @@ -3888,7 +3920,7 @@ class IncomingRoomKeyRequestCancellation { public readonly deviceId: string; public readonly requestId: string; - constructor(event: MatrixEvent) { + public constructor(event: MatrixEvent) { const content = event.getContent(); this.userId = event.getSender()!; diff --git a/src/crypto/olmlib.ts b/src/crypto/olmlib.ts index b77c0b16f58..111e4c16de3 100644 --- a/src/crypto/olmlib.ts +++ b/src/crypto/olmlib.ts @@ -82,7 +82,7 @@ export async function encryptMessageForDevice( recipientUserId: string, recipientDevice: DeviceInfo, payloadFields: Record, -) { +): Promise { const deviceKey = recipientDevice.getIdentityKey(); const sessionId = await olmDevice.getSessionIdForDevice(deviceKey); if (sessionId === null) { @@ -173,7 +173,7 @@ export async function getExistingOlmSessions( for (const deviceInfo of devices) { const deviceId = deviceInfo.deviceId; const key = deviceInfo.getIdentityKey(); - promises.push((async () => { + promises.push((async (): Promise => { const sessionId = await olmDevice.getSessionIdForDevice( key, true, ); @@ -256,7 +256,7 @@ export async function ensureOlmSessionsForDevices( // conditions. If we find that we already have a session, then // we'll resolve olmDevice.sessionsInProgress[key] = new Promise(resolve => { - resolveSession[key] = (v: any) => { + resolveSession[key] = (v: any): void => { delete olmDevice.sessionsInProgress[key]; resolve(v); }; @@ -463,7 +463,7 @@ export async function verifySignature( signingUserId: string, signingDeviceId: string, signingKey: string, -) { +): Promise { const signKeyId = "ed25519:" + signingDeviceId; const signatures = obj.signatures || {}; const userSigs = signatures[signingUserId] || {}; @@ -527,7 +527,7 @@ export function pkSign(obj: IObject, key: PkSigning, userId: string, pubKey: str * @param {string} pubKey The public key to use to verify * @param {string} userId The user ID who signed the object */ -export function pkVerify(obj: IObject, pubKey: string, userId: string) { +export function pkVerify(obj: IObject, pubKey: string, userId: string): void { const keyId = "ed25519:" + pubKey; if (!(obj.signatures && obj.signatures[userId] && obj.signatures[userId][keyId])) { throw new Error("No signature"); diff --git a/src/crypto/store/indexeddb-crypto-store-backend.ts b/src/crypto/store/indexeddb-crypto-store-backend.ts index f525c0c959a..cdf35e787a5 100644 --- a/src/crypto/store/indexeddb-crypto-store-backend.ts +++ b/src/crypto/store/indexeddb-crypto-store-backend.ts @@ -48,11 +48,11 @@ export class Backend implements CryptoStore { /** * @param {IDBDatabase} db */ - constructor(private db: IDBDatabase) { + public constructor(private db: IDBDatabase) { // make sure we close the db on `onversionchange` - otherwise // attempts to delete the database will block (and subsequent // attempts to re-create it will also block). - db.onversionchange = () => { + db.onversionchange = (): void => { logger.log(`versionchange for indexeddb ${this.db.name}: closing`); db.close(); }; @@ -103,7 +103,7 @@ export class Backend implements CryptoStore { `enqueueing key request for ${requestBody.room_id} / ` + requestBody.session_id, ); - txn.oncomplete = () => {resolve(request);}; + txn.oncomplete = (): void => {resolve(request);}; const store = txn.objectStore("outgoingRoomKeyRequests"); store.add(request); }); @@ -157,7 +157,7 @@ export class Backend implements CryptoStore { requestBody.session_id, ]); - cursorReq.onsuccess = () => { + cursorReq.onsuccess = (): void => { const cursor = cursorReq.result; if (!cursor) { // no match found @@ -201,7 +201,7 @@ export class Backend implements CryptoStore { let stateIndex = 0; let result: OutgoingRoomKeyRequest; - function onsuccess(this: IDBRequest) { + function onsuccess(this: IDBRequest): void { const cursor = this.result; if (cursor) { // got a match @@ -243,8 +243,8 @@ export class Backend implements CryptoStore { const index = store.index("state"); const request = index.getAll(wantedState); - request.onsuccess = () => resolve(request.result); - request.onerror = () => reject(request.error); + request.onsuccess = (): void => resolve(request.result); + request.onerror = (): void => reject(request.error); }); } @@ -256,7 +256,7 @@ export class Backend implements CryptoStore { let stateIndex = 0; const results: OutgoingRoomKeyRequest[] = []; - function onsuccess(this: IDBRequest) { + function onsuccess(this: IDBRequest): void { const cursor = this.result; if (cursor) { const keyReq = cursor.value; @@ -309,7 +309,7 @@ export class Backend implements CryptoStore { ): Promise { let result: OutgoingRoomKeyRequest | null = null; - function onsuccess(this: IDBRequest) { + function onsuccess(this: IDBRequest): void { const cursor = this.result; if (!cursor) { return; @@ -348,7 +348,7 @@ export class Backend implements CryptoStore { ): Promise { const txn = this.db.transaction("outgoingRoomKeyRequests", "readwrite"); const cursorReq = txn.objectStore("outgoingRoomKeyRequests").openCursor(requestId); - cursorReq.onsuccess = () => { + cursorReq.onsuccess = (): void => { const cursor = cursorReq.result; if (!cursor) { return; @@ -371,7 +371,7 @@ export class Backend implements CryptoStore { public getAccount(txn: IDBTransaction, func: (accountPickle: string | null) => void): void { const objectStore = txn.objectStore("account"); const getReq = objectStore.get("-"); - getReq.onsuccess = function() { + getReq.onsuccess = function(): void { try { func(getReq.result || null); } catch (e) { @@ -391,7 +391,7 @@ export class Backend implements CryptoStore { ): void { const objectStore = txn.objectStore("account"); const getReq = objectStore.get("crossSigningKeys"); - getReq.onsuccess = function() { + getReq.onsuccess = function(): void { try { func(getReq.result || null); } catch (e) { @@ -407,7 +407,7 @@ export class Backend implements CryptoStore { ): void { const objectStore = txn.objectStore("account"); const getReq = objectStore.get(`ssss_cache:${type}`); - getReq.onsuccess = function() { + getReq.onsuccess = function(): void { try { func(getReq.result || null); } catch (e) { @@ -435,7 +435,7 @@ export class Backend implements CryptoStore { public countEndToEndSessions(txn: IDBTransaction, func: (count: number) => void): void { const objectStore = txn.objectStore("sessions"); const countReq = objectStore.count(); - countReq.onsuccess = function() { + countReq.onsuccess = function(): void { try { func(countReq.result); } catch (e) { @@ -453,7 +453,7 @@ export class Backend implements CryptoStore { const idx = objectStore.index("deviceKey"); const getReq = idx.openCursor(deviceKey); const results: Parameters[2]>[0] = {}; - getReq.onsuccess = function() { + getReq.onsuccess = function(): void { const cursor = getReq.result; if (cursor) { results[cursor.value.sessionId] = { @@ -479,7 +479,7 @@ export class Backend implements CryptoStore { ): void { const objectStore = txn.objectStore("sessions"); const getReq = objectStore.get([deviceKey, sessionId]); - getReq.onsuccess = function() { + getReq.onsuccess = function(): void { try { if (getReq.result) { func({ @@ -498,7 +498,7 @@ export class Backend implements CryptoStore { public getAllEndToEndSessions(txn: IDBTransaction, func: (session: ISessionInfo | null) => void): void { const objectStore = txn.objectStore("sessions"); const getReq = objectStore.openCursor(); - getReq.onsuccess = function() { + getReq.onsuccess = function(): void { try { const cursor = getReq.result; if (cursor) { @@ -546,7 +546,7 @@ export class Backend implements CryptoStore { const objectStore = txn.objectStore("session_problems"); const index = objectStore.index("deviceKey"); const req = index.getAll(deviceKey); - req.onsuccess = () => { + req.onsuccess = (): void => { const problems = req.result; if (!problems.length) { result = null; @@ -583,7 +583,7 @@ export class Backend implements CryptoStore { return new Promise((resolve) => { const { userId, deviceInfo } = device; const getReq = objectStore.get([userId, deviceInfo.deviceId]); - getReq.onsuccess = function() { + getReq.onsuccess = function(): void { if (!getReq.result) { objectStore.put({ userId, deviceId: deviceInfo.deviceId }); ret.push(device); @@ -608,7 +608,7 @@ export class Backend implements CryptoStore { let withheld: IWithheld | null | boolean = false; const objectStore = txn.objectStore("inbound_group_sessions"); const getReq = objectStore.get([senderCurve25519Key, sessionId]); - getReq.onsuccess = function() { + getReq.onsuccess = function(): void { try { if (getReq.result) { session = getReq.result.session; @@ -625,7 +625,7 @@ export class Backend implements CryptoStore { const withheldObjectStore = txn.objectStore("inbound_group_sessions_withheld"); const withheldGetReq = withheldObjectStore.get([senderCurve25519Key, sessionId]); - withheldGetReq.onsuccess = function() { + withheldGetReq.onsuccess = function(): void { try { if (withheldGetReq.result) { withheld = withheldGetReq.result.session; @@ -644,7 +644,7 @@ export class Backend implements CryptoStore { public getAllEndToEndInboundGroupSessions(txn: IDBTransaction, func: (session: ISession | null) => void): void { const objectStore = txn.objectStore("inbound_group_sessions"); const getReq = objectStore.openCursor(); - getReq.onsuccess = function() { + getReq.onsuccess = function(): void { const cursor = getReq.result; if (cursor) { try { @@ -677,7 +677,7 @@ export class Backend implements CryptoStore { const addReq = objectStore.add({ senderCurve25519Key, sessionId, session: sessionData, }); - addReq.onerror = (ev) => { + addReq.onerror = (ev): void => { if (addReq.error?.name === 'ConstraintError') { // This stops the error from triggering the txn's onerror ev.stopPropagation(); @@ -722,7 +722,7 @@ export class Backend implements CryptoStore { public getEndToEndDeviceData(txn: IDBTransaction, func: (deviceData: IDeviceData | null) => void): void { const objectStore = txn.objectStore("device_data"); const getReq = objectStore.get("-"); - getReq.onsuccess = function() { + getReq.onsuccess = function(): void { try { func(getReq.result || null); } catch (e) { @@ -745,7 +745,7 @@ export class Backend implements CryptoStore { const rooms: Parameters[1]>[0] = {}; const objectStore = txn.objectStore("rooms"); const getReq = objectStore.openCursor(); - getReq.onsuccess = function() { + getReq.onsuccess = function(): void { const cursor = getReq.result; if (cursor) { rooms[cursor.key as string] = cursor.value; @@ -771,17 +771,17 @@ export class Backend implements CryptoStore { "readonly", ); txn.onerror = reject; - txn.oncomplete = function() { + txn.oncomplete = function(): void { resolve(sessions); }; const objectStore = txn.objectStore("sessions_needing_backup"); const sessionStore = txn.objectStore("inbound_group_sessions"); const getReq = objectStore.openCursor(); - getReq.onsuccess = function() { + getReq.onsuccess = function(): void { const cursor = getReq.result; if (cursor) { const sessionGetReq = sessionStore.get(cursor.key); - sessionGetReq.onsuccess = function() { + sessionGetReq.onsuccess = function(): void { sessions.push({ senderKey: sessionGetReq.result.senderCurve25519Key, sessionId: sessionGetReq.result.sessionId, @@ -804,7 +804,7 @@ export class Backend implements CryptoStore { return new Promise((resolve, reject) => { const req = objectStore.count(); req.onerror = reject; - req.onsuccess = () => resolve(req.result); + req.onsuccess = (): void => resolve(req.result); }); } @@ -852,7 +852,7 @@ export class Backend implements CryptoStore { } const objectStore = txn.objectStore("shared_history_inbound_group_sessions"); const req = objectStore.get([roomId]); - req.onsuccess = () => { + req.onsuccess = (): void => { const { sessions } = req.result || { sessions: [] }; sessions.push([senderKey, sessionId]); objectStore.put({ roomId, sessions }); @@ -871,7 +871,7 @@ export class Backend implements CryptoStore { const objectStore = txn.objectStore("shared_history_inbound_group_sessions"); const req = objectStore.get([roomId]); return new Promise((resolve, reject) => { - req.onsuccess = () => { + req.onsuccess = (): void => { const { sessions } = req.result || { sessions: [] }; resolve(sessions); }; @@ -891,7 +891,7 @@ export class Backend implements CryptoStore { } const objectStore = txn.objectStore("parked_shared_history"); const req = objectStore.get([roomId]); - req.onsuccess = () => { + req.onsuccess = (): void => { const { parked } = req.result || { parked: [] }; parked.push(parkedData); objectStore.put({ roomId, parked }); @@ -909,7 +909,7 @@ export class Backend implements CryptoStore { } const cursorReq = txn.objectStore("parked_shared_history").openCursor(roomId); return new Promise((resolve, reject) => { - cursorReq.onsuccess = () => { + cursorReq.onsuccess = (): void => { const cursor = cursorReq.result; if (!cursor) { resolve([]); @@ -957,32 +957,32 @@ export class Backend implements CryptoStore { type DbMigration = (db: IDBDatabase) => void; const DB_MIGRATIONS: DbMigration[] = [ - (db) => { createDatabase(db); }, - (db) => { db.createObjectStore("account"); }, - (db) => { + (db): void => { createDatabase(db); }, + (db): void => { db.createObjectStore("account"); }, + (db): void => { const sessionsStore = db.createObjectStore("sessions", { keyPath: ["deviceKey", "sessionId"], }); sessionsStore.createIndex("deviceKey", "deviceKey"); }, - (db) => { + (db): void => { db.createObjectStore("inbound_group_sessions", { keyPath: ["senderCurve25519Key", "sessionId"], }); }, - (db) => { db.createObjectStore("device_data"); }, - (db) => { db.createObjectStore("rooms"); }, - (db) => { + (db): void => { db.createObjectStore("device_data"); }, + (db): void => { db.createObjectStore("rooms"); }, + (db): void => { db.createObjectStore("sessions_needing_backup", { keyPath: ["senderCurve25519Key", "sessionId"], }); }, - (db) => { + (db): void => { db.createObjectStore("inbound_group_sessions_withheld", { keyPath: ["senderCurve25519Key", "sessionId"], }); }, - (db) => { + (db): void => { const problemsStore = db.createObjectStore("session_problems", { keyPath: ["deviceKey", "time"], }); @@ -992,12 +992,12 @@ const DB_MIGRATIONS: DbMigration[] = [ keyPath: ["userId", "deviceId"], }); }, - (db) => { + (db): void => { db.createObjectStore("shared_history_inbound_group_sessions", { keyPath: ["roomId"], }); }, - (db) => { + (db): void => { db.createObjectStore("parked_shared_history", { keyPath: ["roomId"], }); @@ -1037,7 +1037,7 @@ interface IWrappedIDBTransaction extends IDBTransaction { * Aborts a transaction with a given exception * The transaction promise will be rejected with this exception. */ -function abortWithException(txn: IDBTransaction, e: Error) { +function abortWithException(txn: IDBTransaction, e: Error): void { // We cheekily stick our exception onto the transaction object here // We could alternatively make the thing we pass back to the app // an object containing the transaction and exception. @@ -1052,13 +1052,13 @@ function abortWithException(txn: IDBTransaction, e: Error) { function promiseifyTxn(txn: IDBTransaction): Promise { return new Promise((resolve, reject) => { - txn.oncomplete = () => { + txn.oncomplete = (): void => { if ((txn as IWrappedIDBTransaction)._mx_abortexception !== undefined) { reject((txn as IWrappedIDBTransaction)._mx_abortexception); } resolve(null); }; - txn.onerror = (event) => { + txn.onerror = (event): void => { if ((txn as IWrappedIDBTransaction)._mx_abortexception !== undefined) { reject((txn as IWrappedIDBTransaction)._mx_abortexception); } else { @@ -1066,7 +1066,7 @@ function promiseifyTxn(txn: IDBTransaction): Promise { reject(txn.error); } }; - txn.onabort = (event) => { + txn.onabort = (event): void => { if ((txn as IWrappedIDBTransaction)._mx_abortexception !== undefined) { reject((txn as IWrappedIDBTransaction)._mx_abortexception); } else { diff --git a/src/crypto/store/indexeddb-crypto-store.ts b/src/crypto/store/indexeddb-crypto-store.ts index 4fbeafe19df..d743a95decc 100644 --- a/src/crypto/store/indexeddb-crypto-store.ts +++ b/src/crypto/store/indexeddb-crypto-store.ts @@ -73,7 +73,7 @@ export class IndexedDBCryptoStore implements CryptoStore { * @param {IDBFactory} indexedDB global indexedDB instance * @param {string} dbName name of db to connect to */ - constructor(private readonly indexedDB: IDBFactory, private readonly dbName: string) {} + public constructor(private readonly indexedDB: IDBFactory, private readonly dbName: string) {} /** * Ensure the database exists and is up-to-date, or fall back to @@ -99,24 +99,24 @@ export class IndexedDBCryptoStore implements CryptoStore { const req = this.indexedDB.open(this.dbName, IndexedDBCryptoStoreBackend.VERSION); - req.onupgradeneeded = (ev) => { + req.onupgradeneeded = (ev): void => { const db = req.result; const oldVersion = ev.oldVersion; IndexedDBCryptoStoreBackend.upgradeDatabase(db, oldVersion); }; - req.onblocked = () => { + req.onblocked = (): void => { logger.log( `can't yet open IndexedDBCryptoStore because it is open elsewhere`, ); }; - req.onerror = (ev) => { + req.onerror = (ev): void => { logger.log("Error connecting to indexeddb", ev); reject(req.error); }; - req.onsuccess = () => { + req.onsuccess = (): void => { const db = req.result; logger.log(`connected to indexeddb ${this.dbName}`); @@ -179,18 +179,18 @@ export class IndexedDBCryptoStore implements CryptoStore { logger.log(`Removing indexeddb instance: ${this.dbName}`); const req = this.indexedDB.deleteDatabase(this.dbName); - req.onblocked = () => { + req.onblocked = (): void => { logger.log( `can't yet delete IndexedDBCryptoStore because it is open elsewhere`, ); }; - req.onerror = (ev) => { + req.onerror = (ev): void => { logger.log("Error deleting data from indexeddb", ev); reject(req.error); }; - req.onsuccess = () => { + req.onsuccess = (): void => { logger.log(`Removed indexeddb instance: ${this.dbName}`); resolve(); }; @@ -322,7 +322,7 @@ export class IndexedDBCryptoStore implements CryptoStore { * @param {*} txn An active transaction. See doTxn(). * @param {function(string)} func Called with the account pickle */ - public getAccount(txn: IDBTransaction, func: (accountPickle: string | null) => void) { + public getAccount(txn: IDBTransaction, func: (accountPickle: string | null) => void): void { this.backend!.getAccount(txn, func); } diff --git a/src/crypto/store/localStorage-crypto-store.ts b/src/crypto/store/localStorage-crypto-store.ts index 4bc8b45ef7f..977236ef93f 100644 --- a/src/crypto/store/localStorage-crypto-store.ts +++ b/src/crypto/store/localStorage-crypto-store.ts @@ -76,7 +76,7 @@ export class LocalStorageCryptoStore extends MemoryCryptoStore { return false; } - constructor(private readonly store: Storage) { + public constructor(private readonly store: Storage) { super(); } @@ -154,7 +154,7 @@ export class LocalStorageCryptoStore extends MemoryCryptoStore { setJsonItem(this.store, key, problems); } - async getEndToEndSessionProblem(deviceKey: string, timestamp: number): Promise { + public async getEndToEndSessionProblem(deviceKey: string, timestamp: number): Promise { const key = keyEndToEndSessionProblems(deviceKey); const problems = getJsonItem(this.store, key) || []; if (!problems.length) { @@ -408,7 +408,7 @@ export class LocalStorageCryptoStore extends MemoryCryptoStore { setJsonItem(this.store, E2E_PREFIX + `ssss_cache.${type}`, key); } - doTxn(mode: Mode, stores: Iterable, func: (txn: unknown) => T): Promise { + public doTxn(mode: Mode, stores: Iterable, func: (txn: unknown) => T): Promise { return Promise.resolve(func(null)); } } diff --git a/src/crypto/store/memory-crypto-store.ts b/src/crypto/store/memory-crypto-store.ts index 8b0206eef7a..f22379ee81e 100644 --- a/src/crypto/store/memory-crypto-store.ts +++ b/src/crypto/store/memory-crypto-store.ts @@ -279,7 +279,7 @@ export class MemoryCryptoStore implements CryptoStore { // Olm Account - public getAccount(txn: unknown, func: (accountPickle: string | null) => void) { + public getAccount(txn: unknown, func: (accountPickle: string | null) => void): void { func(this.account); } diff --git a/src/crypto/verification/Base.ts b/src/crypto/verification/Base.ts index f2fc1d5f99a..55b349e99c9 100644 --- a/src/crypto/verification/Base.ts +++ b/src/crypto/verification/Base.ts @@ -34,7 +34,7 @@ import { ListenerMap, TypedEventEmitter } from "../../models/typed-event-emitter const timeoutException = new Error("Verification timed out"); export class SwitchStartEventError extends Error { - constructor(public readonly startEvent: MatrixEvent | null) { + public constructor(public readonly startEvent: MatrixEvent | null) { super(); } } @@ -91,7 +91,7 @@ export class VerificationBase< * @param {object} [request] the key verification request object related to * this verification, if any */ - constructor( + public constructor( public readonly channel: IVerificationChannel, public readonly baseApis: MatrixClient, public readonly userId: string, @@ -286,12 +286,12 @@ export class VerificationBase< if (this.promise) return this.promise; this.promise = new Promise((resolve, reject) => { - this.resolve = (...args) => { + this.resolve = (...args): void => { this._done = true; this.endTimer(); resolve(...args); }; - this.reject = (e: Error | MatrixEvent) => { + this.reject = (e: Error | MatrixEvent): void => { this._done = true; this.endTimer(); reject(e); diff --git a/src/crypto/verification/QRCode.ts b/src/crypto/verification/QRCode.ts index 9ff9315a95f..f6bdda17e1a 100644 --- a/src/crypto/verification/QRCode.ts +++ b/src/crypto/verification/QRCode.ts @@ -154,7 +154,7 @@ interface IQrData { } export class QRCodeData { - constructor( + public constructor( public readonly mode: Mode, private readonly sharedSecret: string, // only set when mode is MODE_VERIFY_OTHER_USER, master key of other party at time of generating QR code @@ -283,21 +283,21 @@ export class QRCodeData { private static generateBuffer(qrData: IQrData): Buffer { let buf = Buffer.alloc(0); // we'll concat our way through life - const appendByte = (b) => { + const appendByte = (b): void => { const tmpBuf = Buffer.from([b]); buf = Buffer.concat([buf, tmpBuf]); }; - const appendInt = (i) => { + const appendInt = (i): void => { const tmpBuf = Buffer.alloc(2); tmpBuf.writeInt16BE(i, 0); buf = Buffer.concat([buf, tmpBuf]); }; - const appendStr = (s, enc, withLengthPrefix = true) => { + const appendStr = (s, enc, withLengthPrefix = true): void => { const tmpBuf = Buffer.from(s, enc); if (withLengthPrefix) appendInt(tmpBuf.byteLength); buf = Buffer.concat([buf, tmpBuf]); }; - const appendEncBase64 = (b64) => { + const appendEncBase64 = (b64): void => { const b = decodeBase64(b64); const tmpBuf = Buffer.from(b); buf = Buffer.concat([buf, tmpBuf]); diff --git a/src/crypto/verification/SAS.ts b/src/crypto/verification/SAS.ts index 806b4bf5ec5..5df6d48f6c8 100644 --- a/src/crypto/verification/SAS.ts +++ b/src/crypto/verification/SAS.ts @@ -170,10 +170,12 @@ const macMethods = { "hmac-sha256": "calculate_mac_long_kdf", }; -function calculateMAC(olmSAS: OlmSAS, method: string) { - return function(...args) { +type Method = keyof typeof macMethods; + +function calculateMAC(olmSAS: OlmSAS, method: Method) { + return function(...args): string { const macFunction = olmSAS[macMethods[method]]; - const mac = macFunction.apply(olmSAS, args); + const mac: string = macFunction.apply(olmSAS, args); logger.log("SAS calculateMAC:", method, args, mac); return mac; }; @@ -208,7 +210,7 @@ const calculateKeyAgreement = { */ const KEY_AGREEMENT_LIST = ["curve25519-hkdf-sha256", "curve25519"]; const HASHES_LIST = ["sha256"]; -const MAC_LIST = ["org.matrix.msc3783.hkdf-hmac-sha256", "hkdf-hmac-sha256", "hmac-sha256"]; +const MAC_LIST: Method[] = ["org.matrix.msc3783.hkdf-hmac-sha256", "hkdf-hmac-sha256", "hmac-sha256"]; const SAS_LIST = Object.keys(sasGenerators); const KEY_AGREEMENT_SET = new Set(KEY_AGREEMENT_LIST); @@ -300,13 +302,13 @@ export class SAS extends Base { keyAgreement: string, sasMethods: string[], olmSAS: OlmSAS, - macMethod: string, + macMethod: Method, ): Promise { const sasBytes = calculateKeyAgreement[keyAgreement](this, olmSAS, 6); const verifySAS = new Promise((resolve, reject) => { this.sasEvent = { sas: generateSas(sasBytes, sasMethods), - confirm: async () => { + confirm: async (): Promise => { try { await this.sendMAC(olmSAS, macMethod); resolve(); @@ -443,7 +445,7 @@ export class SAS extends Base { } } - private sendMAC(olmSAS: OlmSAS, method: string): Promise { + private sendMAC(olmSAS: OlmSAS, method: Method): Promise { const mac = {}; const keyList: string[] = []; const baseInfo = "MATRIX_KEY_VERIFICATION_MAC" @@ -475,7 +477,7 @@ export class SAS extends Base { return this.send(EventType.KeyVerificationMac, { mac, keys }); } - private async checkMAC(olmSAS: OlmSAS, content: IContent, method: string): Promise { + private async checkMAC(olmSAS: OlmSAS, content: IContent, method: Method): Promise { const baseInfo = "MATRIX_KEY_VERIFICATION_MAC" + this.userId + this.deviceId + this.baseApis.getUserId() + this.baseApis.deviceId diff --git a/src/crypto/verification/request/InRoomChannel.ts b/src/crypto/verification/request/InRoomChannel.ts index 2a4fb8d8106..664a2dad6a8 100644 --- a/src/crypto/verification/request/InRoomChannel.ts +++ b/src/crypto/verification/request/InRoomChannel.ts @@ -44,7 +44,7 @@ export class InRoomChannel implements IVerificationChannel { * @param {string} roomId id of the room where verification events should be posted in, should be a DM with the given user. * @param {string} userId id of user that the verification request is directed at, should be present in the room. */ - constructor( + public constructor( private readonly client: MatrixClient, public readonly roomId: string, public userId?: string, diff --git a/src/crypto/verification/request/ToDeviceChannel.ts b/src/crypto/verification/request/ToDeviceChannel.ts index 11227291483..30d61525139 100644 --- a/src/crypto/verification/request/ToDeviceChannel.ts +++ b/src/crypto/verification/request/ToDeviceChannel.ts @@ -42,7 +42,7 @@ export class ToDeviceChannel implements IVerificationChannel { public request?: VerificationRequest; // userId and devices of user we're about to verify - constructor( + public constructor( private readonly client: MatrixClient, public readonly userId: string, private readonly devices: string[], diff --git a/src/crypto/verification/request/VerificationRequest.ts b/src/crypto/verification/request/VerificationRequest.ts index 2ff71d62c19..58de4b9a2b0 100644 --- a/src/crypto/verification/request/VerificationRequest.ts +++ b/src/crypto/verification/request/VerificationRequest.ts @@ -116,7 +116,7 @@ export class VerificationRequest< public _cancellingUserId?: string; // Used in tests only private _verifier?: VerificationBase; - constructor( + public constructor( public readonly channel: C, private readonly verificationMethods: Map, private readonly client: MatrixClient, @@ -498,7 +498,7 @@ export class VerificationRequest< */ public waitFor(fn: (request: VerificationRequest) => boolean): Promise { return new Promise((resolve, reject) => { - const check = () => { + const check = (): boolean => { let handled = false; if (fn(this)) { resolve(this); @@ -539,7 +539,7 @@ export class VerificationRequest< private calculatePhaseTransitions(): ITransition[] { const transitions: ITransition[] = [{ phase: PHASE_UNSENT }]; - const phase = () => transitions[transitions.length - 1].phase; + const phase = (): Phase => transitions[transitions.length - 1].phase; // always pass by .request first to be sure channel.userId has been set const hasRequestByThem = this.eventsByThem.has(REQUEST_TYPE); @@ -816,7 +816,7 @@ export class VerificationRequest< } } - private cancelOnTimeout = async () => { + private cancelOnTimeout = async (): Promise => { try { if (this.initiatedByMe) { await this.cancel({ diff --git a/src/embedded.ts b/src/embedded.ts index d1dfb50bf2f..27cf564a670 100644 --- a/src/embedded.ts +++ b/src/embedded.ts @@ -101,7 +101,7 @@ export class RoomWidgetClient extends MatrixClient { private lifecycle?: AbortController; private syncState: SyncState | null = null; - constructor( + public constructor( private readonly widgetApi: WidgetApi, private readonly capabilities: ICapabilities, private readonly roomId: string, @@ -211,7 +211,7 @@ export class RoomWidgetClient extends MatrixClient { if (this.capabilities.turnServers) this.watchTurnServers(); } - public stopClient() { + public stopClient(): void { this.widgetApi.off(`action:${WidgetApiToWidgetAction.SendEvent}`, this.onEvent); this.widgetApi.off(`action:${WidgetApiToWidgetAction.SendToDevice}`, this.onToDevice); @@ -288,7 +288,7 @@ export class RoomWidgetClient extends MatrixClient { return this.syncState; } - private setSyncState(state: SyncState) { + private setSyncState(state: SyncState): void { const oldState = this.syncState; this.syncState = state; this.emit(ClientEvent.Sync, state, oldState); @@ -298,7 +298,7 @@ export class RoomWidgetClient extends MatrixClient { await this.widgetApi.transport.reply(ev.detail, {}); } - private onEvent = async (ev: CustomEvent) => { + private onEvent = async (ev: CustomEvent): Promise => { ev.preventDefault(); // Verify the room ID matches, since it's possible for the client to @@ -317,7 +317,7 @@ export class RoomWidgetClient extends MatrixClient { await this.ack(ev); }; - private onToDevice = async (ev: CustomEvent) => { + private onToDevice = async (ev: CustomEvent): Promise => { ev.preventDefault(); const event = new MatrixEvent({ @@ -333,9 +333,11 @@ export class RoomWidgetClient extends MatrixClient { await this.ack(ev); }; - private async watchTurnServers() { + private async watchTurnServers(): Promise { const servers = this.widgetApi.getTurnServers(); - const onClientStopped = () => servers.return(undefined); + const onClientStopped = (): void => { + servers.return(undefined); + }; this.lifecycle!.signal.addEventListener("abort", onClientStopped); try { diff --git a/src/event-mapper.ts b/src/event-mapper.ts index 57d36c9f68d..6f2e25c1bdd 100644 --- a/src/event-mapper.ts +++ b/src/event-mapper.ts @@ -29,7 +29,7 @@ export function eventMapperFor(client: MatrixClient, options: MapperOpts): Event let preventReEmit = Boolean(options.preventReEmit); const decrypt = options.decrypt !== false; - function mapper(plainOldJsObject: Partial) { + function mapper(plainOldJsObject: Partial): MatrixEvent { if (options.toDevice) { delete plainOldJsObject.room_id; } diff --git a/src/filter-component.ts b/src/filter-component.ts index 759c979216e..85afb0ea707 100644 --- a/src/filter-component.ts +++ b/src/filter-component.ts @@ -73,7 +73,7 @@ export interface IFilterComponent { * @param {Object} filterJson the definition of this filter JSON, e.g. { 'contains_url': true } */ export class FilterComponent { - constructor(private filterJson: IFilterComponent, public readonly userId?: string | undefined | null) {} + public constructor(private filterJson: IFilterComponent, public readonly userId?: string | undefined | null) {} /** * Checks with the filter component matches the given event diff --git a/src/filter.ts b/src/filter.ts index fb499c4da2a..57bd0540d87 100644 --- a/src/filter.ts +++ b/src/filter.ts @@ -31,7 +31,7 @@ import { MatrixEvent } from "./models/event"; * @param {string} keyNesting * @param {*} val */ -function setProp(obj: object, keyNesting: string, val: any) { +function setProp(obj: object, keyNesting: string, val: any): void { const nestedKeys = keyNesting.split("."); let currentObj = obj; for (let i = 0; i < (nestedKeys.length - 1); i++) { @@ -88,7 +88,7 @@ interface IRoomFilter { * @prop {?string} filterId The filter ID */ export class Filter { - static LAZY_LOADING_MESSAGES_FILTER = { + public static LAZY_LOADING_MESSAGES_FILTER = { lazy_load_members: true, }; @@ -110,7 +110,7 @@ export class Filter { private roomFilter?: FilterComponent; private roomTimelineFilter?: FilterComponent; - constructor(public readonly userId: string | undefined | null, public filterId?: string) {} + public constructor(public readonly userId: string | undefined | null, public filterId?: string) {} /** * Get the ID of this filter on your homeserver (if known) @@ -132,7 +132,7 @@ export class Filter { * Set the JSON body of the filter * @param {Object} definition The filter definition */ - public setDefinition(definition: IFilterDefinition) { + public setDefinition(definition: IFilterDefinition): void { this.definition = definition; // This is all ported from synapse's FilterCollection() @@ -225,7 +225,7 @@ export class Filter { * Set the max number of events to return for each room's timeline. * @param {Number} limit The max number of events to return for each room. */ - public setTimelineLimit(limit: number) { + public setTimelineLimit(limit: number): void { setProp(this.definition, "room.timeline.limit", limit); } @@ -255,7 +255,7 @@ export class Filter { * @param {boolean} includeLeave True to make rooms the user has left appear * in responses. */ - public setIncludeLeaveRooms(includeLeave: boolean) { + public setIncludeLeaveRooms(includeLeave: boolean): void { setProp(this.definition, "room.include_leave", includeLeave); } } diff --git a/src/http-api/errors.ts b/src/http-api/errors.ts index be63c94fdf0..5ae0a0f5099 100644 --- a/src/http-api/errors.ts +++ b/src/http-api/errors.ts @@ -31,7 +31,7 @@ interface IErrorJson extends Partial { * @param {number} httpStatus The HTTP response status code. */ export class HTTPError extends Error { - constructor(msg: string, public readonly httpStatus?: number) { + public constructor(msg: string, public readonly httpStatus?: number) { super(msg); } } @@ -51,7 +51,7 @@ export class MatrixError extends HTTPError { public readonly errcode?: string; public data: IErrorJson; - constructor( + public constructor( errorJson: IErrorJson = {}, public readonly httpStatus?: number, public url?: string, @@ -79,11 +79,11 @@ export class MatrixError extends HTTPError { * @constructor */ export class ConnectionError extends Error { - constructor(message: string, cause?: Error) { + public constructor(message: string, cause?: Error) { super(message + (cause ? `: ${cause.message}` : "")); } - get name() { + public get name(): string { return "ConnectionError"; } } diff --git a/src/http-api/fetch.ts b/src/http-api/fetch.ts index 92413c094af..71ba098e303 100644 --- a/src/http-api/fetch.ts +++ b/src/http-api/fetch.ts @@ -41,7 +41,7 @@ export type ResponseType = export class FetchHttpApi { private abortController = new AbortController(); - constructor( + public constructor( private eventEmitter: TypedEventEmitter, public readonly opts: O, ) { diff --git a/src/http-api/index.ts b/src/http-api/index.ts index 0c9b0b62314..c7f782d8972 100644 --- a/src/http-api/index.ts +++ b/src/http-api/index.ts @@ -77,7 +77,7 @@ export class MatrixHttpApi extends FetchHttpApi { if (global.XMLHttpRequest) { const xhr = new global.XMLHttpRequest(); - const timeoutFn = function() { + const timeoutFn = function(): void { xhr.abort(); defer.reject(new Error("Timeout")); }; @@ -85,7 +85,7 @@ export class MatrixHttpApi extends FetchHttpApi { // set an initial timeout of 30s; we'll advance it each time we get a progress notification let timeoutTimer = callbacks.setTimeout(timeoutFn, 30000); - xhr.onreadystatechange = function() { + xhr.onreadystatechange = function(): void { switch (xhr.readyState) { case global.XMLHttpRequest.DONE: callbacks.clearTimeout(timeoutTimer); @@ -113,7 +113,7 @@ export class MatrixHttpApi extends FetchHttpApi { } }; - xhr.upload.onprogress = (ev: ProgressEvent) => { + xhr.upload.onprogress = (ev: ProgressEvent): void => { callbacks.clearTimeout(timeoutTimer); upload.loaded = ev.loaded; upload.total = ev.total; diff --git a/src/http-api/utils.ts b/src/http-api/utils.ts index c7e39477089..50466095002 100644 --- a/src/http-api/utils.ts +++ b/src/http-api/utils.ts @@ -36,13 +36,13 @@ export function anySignal(signals: AbortSignal[]): { } { const controller = new AbortController(); - function cleanup() { + function cleanup(): void { for (const signal of signals) { signal.removeEventListener("abort", onAbort); } } - function onAbort() { + function onAbort(): void { controller.abort(); cleanup(); } diff --git a/src/indexeddb-helpers.ts b/src/indexeddb-helpers.ts index 84de7c0e9ef..695a3145ff2 100644 --- a/src/indexeddb-helpers.ts +++ b/src/indexeddb-helpers.ts @@ -26,13 +26,13 @@ export function exists(indexedDB: IDBFactory, dbName: string): Promise return new Promise((resolve, reject) => { let exists = true; const req = indexedDB.open(dbName); - req.onupgradeneeded = () => { + req.onupgradeneeded = (): void => { // Since we did not provide an explicit version when opening, this event // should only fire if the DB did not exist before at any version. exists = false; }; - req.onblocked = () => reject(req.error); - req.onsuccess = () => { + req.onblocked = (): void => reject(req.error); + req.onsuccess = (): void => { const db = req.result; db.close(); if (!exists) { @@ -45,6 +45,6 @@ export function exists(indexedDB: IDBFactory, dbName: string): Promise } resolve(exists); }; - req.onerror = ev => reject(req.error); + req.onerror = (): void => reject(req.error); }); } diff --git a/src/interactive-auth.ts b/src/interactive-auth.ts index 80f3e95cf40..df4a42c2726 100644 --- a/src/interactive-auth.ts +++ b/src/interactive-auth.ts @@ -100,7 +100,7 @@ class NoAuthFlowFoundError extends Error { public name = "NoAuthFlowFoundError"; // eslint-disable-next-line @typescript-eslint/naming-convention, camelcase - constructor(m: string, public readonly required_stages: string[], public readonly flows: IFlow[]) { + public constructor(m: string, public readonly required_stages: string[], public readonly flows: IFlow[]) { super(m); } } @@ -218,7 +218,7 @@ export class InteractiveAuth { // the promise the will resolve/reject when it completes private submitPromise: Promise | null = null; - constructor(opts: IOpts) { + public constructor(opts: IOpts) { this.matrixClient = opts.matrixClient; this.data = opts.authData || {}; this.requestCallback = opts.doRequest; @@ -419,7 +419,7 @@ export class InteractiveAuth { /** * Requests a new email token and sets the email sid for the validation session */ - public requestEmailToken = async () => { + public requestEmailToken = async (): Promise => { if (!this.requestingEmailToken) { logger.trace("Requesting email token. Attempt: " + this.emailAttempt); // If we've picked a flow with email auth, we send the email diff --git a/src/logger.ts b/src/logger.ts index f2fb821673f..de1ca6619c1 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -35,7 +35,7 @@ const DEFAULT_NAMESPACE = "matrix"; // console methods at initialization time by a factory that looks up the console methods // when logging so we always get the current value of console methods. log.methodFactory = function(methodName, logLevel, loggerName) { - return function(this: PrefixedLogger, ...args) { + return function(this: PrefixedLogger, ...args): void { /* eslint-disable @typescript-eslint/no-invalid-this */ if (this.prefix) { args.unshift(this.prefix); @@ -67,7 +67,7 @@ export interface PrefixedLogger extends Logger { prefix: string; } -function extendLogger(logger: Logger) { +function extendLogger(logger: Logger): void { (logger).withPrefix = function(prefix: string): PrefixedLogger { const existingPrefix = this.prefix || ""; return getPrefixedLogger(existingPrefix + prefix); diff --git a/src/matrix.ts b/src/matrix.ts index e89feddeb04..421e0e6ed87 100644 --- a/src/matrix.ts +++ b/src/matrix.ts @@ -21,6 +21,7 @@ import { MemoryStore } from "./store/memory"; import { MatrixScheduler } from "./scheduler"; import { MatrixClient, ICreateClientOpts } from "./client"; import { RoomWidgetClient, ICapabilities } from "./embedded"; +import { CryptoStore } from "./crypto/store/base"; export * from "./client"; export * from "./embedded"; @@ -64,7 +65,7 @@ export { } from "./webrtc/groupCall"; export type { GroupCall } from "./webrtc/groupCall"; -let cryptoStoreFactory = () => new MemoryCryptoStore; +let cryptoStoreFactory = (): CryptoStore => new MemoryCryptoStore; /** * Configure a different factory to be used for creating crypto stores @@ -72,7 +73,7 @@ let cryptoStoreFactory = () => new MemoryCryptoStore; * @param {Function} fac a function which will return a new * {@link module:crypto.store.base~CryptoStore}. */ -export function setCryptoStoreFactory(fac) { +export function setCryptoStoreFactory(fac: () => CryptoStore): void { cryptoStoreFactory = fac; } diff --git a/src/models/MSC3089TreeSpace.ts b/src/models/MSC3089TreeSpace.ts index 762ef586ab6..f437eab84da 100644 --- a/src/models/MSC3089TreeSpace.ts +++ b/src/models/MSC3089TreeSpace.ts @@ -172,7 +172,7 @@ export class MSC3089TreeSpace { const currentPls = this.room.currentState.getStateEvents(EventType.RoomPowerLevels, ""); if (Array.isArray(currentPls)) throw new Error("Unexpected return type for power levels"); - const pls = currentPls.getContent() || {}; + const pls = currentPls?.getContent() || {}; const viewLevel = pls['users_default'] || 0; const editLevel = pls['events_default'] || 50; const adminLevel = pls['events']?.[EventType.RoomPowerLevels] || 100; @@ -207,7 +207,7 @@ export class MSC3089TreeSpace { const currentPls = this.room.currentState.getStateEvents(EventType.RoomPowerLevels, ""); if (Array.isArray(currentPls)) throw new Error("Unexpected return type for power levels"); - const pls = currentPls.getContent() || {}; + const pls = currentPls?.getContent() || {}; const viewLevel = pls['users_default'] || 0; const editLevel = pls['events_default'] || 50; const adminLevel = pls['events']?.[EventType.RoomPowerLevels] || 100; diff --git a/src/models/beacon.ts b/src/models/beacon.ts index fc817b05e24..6e20b229bd5 100644 --- a/src/models/beacon.ts +++ b/src/models/beacon.ts @@ -56,7 +56,7 @@ export class Beacon extends TypedEventEmitter; private _latestLocationEvent?: MatrixEvent; - constructor( + public constructor( private rootEvent: MatrixEvent, ) { super(); @@ -180,7 +180,7 @@ export class Beacon extends TypedEventEmitter { + private clearLatestLocation = (): void => { this._latestLocationEvent = undefined; this.emit(BeaconEvent.LocationUpdate, this.latestLocationState!); }; diff --git a/src/models/event-context.ts b/src/models/event-context.ts index a1a43e98f50..60252627bde 100644 --- a/src/models/event-context.ts +++ b/src/models/event-context.ts @@ -42,7 +42,7 @@ export class EventContext { * * @constructor */ - constructor(public readonly ourEvent: MatrixEvent) { + public constructor(public readonly ourEvent: MatrixEvent) { this.timeline = [ourEvent]; } diff --git a/src/models/event-timeline-set.ts b/src/models/event-timeline-set.ts index a69715dc381..6dd2a0e7740 100644 --- a/src/models/event-timeline-set.ts +++ b/src/models/event-timeline-set.ts @@ -36,7 +36,7 @@ if (DEBUG) { // using bind means that we get to keep useful line numbers in the console debuglog = logger.log.bind(logger); } else { - debuglog = function() {}; + debuglog = function(): void {}; } interface IOpts { @@ -135,7 +135,7 @@ export class EventTimelineSet extends TypedEventEmitterThis property is experimental and may change. */ - constructor(public event: Partial = {}) { + public constructor(public event: Partial = {}) { super(); // intern the values of matrix events to force share strings and reduce the @@ -352,7 +352,7 @@ export class MatrixEvent extends TypedEventEmitter { await this.client.redactEvent(event.getRoomId()!, event.getId()!); } @@ -314,7 +314,7 @@ export class IgnoredInvites { /** * Modify in place the `IGNORE_INVITES_POLICIES` object from account data. */ - private async withIgnoreInvitesPolicies(cb: (ignoreInvitesPolicies: {[key: string]: any}) => void) { + private async withIgnoreInvitesPolicies(cb: (ignoreInvitesPolicies: {[key: string]: any}) => void): Promise { const { policies, ignoreInvitesPolicies } = this.getPoliciesAndIgnoreInvitesPolicies(); cb(ignoreInvitesPolicies); policies[IGNORE_INVITES_ACCOUNT_EVENT_KEY.name] = ignoreInvitesPolicies; diff --git a/src/models/read-receipt.ts b/src/models/read-receipt.ts index 3d78991478f..391cd98117e 100644 --- a/src/models/read-receipt.ts +++ b/src/models/read-receipt.ts @@ -33,7 +33,7 @@ export function synthesizeReceipt(userId: string, event: MatrixEvent, receiptTyp [receiptType]: { [userId]: { ts: event.getTs(), - threadId: event.threadRootId ?? MAIN_ROOM_TIMELINE, + thread_id: event.threadRootId ?? MAIN_ROOM_TIMELINE, }, }, }, diff --git a/src/models/related-relations.ts b/src/models/related-relations.ts index 09c618ae8ee..f619ea059fe 100644 --- a/src/models/related-relations.ts +++ b/src/models/related-relations.ts @@ -29,11 +29,11 @@ export class RelatedRelations { return this.relations.reduce((c, p) => [...c, ...p.getRelations()], []); } - public on(ev: T, fn: Listener) { + public on(ev: T, fn: Listener): void { this.relations.forEach(r => r.on(ev, fn)); } - public off(ev: T, fn: Listener) { + public off(ev: T, fn: Listener): void { this.relations.forEach(r => r.off(ev, fn)); } } diff --git a/src/models/relations-container.ts b/src/models/relations-container.ts index edb2616107e..55e098a3f6d 100644 --- a/src/models/relations-container.ts +++ b/src/models/relations-container.ts @@ -26,7 +26,7 @@ export class RelationsContainer { // this.relations.get(parentEventId).get(relationType).get(relationEventType) private relations = new Map>>(); - constructor(private readonly client: MatrixClient, private readonly room?: Room) { + public constructor(private readonly client: MatrixClient, private readonly room?: Room) { } /** @@ -98,7 +98,7 @@ export class RelationsContainer { const relation = event.getRelation(); if (!relation) return; - const onEventDecrypted = () => { + const onEventDecrypted = (): void => { if (event.isDecryptionFailure()) { // This could for example happen if the encryption keys are not yet available. // The event may still be decrypted later. Register the listener again. diff --git a/src/models/relations.ts b/src/models/relations.ts index a384c9b7f26..00fb7f6e979 100644 --- a/src/models/relations.ts +++ b/src/models/relations.ts @@ -60,7 +60,7 @@ export class Relations extends TypedEventEmitter { if (this.relationEventIds.has(event.getId()!)) { return; } @@ -123,7 +123,7 @@ export class Relations extends TypedEventEmitter { if (!this.relations.has(event)) { return; } @@ -160,7 +160,7 @@ export class Relations extends TypedEventEmitter { + private onEventStatus = (event: MatrixEvent, status: EventStatus | null): void => { if (!event.isSending()) { // Sending is done, so we don't need to listen anymore event.removeListener(MatrixEventEvent.Status, this.onEventStatus); @@ -356,7 +356,7 @@ export class Relations extends TypedEventEmitter { if (this.targetEvent) { return; } @@ -374,7 +374,7 @@ export class Relations extends TypedEventEmitter * events dictionary, keyed on the event type and then the state_key value. * @prop {string} paginationToken The pagination token for this state. */ - constructor(public readonly roomId: string, private oobMemberFlags = { status: OobStatus.NotStarted }) { + public constructor(public readonly roomId: string, private oobMemberFlags = { status: OobStatus.NotStarted }) { super(); this.updateModifiedTime(); } @@ -250,8 +250,8 @@ export class RoomState extends TypedEventEmitter * undefined, else a single event (or null if no match found). */ public getStateEvents(eventType: EventType | string): MatrixEvent[]; - public getStateEvents(eventType: EventType | string, stateKey: string): MatrixEvent; - public getStateEvents(eventType: EventType | string, stateKey?: string) { + public getStateEvents(eventType: EventType | string, stateKey: string): MatrixEvent | null; + public getStateEvents(eventType: EventType | string, stateKey?: string): MatrixEvent[] | MatrixEvent | null { if (!this.events.has(eventType)) { // no match return stateKey === undefined ? [] : null; @@ -342,7 +342,7 @@ export class RoomState extends TypedEventEmitter * @fires module:client~MatrixClient#event:"RoomState.events" * @fires module:client~MatrixClient#event:"RoomStateEvent.Marker" */ - public setStateEvents(stateEvents: MatrixEvent[], markerFoundOptions?: IMarkerFoundOptions) { + public setStateEvents(stateEvents: MatrixEvent[], markerFoundOptions?: IMarkerFoundOptions): void { this.updateModifiedTime(); // update the core event dict diff --git a/src/models/room-summary.ts b/src/models/room-summary.ts index d01b470ae8e..ba279d2d4fc 100644 --- a/src/models/room-summary.ts +++ b/src/models/room-summary.ts @@ -46,6 +46,6 @@ interface IInfo { * @param {Number} info.timestamp The timestamp for this room. */ export class RoomSummary { - constructor(public readonly roomId: string, info?: IInfo) {} + public constructor(public readonly roomId: string, info?: IInfo) {} } diff --git a/src/models/room.ts b/src/models/room.ts index 21b06a6ced6..de3d18733de 100644 --- a/src/models/room.ts +++ b/src/models/room.ts @@ -317,7 +317,7 @@ export class Room extends ReadReceipt { * @param {boolean} [opts.timelineSupport = false] Set to true to enable improved * timeline support. */ - constructor( + public constructor( public readonly roomId: string, public readonly client: MatrixClient, public readonly myUserId: string, @@ -1183,7 +1183,7 @@ export class Room extends ReadReceipt { * for this type. */ public getUnreadNotificationCount(type = NotificationCountType.Total): number { - let count = this.notificationCounts[type] ?? 0; + let count = this.getRoomUnreadNotificationCount(type); if (this.client.canSupport.get(Feature.ThreadUnreadNotifications) !== ServerSupport.Unsupported) { for (const threadNotification of this.threadNotifications.values()) { count += threadNotification[type] ?? 0; @@ -1192,6 +1192,27 @@ export class Room extends ReadReceipt { return count; } + /** + * Get the notification for the event context (room or thread timeline) + */ + public getUnreadCountForEventContext(type = NotificationCountType.Total, event: MatrixEvent): number { + const isThreadEvent = !!event.threadRootId && !event.isThreadRoot; + + return (isThreadEvent + ? this.getThreadUnreadNotificationCount(event.threadRootId, type) + : this.getRoomUnreadNotificationCount(type)) ?? 0; + } + + /** + * Get one of the notification counts for this room + * @param {String} type The type of notification count to get. default: 'total' + * @return {Number} The notification count, or undefined if there is no count + * for this type. + */ + public getRoomUnreadNotificationCount(type = NotificationCountType.Total): number { + return this.notificationCounts[type] ?? 0; + } + /** * @experimental * Get one of the notification counts for a thread @@ -1758,20 +1779,17 @@ export class Room extends ReadReceipt { let latestMyThreadsRootEvent: MatrixEvent | undefined; const roomState = this.getLiveTimeline().getState(EventTimeline.FORWARDS); for (const rootEvent of threadRoots) { - this.threadsTimelineSets[0]?.addLiveEvent(rootEvent, { + const opts = { duplicateStrategy: DuplicateStrategy.Ignore, fromCache: false, roomState, - }); + }; + this.threadsTimelineSets[0]?.addLiveEvent(rootEvent, opts); const threadRelationship = rootEvent .getServerAggregatedRelation(THREAD_RELATION_TYPE.name); if (threadRelationship?.current_user_participated) { - this.threadsTimelineSets[1]?.addLiveEvent(rootEvent, { - duplicateStrategy: DuplicateStrategy.Ignore, - fromCache: false, - roomState, - }); + this.threadsTimelineSets[1]?.addLiveEvent(rootEvent, opts); latestMyThreadsRootEvent = rootEvent; } } @@ -1950,7 +1968,7 @@ export class Room extends ReadReceipt { )); } - private updateThreadRootEvents = (thread: Thread, toStartOfTimeline: boolean) => { + private updateThreadRootEvents = (thread: Thread, toStartOfTimeline: boolean): void => { if (thread.length) { this.updateThreadRootEvent(this.threadsTimelineSets?.[0], thread, toStartOfTimeline); if (thread.hasCurrentUserParticipated) { @@ -1963,7 +1981,7 @@ export class Room extends ReadReceipt { timelineSet: Optional, thread: Thread, toStartOfTimeline: boolean, - ) => { + ): void => { if (timelineSet && thread.rootEvent) { if (Thread.hasServerSideSupport) { timelineSet.addLiveEvent(thread.rootEvent, { @@ -2050,7 +2068,7 @@ export class Room extends ReadReceipt { redactedEvent.getType(), redactedEvent.getStateKey()!, ); - if (currentStateEvent.getId() === redactedEvent.getId()) { + if (currentStateEvent?.getId() === redactedEvent.getId()) { this.currentState.setStateEvents([redactedEvent]); } } @@ -2913,7 +2931,7 @@ export class Room extends ReadReceipt { let excludedUserIds: string[] = []; const mFunctionalMembers = this.currentState.getStateEvents(UNSTABLE_ELEMENT_FUNCTIONAL_USERS.name, ""); if (Array.isArray(mFunctionalMembers?.getContent().service_members)) { - excludedUserIds = mFunctionalMembers.getContent().service_members; + excludedUserIds = mFunctionalMembers!.getContent().service_members; } // get members that are NOT ourselves and are actually in the room. @@ -3077,7 +3095,7 @@ export class Room extends ReadReceipt { originalEvent.applyVisibilityEvent(visibilityChange); } - private redactVisibilityChangeEvent(event: MatrixEvent) { + private redactVisibilityChangeEvent(event: MatrixEvent): void { // Sanity checks. if (!event.isVisibilityEvent) { throw new Error("expected a visibility change event"); diff --git a/src/models/search-result.ts b/src/models/search-result.ts index 99f9c5ddaa3..f598301d785 100644 --- a/src/models/search-result.ts +++ b/src/models/search-result.ts @@ -60,5 +60,5 @@ export class SearchResult { * * @constructor */ - constructor(public readonly rank: number, public readonly context: EventContext) {} + public constructor(public readonly rank: number, public readonly context: EventContext) {} } diff --git a/src/models/thread.ts b/src/models/thread.ts index 3c02558bf34..8c9826c35ec 100644 --- a/src/models/thread.ts +++ b/src/models/thread.ts @@ -94,7 +94,7 @@ export class Thread extends ReadReceipt { public initialEventsFetched = !Thread.hasServerSideSupport; - constructor( + public constructor( public readonly id: string, public rootEvent: MatrixEvent | undefined, opts: IThreadOpts, @@ -123,7 +123,7 @@ export class Thread extends ReadReceipt { this.room.on(MatrixEventEvent.BeforeRedaction, this.onBeforeRedaction); this.room.on(RoomEvent.Redaction, this.onRedaction); this.room.on(RoomEvent.LocalEchoUpdated, this.onEcho); - this.timelineSet.on(RoomEvent.Timeline, this.onEcho); + this.timelineSet.on(RoomEvent.Timeline, this.onTimelineEvent); // even if this thread is thought to be originating from this client, we initialise it as we may be in a // gappy sync and a thread around this event may already exist. @@ -167,7 +167,7 @@ export class Thread extends ReadReceipt { Thread.hasServerSideFwdPaginationSupport = status; } - private onBeforeRedaction = (event: MatrixEvent, redaction: MatrixEvent) => { + private onBeforeRedaction = (event: MatrixEvent, redaction: MatrixEvent): void => { if (event?.isRelation(THREAD_RELATION_TYPE.name) && this.room.eventShouldLiveIn(event).threadId === this.id && event.getId() !== this.id && // the root event isn't counted in the length so ignore this redaction @@ -178,7 +178,7 @@ export class Thread extends ReadReceipt { } }; - private onRedaction = async (event: MatrixEvent) => { + private onRedaction = async (event: MatrixEvent): Promise => { if (event.threadRootId !== this.id) return; // ignore redactions for other timelines if (this.replyCount <= 0) { for (const threadEvent of this.events) { @@ -192,7 +192,19 @@ export class Thread extends ReadReceipt { } }; - private onEcho = async (event: MatrixEvent) => { + private onTimelineEvent = ( + event: MatrixEvent, + room: Room | undefined, + toStartOfTimeline: boolean | undefined, + ): void => { + // Add a synthesized receipt when paginating forward in the timeline + if (!toStartOfTimeline) { + room!.addLocalEchoReceipt(event.getSender()!, event, ReceiptType.Read); + } + this.onEcho(event); + }; + + private onEcho = async (event: MatrixEvent): Promise => { if (event.threadRootId !== this.id) return; // ignore echoes for other timelines if (this.lastEvent === event) return; if (!event.isRelation(THREAD_RELATION_TYPE.name)) return; @@ -349,7 +361,7 @@ export class Thread extends ReadReceipt { /** * Finds an event by ID in the current thread */ - public findEventById(eventId: string) { + public findEventById(eventId: string): MatrixEvent | undefined { // Check the lastEvent as it may have been created based on a bundled relationship and not in a timeline if (this.lastEvent?.getId() === eventId) { return this.lastEvent; @@ -361,7 +373,7 @@ export class Thread extends ReadReceipt { /** * Return last reply to the thread, if known. */ - public lastReply(matches: (ev: MatrixEvent) => boolean = () => true): MatrixEvent | null { + public lastReply(matches: (ev: MatrixEvent) => boolean = (): boolean => true): MatrixEvent | null { for (let i = this.events.length - 1; i >= 0; i--) { const event = this.events[i]; if (matches(event)) { diff --git a/src/models/user.ts b/src/models/user.ts index 31d113dbe72..5d92ca494cb 100644 --- a/src/models/user.ts +++ b/src/models/user.ts @@ -76,7 +76,7 @@ export class User extends TypedEventEmitter { * @prop {Object} events The events describing this user. * @prop {MatrixEvent} events.presence The m.presence event for this user. */ - constructor(public readonly userId: string) { + public constructor(public readonly userId: string) { super(); this.displayName = userId; this.rawDisplayName = userId; diff --git a/src/pushprocessor.ts b/src/pushprocessor.ts index 7c127d4c7ca..4538b97214e 100644 --- a/src/pushprocessor.ts +++ b/src/pushprocessor.ts @@ -126,7 +126,7 @@ export class PushProcessor { * @constructor * @param {Object} client The Matrix client object to use */ - constructor(private readonly client: MatrixClient) {} + public constructor(private readonly client: MatrixClient) {} /** * Convert a list of actions into a object with the actions as keys and their values diff --git a/src/realtime-callbacks.ts b/src/realtime-callbacks.ts index 222c9476390..4677b0c1ed1 100644 --- a/src/realtime-callbacks.ts +++ b/src/realtime-callbacks.ts @@ -48,7 +48,7 @@ type Callback = { const callbackList: Callback[] = []; // var debuglog = logger.log.bind(logger); -const debuglog = function(...params: any[]) {}; +const debuglog = function(...params: any[]): void {}; /** * reimplementation of window.setTimeout, which will call the callback if diff --git a/src/rendezvous/MSC3906Rendezvous.ts b/src/rendezvous/MSC3906Rendezvous.ts index c18af591268..a93d4c476ac 100644 --- a/src/rendezvous/MSC3906Rendezvous.ts +++ b/src/rendezvous/MSC3906Rendezvous.ts @@ -143,11 +143,11 @@ export class MSC3906Rendezvous { return await this.channel.receive() as MSC3906RendezvousPayload; } - private async send(payload: MSC3906RendezvousPayload) { + private async send(payload: MSC3906RendezvousPayload): Promise { await this.channel.send(payload); } - public async declineLoginOnExistingDevice() { + public async declineLoginOnExistingDevice(): Promise { logger.info('User declined sign in'); await this.send({ type: PayloadType.Finish, outcome: Outcome.Declined }); } diff --git a/src/rendezvous/RendezvousError.ts b/src/rendezvous/RendezvousError.ts index 1c1f6af9c7c..8b76fc1186c 100644 --- a/src/rendezvous/RendezvousError.ts +++ b/src/rendezvous/RendezvousError.ts @@ -17,7 +17,7 @@ limitations under the License. import { RendezvousFailureReason } from "."; export class RendezvousError extends Error { - constructor(message: string, public readonly code: RendezvousFailureReason) { + public constructor(message: string, public readonly code: RendezvousFailureReason) { super(message); } } diff --git a/src/room-hierarchy.ts b/src/room-hierarchy.ts index 6d95db2a5f9..c15f2ac56dc 100644 --- a/src/room-hierarchy.ts +++ b/src/room-hierarchy.ts @@ -47,7 +47,7 @@ export class RoomHierarchy { * @param {boolean} suggestedOnly whether to only return rooms with suggested=true. * @constructor */ - constructor( + public constructor( public readonly root: Room, private readonly pageSize?: number, private readonly maxDepth?: number, diff --git a/src/scheduler.ts b/src/scheduler.ts index 7a1a99dedb3..0c15032bcac 100644 --- a/src/scheduler.ts +++ b/src/scheduler.ts @@ -97,9 +97,9 @@ export class MatrixScheduler { * @see module:scheduler~queueAlgorithm */ // eslint-disable-next-line @typescript-eslint/naming-convention - public static QUEUE_MESSAGES(event: MatrixEvent) { + public static QUEUE_MESSAGES(event: MatrixEvent): "message" | null { // enqueue messages or events that associate with another event (redactions and relations) - if (event.getType() === EventType.RoomMessage || event.hasAssocation()) { + if (event.getType() === EventType.RoomMessage || event.hasAssociation()) { // put these events in the 'message' queue. return "message"; } @@ -116,7 +116,7 @@ export class MatrixScheduler { private activeQueues: string[] = []; private procFn: ProcessFunction | null = null; - constructor( + public constructor( public readonly retryAlgorithm = MatrixScheduler.RETRY_BACKOFF_RATELIMIT, public readonly queueAlgorithm = MatrixScheduler.QUEUE_MESSAGES, ) {} @@ -283,7 +283,7 @@ export class MatrixScheduler { } } -function debuglog(...args: any[]) { +function debuglog(...args: any[]): void { if (DEBUG) { logger.log(...args); } diff --git a/src/sliding-sync-sdk.ts b/src/sliding-sync-sdk.ts index 962d824a8e5..03a17bed327 100644 --- a/src/sliding-sync-sdk.ts +++ b/src/sliding-sync-sdk.ts @@ -45,7 +45,7 @@ import { RoomMemberEvent } from "./models/room-member"; const FAILED_SYNC_ERROR_THRESHOLD = 3; class ExtensionE2EE implements Extension { - constructor(private readonly crypto: Crypto) {} + public constructor(private readonly crypto: Crypto) {} public name(): string { return "e2ee"; @@ -95,7 +95,7 @@ class ExtensionE2EE implements Extension { class ExtensionToDevice implements Extension { private nextBatch: string | null = null; - constructor(private readonly client: MatrixClient) {} + public constructor(private readonly client: MatrixClient) {} public name(): string { return "to_device"; @@ -170,7 +170,7 @@ class ExtensionToDevice implements Extension { } class ExtensionAccountData implements Extension { - constructor(private readonly client: MatrixClient) {} + public constructor(private readonly client: MatrixClient) {} public name(): string { return "account_data"; @@ -233,6 +233,72 @@ class ExtensionAccountData implements Extension { } } +class ExtensionTyping implements Extension { + public constructor(private readonly client: MatrixClient) {} + + public name(): string { + return "typing"; + } + + public when(): ExtensionState { + return ExtensionState.PostProcess; + } + + public onRequest(isInitial: boolean): object | undefined { + if (!isInitial) { + return undefined; // don't send a JSON object for subsequent requests, we don't need to. + } + return { + enabled: true, + }; + } + + public onResponse(data: {rooms: Record}): void { + if (!data || !data.rooms) { + return; + } + + for (const roomId in data.rooms) { + processEphemeralEvents( + this.client, roomId, [data.rooms[roomId]], + ); + } + } +} + +class ExtensionReceipts implements Extension { + public constructor(private readonly client: MatrixClient) {} + + public name(): string { + return "receipts"; + } + + public when(): ExtensionState { + return ExtensionState.PostProcess; + } + + public onRequest(isInitial: boolean): object | undefined { + if (isInitial) { + return { + enabled: true, + }; + } + return undefined; // don't send a JSON object for subsequent requests, we don't need to. + } + + public onResponse(data: {rooms: Record}): void { + if (!data || !data.rooms) { + return; + } + + for (const roomId in data.rooms) { + processEphemeralEvents( + this.client, roomId, [data.rooms[roomId]], + ); + } + } +} + /** * A copy of SyncApi such that it can be used as a drop-in replacement for sync v2. For the actual * sliding sync API, see sliding-sync.ts or the class SlidingSync. @@ -244,7 +310,7 @@ export class SlidingSyncSdk { private failCount = 0; private notifEvents: MatrixEvent[] = []; // accumulator of sync events in the current sync response - constructor( + public constructor( private readonly slidingSync: SlidingSync, private readonly client: MatrixClient, private readonly opts: Partial = {}, @@ -256,7 +322,7 @@ export class SlidingSyncSdk { this.opts.experimentalThreadSupport = this.opts.experimentalThreadSupport === true; if (!opts.canResetEntireTimeline) { - opts.canResetEntireTimeline = (_roomId: string) => { + opts.canResetEntireTimeline = (_roomId: string): boolean => { return false; }; } @@ -273,6 +339,8 @@ export class SlidingSyncSdk { const extensions: Extension[] = [ new ExtensionToDevice(this.client), new ExtensionAccountData(this.client), + new ExtensionTyping(this.client), + new ExtensionReceipts(this.client), ]; if (this.opts.crypto) { extensions.push( @@ -348,7 +416,7 @@ export class SlidingSyncSdk { * Sync rooms the user has left. * @return {Promise} Resolved when they've been added to the store. */ - public async syncLeftRooms() { + public async syncLeftRooms(): Promise { return []; // TODO } @@ -457,7 +525,7 @@ export class SlidingSyncSdk { return false; } - private async processRoomData(client: MatrixClient, room: Room, roomData: MSC3575RoomData) { + private async processRoomData(client: MatrixClient, room: Room, roomData: MSC3575RoomData): Promise { roomData = ensureNameEvent(client, room.roomId, roomData); const stateEvents = mapEvents(this.client, room.roomId, roomData.required_state); // Prevent events from being decrypted ahead of time @@ -632,10 +700,10 @@ export class SlidingSyncSdk { // we'll purge this once we've fully processed the sync response this.addNotifications(timelineEvents); - const processRoomEvent = async (e: MatrixEvent) => { + const processRoomEvent = async (e: MatrixEvent): Promise => { client.emit(ClientEvent.Event, e); if (e.isState() && e.getType() == EventType.RoomEncryption && this.opts.crypto) { - await this.opts.crypto.onCryptoEvent(e); + await this.opts.crypto.onCryptoEvent(room, e); } }; @@ -767,7 +835,7 @@ export class SlidingSyncSdk { /** * Main entry point. Blocks until stop() is called. */ - public async sync() { + public async sync(): Promise { logger.debug("Sliding sync init loop"); // 1) We need to get push rules so we can check if events should bing as we get @@ -888,3 +956,16 @@ function mapEvents(client: MatrixClient, roomId: string | undefined, events: obj return mapper(e); }); } + +function processEphemeralEvents(client: MatrixClient, roomId: string, ephEvents: IMinimalEvent[]): void { + const ephemeralEvents = mapEvents(client, roomId, ephEvents); + const room = client.getRoom(roomId); + if (!room) { + logger.warn("got ephemeral events for room but room doesn't exist on client:", roomId); + return; + } + room.addEphemeralEvents(ephemeralEvents); + ephemeralEvents.forEach((e) => { + client.emit(ClientEvent.Event, e); + }); +} diff --git a/src/sliding-sync.ts b/src/sliding-sync.ts index 8c4a933431c..171bf55c32d 100644 --- a/src/sliding-sync.ts +++ b/src/sliding-sync.ts @@ -27,6 +27,10 @@ import { HTTPError } from "./http-api"; // to determine the max time we're willing to wait. const BUFFER_PERIOD_MS = 10 * 1000; +export const MSC3575_WILDCARD = "*"; +export const MSC3575_STATE_KEY_ME = "$ME"; +export const MSC3575_STATE_KEY_LAZY = "$LAZY"; + /** * Represents a subscription to a room or set of rooms. Controls which events are returned. */ @@ -165,7 +169,7 @@ class SlidingList { * Construct a new sliding list. * @param {MSC3575List} list The range, sort and filter values to use for this list. */ - constructor(list: MSC3575List) { + public constructor(list: MSC3575List) { this.replaceList(list); } @@ -369,7 +373,7 @@ export class SlidingSync extends TypedEventEmitter { this.abortController = new AbortController(); let currentPos: string | undefined; diff --git a/src/store/indexeddb-local-backend.ts b/src/store/indexeddb-local-backend.ts index 597ed511741..af4b5fc6519 100644 --- a/src/store/indexeddb-local-backend.ts +++ b/src/store/indexeddb-local-backend.ts @@ -25,7 +25,7 @@ import { IndexedToDeviceBatch, ToDeviceBatchWithTxnId } from "../models/ToDevice type DbMigration = (db: IDBDatabase) => void; const DB_MIGRATIONS: DbMigration[] = [ - (db) => { + (db): void => { // Make user store, clobber based on user ID. (userId property of User objects) db.createObjectStore("users", { keyPath: ["userId"] }); @@ -36,15 +36,15 @@ const DB_MIGRATIONS: DbMigration[] = [ // Make /sync store (sync tokens, room data, etc), always clobber (const key). db.createObjectStore("sync", { keyPath: ["clobber"] }); }, - (db) => { + (db): void => { const oobMembersStore = db.createObjectStore( "oob_membership_events", { keyPath: ["room_id", "state_key"], }); oobMembersStore.createIndex("room", "room_id"); }, - (db) => { db.createObjectStore("client_options", { keyPath: ["clobber"] }); }, - (db) => { db.createObjectStore("to_device_queue", { autoIncrement: true }); }, + (db): void => { db.createObjectStore("client_options", { keyPath: ["clobber"] }); }, + (db): void => { db.createObjectStore("to_device_queue", { autoIncrement: true }); }, // Expand as needed. ]; const VERSION = DB_MIGRATIONS.length; @@ -67,11 +67,11 @@ function selectQuery( const query = store.openCursor(keyRange); return new Promise((resolve, reject) => { const results: T[] = []; - query.onerror = () => { + query.onerror = (): void => { reject(new Error("Query failed: " + query.error)); }; // collect results - query.onsuccess = () => { + query.onsuccess = (): void => { const cursor = query.result; if (!cursor) { resolve(results); @@ -85,10 +85,10 @@ function selectQuery( function txnAsPromise(txn: IDBTransaction): Promise { return new Promise((resolve, reject) => { - txn.oncomplete = function(event) { + txn.oncomplete = function(event): void { resolve(event); }; - txn.onerror = function() { + txn.onerror = function(): void { reject(txn.error); }; }); @@ -96,10 +96,10 @@ function txnAsPromise(txn: IDBTransaction): Promise { function reqAsEventPromise(req: IDBRequest): Promise { return new Promise((resolve, reject) => { - req.onsuccess = function(event) { + req.onsuccess = function(event): void { resolve(event); }; - req.onerror = function() { + req.onerror = function(): void { reject(req.error); }; }); @@ -107,8 +107,8 @@ function reqAsEventPromise(req: IDBRequest): Promise { function reqAsPromise(req: IDBRequest): Promise { return new Promise((resolve, reject) => { - req.onsuccess = () => resolve(req); - req.onerror = (err) => reject(err); + req.onsuccess = (): void => resolve(req); + req.onerror = (err): void => reject(err); }); } @@ -141,7 +141,7 @@ export class LocalIndexedDBStoreBackend implements IIndexedDBBackend { * @param {string=} dbName Optional database name. The same name must be used * to open the same database. */ - constructor(private readonly indexedDB: IDBFactory, dbName = "default") { + public constructor(private readonly indexedDB: IDBFactory, dbName = "default") { this.dbName = "matrix-js-sdk:" + dbName; this.syncAccumulator = new SyncAccumulator(); } @@ -161,7 +161,7 @@ export class LocalIndexedDBStoreBackend implements IIndexedDBBackend { logger.log(`LocalIndexedDBStoreBackend.connect: connecting...`); const req = this.indexedDB.open(this.dbName, VERSION); - req.onupgradeneeded = (ev) => { + req.onupgradeneeded = (ev): void => { const db = req.result; const oldVersion = ev.oldVersion; logger.log( @@ -176,22 +176,22 @@ export class LocalIndexedDBStoreBackend implements IIndexedDBBackend { }); }; - req.onblocked = () => { + req.onblocked = (): void => { logger.log(`can't yet open LocalIndexedDBStoreBackend because it is open elsewhere`); }; logger.log(`LocalIndexedDBStoreBackend.connect: awaiting connection...`); - return reqAsEventPromise(req).then(() => { + return reqAsEventPromise(req).then(async () => { logger.log(`LocalIndexedDBStoreBackend.connect: connected`); this.db = req.result; // add a poorly-named listener for when deleteDatabase is called // so we can close our db connections. - this.db.onversionchange = () => { + this.db.onversionchange = (): void => { this.db?.close(); }; - return this.init(); + await this.init(); }); } @@ -204,7 +204,7 @@ export class LocalIndexedDBStoreBackend implements IIndexedDBBackend { * Having connected, load initial data from the database and prepare for use * @return {Promise} Resolves on success */ - private init() { + private init(): Promise { return Promise.all([ this.loadAccountData(), this.loadSyncData(), @@ -243,7 +243,7 @@ export class LocalIndexedDBStoreBackend implements IIndexedDBBackend { // were all known already let oobWritten = false; - request.onsuccess = () => { + request.onsuccess = (): void => { const cursor = request.result; if (!cursor) { // Unknown room @@ -260,7 +260,7 @@ export class LocalIndexedDBStoreBackend implements IIndexedDBBackend { } cursor.continue(); }; - request.onerror = (err) => { + request.onerror = (err): void => { reject(err); }; }).then((events) => { @@ -346,11 +346,11 @@ export class LocalIndexedDBStoreBackend implements IIndexedDBBackend { logger.log(`Removing indexeddb instance: ${this.dbName}`); const req = this.indexedDB.deleteDatabase(this.dbName); - req.onblocked = () => { + req.onblocked = (): void => { logger.log(`can't yet delete indexeddb ${this.dbName} because it is open elsewhere`); }; - req.onerror = () => { + req.onerror = (): void => { // in firefox, with indexedDB disabled, this fails with a // DOMError. We treat this as non-fatal, so that we can still // use the app. @@ -358,7 +358,7 @@ export class LocalIndexedDBStoreBackend implements IIndexedDBBackend { resolve(); }; - req.onsuccess = () => { + req.onsuccess = (): void => { logger.log(`Removed indexeddb instance: ${this.dbName}`); resolve(); }; diff --git a/src/store/indexeddb-remote-backend.ts b/src/store/indexeddb-remote-backend.ts index 39486fe2325..1ca6fc03a38 100644 --- a/src/store/indexeddb-remote-backend.ts +++ b/src/store/indexeddb-remote-backend.ts @@ -42,7 +42,7 @@ export class RemoteIndexedDBStoreBackend implements IIndexedDBBackend { * @param {string=} dbName Optional database name. The same name must be used * to open the same database. */ - constructor( + public constructor( private readonly workerFactory: () => Worker, private readonly dbName?: string, ) {} diff --git a/src/store/indexeddb-store-worker.ts b/src/store/indexeddb-store-worker.ts index 691a21f861b..57e7da983be 100644 --- a/src/store/indexeddb-store-worker.ts +++ b/src/store/indexeddb-store-worker.ts @@ -45,7 +45,7 @@ export class IndexedDBStoreWorker { * @param {function} postMessage The web worker postMessage function that * should be used to communicate back to the main script. */ - constructor(private readonly postMessage: InstanceType["postMessage"]) {} + public constructor(private readonly postMessage: InstanceType["postMessage"]) {} /** * Passes a message event from the main script into the class. This method diff --git a/src/store/indexeddb.ts b/src/store/indexeddb.ts index 961e66fd3e0..55f8261faa8 100644 --- a/src/store/indexeddb.ts +++ b/src/store/indexeddb.ts @@ -53,7 +53,7 @@ type EventHandlerMap = { }; export class IndexedDBStore extends MemoryStore { - static exists(indexedDB: IDBFactory, dbName: string): Promise { + public static exists(indexedDB: IDBFactory, dbName: string): Promise { return LocalIndexedDBStoreBackend.exists(indexedDB, dbName); } @@ -109,7 +109,7 @@ export class IndexedDBStore extends MemoryStore { * this API if you need to perform specific indexeddb actions like deleting the * database. */ - constructor(opts: IOpts) { + public constructor(opts: IOpts) { super(opts); if (!opts.indexedDB) { diff --git a/src/store/memory.ts b/src/store/memory.ts index 24b79fccbd6..f24ab2d976d 100644 --- a/src/store/memory.ts +++ b/src/store/memory.ts @@ -69,7 +69,7 @@ export class MemoryStore implements IStore { private pendingToDeviceBatches: IndexedToDeviceBatch[] = []; private nextToDeviceBatchId = 0; - constructor(opts: IOpts = {}) { + public constructor(opts: IOpts = {}) { this.localStorage = opts.localStorage; } @@ -90,7 +90,7 @@ export class MemoryStore implements IStore { * Set the token to stream from. * @param {string} token The token to stream from. */ - public setSyncToken(token: string) { + public setSyncToken(token: string): void { this.syncToken = token; } @@ -98,7 +98,7 @@ export class MemoryStore implements IStore { * Store the given room. * @param {Room} room The room to be stored. All properties must be stored. */ - public storeRoom(room: Room) { + public storeRoom(room: Room): void { this.rooms[room.roomId] = room; // add listeners for room member changes so we can keep the room member // map up-to-date. @@ -116,7 +116,7 @@ export class MemoryStore implements IStore { * @param {RoomState} state * @param {RoomMember} member */ - private onRoomMember = (event: MatrixEvent | null, state: RoomState, member: RoomMember) => { + private onRoomMember = (event: MatrixEvent | null, state: RoomState, member: RoomMember): void => { if (member.membership === "invite") { // We do NOT add invited members because people love to typo user IDs // which would then show up in these lists (!) @@ -217,7 +217,7 @@ export class MemoryStore implements IStore { * @param {string} token The token associated with these events. * @param {boolean} toStart True if these are paginated results. */ - public storeEvents(room: Room, events: MatrixEvent[], token: string, toStart: boolean) { + public storeEvents(room: Room, events: MatrixEvent[], token: string, toStart: boolean): void { // no-op because they've already been added to the room instance. } @@ -275,7 +275,7 @@ export class MemoryStore implements IStore { * @param {string} filterName * @param {string} filterId */ - public setFilterIdByName(filterName: string, filterId?: string) { + public setFilterIdByName(filterName: string, filterId?: string): void { if (!this.localStorage) { return; } diff --git a/src/store/stub.ts b/src/store/stub.ts index 64a35d513eb..746bc521ffb 100644 --- a/src/store/stub.ts +++ b/src/store/stub.ts @@ -56,7 +56,7 @@ export class StubStore implements IStore { * Set the sync token. * @param {string} token */ - public setSyncToken(token: string) { + public setSyncToken(token: string): void { this.fromToken = token; } @@ -64,7 +64,7 @@ export class StubStore implements IStore { * No-op. * @param {Room} room */ - public storeRoom(room: Room) {} + public storeRoom(room: Room): void {} /** * No-op. @@ -87,7 +87,7 @@ export class StubStore implements IStore { * Permanently delete a room. * @param {string} roomId */ - public removeRoom(roomId: string) { + public removeRoom(roomId: string): void { return; } @@ -103,7 +103,7 @@ export class StubStore implements IStore { * No-op. * @param {User} user */ - public storeUser(user: User) {} + public storeUser(user: User): void {} /** * No-op. @@ -139,13 +139,13 @@ export class StubStore implements IStore { * @param {string} token The token associated with these events. * @param {boolean} toStart True if these are paginated results. */ - public storeEvents(room: Room, events: MatrixEvent[], token: string, toStart: boolean) {} + public storeEvents(room: Room, events: MatrixEvent[], token: string, toStart: boolean): void {} /** * Store a filter. * @param {Filter} filter */ - public storeFilter(filter: Filter) {} + public storeFilter(filter: Filter): void {} /** * Retrieve a filter. @@ -171,13 +171,13 @@ export class StubStore implements IStore { * @param {string} filterName * @param {string} filterId */ - public setFilterIdByName(filterName: string, filterId?: string) {} + public setFilterIdByName(filterName: string, filterId?: string): void {} /** * Store user-scoped account data events * @param {Array} events The events to store. */ - public storeAccountDataEvents(events: MatrixEvent[]) {} + public storeAccountDataEvents(events: MatrixEvent[]): void {} /** * Get account data event by event type @@ -209,7 +209,7 @@ export class StubStore implements IStore { /** * Save does nothing as there is no backing data store. */ - public save() {} + public save(): void {} /** * Startup does nothing. diff --git a/src/sync-accumulator.ts b/src/sync-accumulator.ts index 484e96e4f70..19d5cf1312e 100644 --- a/src/sync-accumulator.ts +++ b/src/sync-accumulator.ts @@ -211,7 +211,7 @@ export class SyncAccumulator { * never be more. This cannot be 0 or else it makes it impossible to scroll * back in a room. Default: 50. */ - constructor(private readonly opts: IOpts = {}) { + public constructor(private readonly opts: IOpts = {}) { this.opts.maxTimelineEntries = this.opts.maxTimelineEntries || 50; } diff --git a/src/sync.ts b/src/sync.ts index d85bfc4cb6f..20cab67b6eb 100644 --- a/src/sync.ts +++ b/src/sync.ts @@ -106,7 +106,7 @@ function getFilterName(userId: string, suffix?: string): string { return `FILTER_SYNC_${userId}` + (suffix ? "_" + suffix : ""); } -function debuglog(...params) { +function debuglog(...params): void { if (!DEBUG) return; logger.log(...params); } @@ -175,7 +175,7 @@ export class SyncApi { private failedSyncCount = 0; // Number of consecutive failed /sync requests private storeIsInvalid = false; // flag set if the store needs to be cleared before we can start - constructor(private readonly client: MatrixClient, private readonly opts: Partial = {}) { + public constructor(private readonly client: MatrixClient, private readonly opts: Partial = {}) { this.opts.initialSyncLimit = this.opts.initialSyncLimit ?? 8; this.opts.resolveInvitesToProfiles = this.opts.resolveInvitesToProfiles || false; this.opts.pollTimeout = this.opts.pollTimeout || (30 * 1000); @@ -183,7 +183,7 @@ export class SyncApi { this.opts.experimentalThreadSupport = this.opts.experimentalThreadSupport === true; if (!opts.canResetEntireTimeline) { - opts.canResetEntireTimeline = (roomId: string) => { + opts.canResetEntireTimeline = (roomId: string): boolean => { return false; }; } @@ -554,7 +554,7 @@ export class SyncApi { return false; } - private getPushRules = async () => { + private getPushRules = async (): Promise => { try { debuglog("Getting push rules..."); const result = await this.client.getPushRules(); @@ -572,7 +572,7 @@ export class SyncApi { } }; - private buildDefaultFilter = () => { + private buildDefaultFilter = (): Filter => { const filter = new Filter(this.client.credentials.userId); if (this.client.canSupport.get(Feature.ThreadUnreadNotifications) !== ServerSupport.Unsupported) { filter.setUnreadThreadNotifications(true); @@ -580,7 +580,7 @@ export class SyncApi { return filter; }; - private checkLazyLoadStatus = async () => { + private checkLazyLoadStatus = async (): Promise => { debuglog("Checking lazy load status..."); if (this.opts.lazyLoadMembers && this.client.isGuest()) { this.opts.lazyLoadMembers = false; @@ -1362,6 +1362,18 @@ export class SyncApi { } } + // process any crypto events *before* emitting the RoomStateEvent events. This + // avoids a race condition if the application tries to send a message after the + // state event is processed, but before crypto is enabled, which then causes the + // crypto layer to complain. + if (this.opts.crypto) { + for (const e of stateEvents.concat(events)) { + if (e.isState() && e.getType() === EventType.RoomEncryption && e.getStateKey() === "") { + await this.opts.crypto.onCryptoEvent(room, e); + } + } + } + try { await this.injectRoomEvents(room, stateEvents, events, syncEventData.fromCache); } catch (e) { @@ -1389,21 +1401,11 @@ export class SyncApi { this.processEventsForNotifs(room, events); - const processRoomEvent = async (e) => { - client.emit(ClientEvent.Event, e); - if (e.isState() && e.getType() == "m.room.encryption" && this.opts.crypto) { - await this.opts.crypto.onCryptoEvent(e); - } - }; - - await utils.promiseMapSeries(stateEvents, processRoomEvent); - await utils.promiseMapSeries(events, processRoomEvent); - ephemeralEvents.forEach(function(e) { - client.emit(ClientEvent.Event, e); - }); - accountDataEvents.forEach(function(e) { - client.emit(ClientEvent.Event, e); - }); + const emitEvent = (e: MatrixEvent): boolean => client.emit(ClientEvent.Event, e); + stateEvents.forEach(emitEvent); + events.forEach(emitEvent); + ephemeralEvents.forEach(emitEvent); + accountDataEvents.forEach(emitEvent); // Decrypt only the last message in all rooms to make sure we can generate a preview // And decrypt all events after the recorded read receipt to ensure an accurate @@ -1521,7 +1523,7 @@ export class SyncApi { * @param {boolean} connDidFail True if a connectivity failure has been detected. Optional. */ private pokeKeepAlive(connDidFail = false): void { - const success = () => { + const success = (): void => { clearTimeout(this.keepAliveTimer); if (this.connectionReturnedDefer) { this.connectionReturnedDefer.resolve(connDidFail); diff --git a/src/timeline-window.ts b/src/timeline-window.ts index c5560bdf4b0..5498600a99c 100644 --- a/src/timeline-window.ts +++ b/src/timeline-window.ts @@ -32,7 +32,7 @@ const DEBUG = false; /** * @private */ -const debuglog = DEBUG ? logger.log.bind(logger) : function() {}; +const debuglog = DEBUG ? logger.log.bind(logger) : function(): void {}; /** * the number of times we ask the server for more events before giving up @@ -84,7 +84,7 @@ export class TimelineWindow { * * @constructor */ - constructor( + public constructor( private readonly client: MatrixClient, private readonly timelineSet: EventTimelineSet, opts: IOpts = {}, @@ -104,7 +104,7 @@ export class TimelineWindow { public load(initialEventId?: string, initialWindowSize = 20): Promise { // given an EventTimeline, find the event we were looking for, and initialise our // fields so that the event in question is in the middle of the window. - const initFields = (timeline: Optional) => { + const initFields = (timeline: Optional): void => { if (!timeline) { throw new Error("No timeline given to initFields"); } @@ -430,7 +430,7 @@ export class TimelineIndex { public pendingPaginate?: Promise; // index: the indexes are relative to BaseIndex, so could well be negative. - constructor(public timeline: EventTimeline, public index: number) {} + public constructor(public timeline: EventTimeline, public index: number) {} /** * @return {number} the minimum possible value for the index in the current diff --git a/src/utils.ts b/src/utils.ts index d4364f18a54..71871d3b737 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -178,7 +178,7 @@ export function removeElement( * @param {*} value The thing to check. * @return {boolean} True if it is a function. */ -export function isFunction(value: any) { +export function isFunction(value: any): boolean { return Object.prototype.toString.call(value) === "[object Function]"; } @@ -189,7 +189,7 @@ export function isFunction(value: any) { * @throws If the object is missing keys. */ // note using 'keys' here would shadow the 'keys' function defined above -export function checkObjectHasKeys(obj: object, keys: string[]) { +export function checkObjectHasKeys(obj: object, keys: string[]): void { for (const key of keys) { if (!obj.hasOwnProperty(key)) { throw new Error("Missing required key: " + key); @@ -383,7 +383,7 @@ export function globToRegexp(glob: string, extended = false): string { if (!extended) { replacements.push([ /\\\[(!|)(.*)\\]/g, - (_match: string, neg: string, pat: string) => [ + (_match: string, neg: string, pat: string): string => [ '[', neg ? '^' : '', pat.replace(/\\-/, '-'), @@ -490,7 +490,7 @@ export function simpleRetryOperation(promiseFn: (attempt: number) => Promise< * The default alphabet used by string averaging in this SDK. This matches * all usefully printable ASCII characters (0x20-0x7E, inclusive). */ -export const DEFAULT_ALPHABET = (() => { +export const DEFAULT_ALPHABET = ((): string => { let str = ""; for (let c = 0x20; c <= 0x7E; c++) { str += String.fromCharCode(c); @@ -694,3 +694,16 @@ export function sortEventsByLatestContentTimestamp(left: MatrixEvent, right: Mat export function isSupportedReceiptType(receiptType: string): boolean { return [ReceiptType.Read, ReceiptType.ReadPrivate].includes(receiptType as ReceiptType); } + +/** + * Determines whether two maps are equal. + * @param eq The equivalence relation to compare values by. Defaults to strict equality. + */ +export function mapsEqual(x: Map, y: Map, eq = (v1: V, v2: V): boolean => v1 === v2): boolean { + if (x.size !== y.size) return false; + for (const [k, v1] of x) { + const v2 = y.get(k); + if (v2 === undefined || !eq(v1, v2)) return false; + } + return true; +} diff --git a/src/webrtc/audioContext.ts b/src/webrtc/audioContext.ts index 8a9ceb15da6..0e08574b8c8 100644 --- a/src/webrtc/audioContext.ts +++ b/src/webrtc/audioContext.ts @@ -35,7 +35,7 @@ export const acquireContext = (): AudioContext => { * released, allowing the context and associated hardware resources to be * cleaned up if nothing else is using it. */ -export const releaseContext = () => { +export const releaseContext = (): void => { refCount--; if (refCount === 0) { audioContext?.close(); diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index ff7d0663d1c..c27ccdb61cf 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -282,7 +282,7 @@ const SFU_KEEP_ALIVE_INTERVAL = 30 * 1000; // 30 seconds export class CallError extends Error { public readonly code: string; - constructor(code: CallErrorCode, msg: string, err: Error) { + public constructor(code: CallErrorCode, msg: string, err: Error) { // Still don't think there's any way to have proper nested errors super(msg + ": " + err); @@ -423,7 +423,7 @@ export class MatrixCall extends TypedEventEmitter { + const onState = (state: CallState): void => { if (state !== CallState.Ringing) { clearTimeout(ringingTimer); this.off(CallEvent.State, onState); @@ -1088,6 +1096,7 @@ export class MatrixCall extends TypedEventEmitter { + const onRemoveTrack = (): void => { if (stream.getTracks().length === 0) { logger.info(`Call ${this.callId} removing track streamId: ${stream.id}`); this.deleteFeedByStream(stream); @@ -2804,6 +2813,7 @@ export class MatrixCall extends TypedEventEmitter>; private eventBufferPromiseChain?: Promise; - constructor(client: MatrixClient) { + public constructor(client: MatrixClient) { this.client = client; this.calls = new Map(); // The sync code always emits one event at a time, so it will patiently @@ -61,13 +61,13 @@ export class CallEventHandler { this.candidateEventsByCall = new Map>(); } - public start() { + public start(): void { this.client.on(ClientEvent.Sync, this.onSync); this.client.on(RoomEvent.Timeline, this.onRoomTimeline); this.client.on(ClientEvent.ToDeviceEvent, this.onToDeviceEvent); } - public stop() { + public stop(): void { this.client.removeListener(ClientEvent.Sync, this.onSync); this.client.removeListener(RoomEvent.Timeline, this.onRoomTimeline); this.client.removeListener(ClientEvent.ToDeviceEvent, this.onToDeviceEvent); @@ -87,7 +87,7 @@ export class CallEventHandler { } }; - private async evaluateEventBuffer(eventBuffer: MatrixEvent[]) { + private async evaluateEventBuffer(eventBuffer: MatrixEvent[]): Promise { await Promise.all(eventBuffer.map((event) => this.client.decryptEventIfNeeded(event))); const callEvents = eventBuffer.filter((event) => { @@ -125,7 +125,7 @@ export class CallEventHandler { } } - private onRoomTimeline = (event: MatrixEvent) => { + private onRoomTimeline = (event: MatrixEvent): void => { this.callEventBuffer.push(event); }; @@ -178,7 +178,7 @@ export class CallEventHandler { } }; - private async handleCallEvent(event: MatrixEvent) { + private async handleCallEvent(event: MatrixEvent): Promise { this.client.emit(ClientEvent.ReceivedVoipEvent, event); const content = event.getContent(); @@ -189,7 +189,6 @@ export class CallEventHandler { const groupCallId = content.conf_id; const type = event.getType() as EventType; const senderId = event.getSender()!; - const weSentTheEvent = senderId === this.client.credentials.userId; let call = content.call_id ? this.calls.get(content.call_id) : undefined; let opponentDeviceId: string | undefined; @@ -220,6 +219,9 @@ export class CallEventHandler { } } + const weSentTheEvent = senderId === this.client.credentials.userId + && (opponentDeviceId === undefined || opponentDeviceId === this.client.getDeviceId()!); + if (!callRoomId) return; if (type === EventType.CallInvite) { diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 965a0444123..bc1c344505c 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -29,6 +29,7 @@ export interface ICallFeedOpts { client: MatrixClient; roomId?: string; userId: string; + deviceId: string | undefined; stream: MediaStream; purpose: SDPStreamMetadataPurpose; /** @@ -63,6 +64,7 @@ export class CallFeed extends TypedEventEmitter public stream: MediaStream; public sdpMetadataStreamId: string; public userId: string; + public readonly deviceId: string | undefined; public purpose: SDPStreamMetadataPurpose; public speakingVolumeSamples: number[]; @@ -80,12 +82,13 @@ export class CallFeed extends TypedEventEmitter private volumeLooperTimeout?: ReturnType; private _disposed = false; - constructor(opts: ICallFeedOpts) { + public constructor(opts: ICallFeedOpts) { super(); this.client = opts.client; this.roomId = opts.roomId; this.userId = opts.userId; + this.deviceId = opts.deviceId; this.purpose = opts.purpose; this.audioMuted = opts.audioMuted; this.videoMuted = opts.videoMuted; @@ -156,7 +159,8 @@ export class CallFeed extends TypedEventEmitter * @returns {boolean} is local? */ public isLocal(): boolean { - return this.userId === this.client.getUserId(); + return this.userId === this.client.getUserId() + && (this.deviceId === undefined || this.deviceId === this.client.getDeviceId()); } /** @@ -227,11 +231,11 @@ export class CallFeed extends TypedEventEmitter } } - public setSpeakingThreshold(threshold: number) { + public setSpeakingThreshold(threshold: number): void { this.speakingThreshold = threshold; } - private volumeLooper = () => { + private volumeLooper = (): void => { if (!this.analyser) return; if (!this.measuringVolumeActivity) return; @@ -282,6 +286,7 @@ export class CallFeed extends TypedEventEmitter client: this.client, roomId: this.roomId, userId: this.userId, + deviceId: this.deviceId, stream, purpose: this.purpose, audioMuted: this.audioMuted, diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 9da1e2477e9..2d63456f089 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -1,6 +1,6 @@ import { TypedEventEmitter } from "../models/typed-event-emitter"; import { CallFeed, SPEAKING_THRESHOLD } from "./callFeed"; -import { IFocusInfo, MatrixClient } from "../client"; +import { IFocusInfo, MatrixClient, IMyDevice } from "../client"; import { CallErrorCode, CallEvent, @@ -14,15 +14,16 @@ import { } from "./call"; import { RoomMember } from "../models/room-member"; import { Room } from "../models/room"; +import { RoomStateEvent } from "../models/room-state"; import { logger } from "../logger"; import { ReEmitter } from "../ReEmitter"; import { SDPStreamMetadataPurpose } from "./callEventTypes"; -import { ISendEventResponse } from "../@types/requests"; import { MatrixEvent } from "../models/event"; import { EventType } from "../@types/event"; import { CallEventHandlerEvent } from "./callEventHandler"; import { GroupCallEventHandlerEvent } from "./groupCallEventHandler"; import { IScreensharingOpts } from "./mediaHandler"; +import { mapsEqual } from "../utils"; export enum GroupCallIntent { Ring = "m.ring", @@ -53,15 +54,15 @@ export enum GroupCallEvent { export type GroupCallEventHandlerMap = { [GroupCallEvent.GroupCallStateChanged]: (newState: GroupCallState, oldState: GroupCallState) => void; - [GroupCallEvent.ActiveSpeakerChanged]: (activeSpeaker: string) => void; - [GroupCallEvent.CallsChanged]: (calls: MatrixCall[]) => void; + [GroupCallEvent.ActiveSpeakerChanged]: (activeSpeaker: CallFeed | undefined) => void; + [GroupCallEvent.CallsChanged]: (calls: Map>) => void; [GroupCallEvent.UserMediaFeedsChanged]: (feeds: CallFeed[]) => void; [GroupCallEvent.ScreenshareFeedsChanged]: (feeds: CallFeed[]) => void; [GroupCallEvent.LocalScreenshareStateChanged]: ( isScreensharing: boolean, feed?: CallFeed, sourceId?: string, ) => void; [GroupCallEvent.LocalMuteStateChanged]: (audioMuted: boolean, videoMuted: boolean) => void; - [GroupCallEvent.ParticipantsChanged]: (participants: RoomMember[]) => void; + [GroupCallEvent.ParticipantsChanged]: (participants: Map>) => void; [GroupCallEvent.Error]: (error: GroupCallError) => void; }; @@ -74,7 +75,7 @@ export enum GroupCallErrorCode { export class GroupCallError extends Error { public code: string; - constructor(code: GroupCallErrorCode, msg: string, err?: Error) { + public constructor(code: GroupCallErrorCode, msg: string, err?: Error) { // Still don't think there's any way to have proper nested errors if (err) { super(msg + ": " + err); @@ -87,13 +88,13 @@ export class GroupCallError extends Error { } export class GroupCallUnknownDeviceError extends GroupCallError { - constructor(public userId: string) { + public constructor(public userId: string) { super(GroupCallErrorCode.UnknownDevice, "No device found for " + userId); } } export class OtherUserSpeakingError extends Error { - constructor() { + public constructor() { super("Cannot unmute: another user is speaking"); } } @@ -121,6 +122,7 @@ export interface IGroupCallRoomState { export interface IGroupCallRoomMemberDevice { "device_id": string; "session_id": string; + "expires_ts": number; "feeds": IGroupCallRoomMemberFeed[]; "m.foci.active"?: IFocusInfo[]; "m.foci.preferred"?: IFocusInfo[]; @@ -133,18 +135,21 @@ export interface IGroupCallRoomMemberCallState { export interface IGroupCallRoomMemberState { "m.calls": IGroupCallRoomMemberCallState[]; - "m.expires_ts": number; } export enum GroupCallState { LocalCallFeedUninitialized = "local_call_feed_uninitialized", InitializingLocalCallFeed = "initializing_local_call_feed", LocalCallFeedInitialized = "local_call_feed_initialized", - Entering = "entering", Entered = "entered", Ended = "ended", } +export interface ParticipantState { + sessionId: string; + screensharing: boolean; +} + interface ICallHandlers { onCallFeedsChanged: (feeds: CallFeed[]) => void; onCallStateChanged: (state: CallState, oldState: CallState | undefined) => void; @@ -152,14 +157,7 @@ interface ICallHandlers { onCallReplaced: (newCall: MatrixCall) => void; } -const CALL_MEMBER_STATE_TIMEOUT = 1000 * 60 * 60; // 1 hour - -const callMemberStateIsExpired = (event: MatrixEvent): boolean => { - const now = Date.now(); - const content = event?.getContent() ?? {}; - const expiresAt = typeof content["m.expires_ts"] === "number" ? content["m.expires_ts"] : -Infinity; - return expiresAt <= now; -}; +const DEVICE_TIMEOUT = 1000 * 60 * 60; // 1 hour function getCallUserId(call: MatrixCall): string | null { return call.getOpponentMember()?.userId || call.invitee || null; @@ -175,30 +173,28 @@ export class GroupCall extends TypedEventEmitter< public participantTimeout = 1000 * 15; public pttMaxTransmitTime = 1000 * 20; - public state = GroupCallState.LocalCallFeedUninitialized; - public activeSpeaker?: string; // userId + public activeSpeaker?: CallFeed; public localCallFeed?: CallFeed; public localScreenshareFeed?: CallFeed; public localDesktopCapturerSourceId?: string; - public calls: MatrixCall[] = []; - public participants: RoomMember[] = []; - public userMediaFeeds: CallFeed[] = []; - public screenshareFeeds: CallFeed[] = []; + public readonly calls = new Map>(); + public readonly userMediaFeeds: CallFeed[] = []; + public readonly screenshareFeeds: CallFeed[] = []; public groupCallId: string; + public foci: IFocusInfo[] = []; - private callHandlers: Map = new Map(); - private activeSpeakerLoopTimeout?: ReturnType; - private retryCallLoopTimeout?: ReturnType; - private retryCallCounts: Map = new Map(); + private callHandlers = new Map>(); // User ID -> device ID -> handlers + private activeSpeakerLoopInterval?: ReturnType; + private retryCallLoopInterval?: ReturnType; + private retryCallCounts: Map> = new Map(); private reEmitter: ReEmitter; private transmitTimer: ReturnType | null = null; - private memberStateExpirationTimers: Map> = new Map(); + private participantsExpirationTimer: ReturnType | null = null; private resendMemberStateTimer: ReturnType | null = null; private initWithAudioMuted = false; private initWithVideoMuted = false; - private foci: IFocusInfo[] = []; - constructor( + public constructor( private client: MatrixClient, public room: Room, public type: GroupCallType, @@ -210,15 +206,22 @@ export class GroupCall extends TypedEventEmitter< ) { super(); this.reEmitter = new ReEmitter(this); - this.groupCallId = groupCallId || genCallID(); - - for (const stateEvent of this.getMemberStateEvents()) { - this.onMemberStateChanged(stateEvent); - } + this.groupCallId = groupCallId ?? genCallID(); + this.creationTs = room.currentState.getStateEvents( + EventType.GroupCallPrefix, this.groupCallId, + )?.getTs() ?? null; + this.updateParticipants(); + + room.on(RoomStateEvent.Update, this.onRoomState); + this.on(GroupCallEvent.ParticipantsChanged, this.onParticipantsChanged); + this.on(GroupCallEvent.GroupCallStateChanged, this.onStateChanged); + this.on(GroupCallEvent.LocalScreenshareStateChanged, this.onLocalFeedsChanged); } - public async create() { + public async create(): Promise { + this.creationTs = Date.now(); this.client.groupCallEventHandler!.groupCalls.set(this.room.roomId, this); + this.client.emit(GroupCallEventHandlerEvent.Outgoing, this); const groupCallState: IGroupCallRoomState = { "m.intent": this.intent, @@ -239,10 +242,69 @@ export class GroupCall extends TypedEventEmitter< return this; } - private setState(newState: GroupCallState): void { - const oldState = this.state; - this.state = newState; - this.emit(GroupCallEvent.GroupCallStateChanged, newState, oldState); + private _state = GroupCallState.LocalCallFeedUninitialized; + + /** + * The group call's state. + */ + public get state(): GroupCallState { + return this._state; + } + + private set state(value: GroupCallState) { + const prevValue = this._state; + if (value !== prevValue) { + this._state = value; + this.emit(GroupCallEvent.GroupCallStateChanged, value, prevValue); + } + } + + private _participants = new Map>(); + + /** + * The current participants in the call, as a map from members to device IDs + * to participant info. + */ + public get participants(): Map> { + return this._participants; + } + + private set participants(value: Map>) { + const prevValue = this._participants; + const participantStateEqual = (x: ParticipantState, y: ParticipantState): boolean => + x.sessionId === y.sessionId && x.screensharing === y.screensharing; + const deviceMapsEqual = (x: Map, y: Map): boolean => + mapsEqual(x, y, participantStateEqual); + + // Only update if the map actually changed + if (!mapsEqual(value, prevValue, deviceMapsEqual)) { + this._participants = value; + this.emit(GroupCallEvent.ParticipantsChanged, value); + } + } + + private _creationTs: number | null = null; + + /** + * The timestamp at which the call was created, or null if it has not yet + * been created. + */ + public get creationTs(): number | null { + return this._creationTs; + } + + private set creationTs(value: number | null) { + this._creationTs = value; + } + + /** + * Executes the given callback on all calls in this group call. + * @param f The callback. + */ + public forEachCall(f: (call: MatrixCall) => void): void { + for (const deviceMap of this.calls.values()) { + for (const call of deviceMap.values()) f(call); + } } private getPreferredFoci(): IFocusInfo[] { @@ -264,8 +326,9 @@ export class GroupCall extends TypedEventEmitter< } public hasLocalParticipant(): boolean { - const userId = this.client.getUserId(); - return this.participants.some((member) => member.userId === userId); + return this.participants.get( + this.room.getMember(this.client.getUserId()!)!, + )?.has(this.client.getDeviceId()!) ?? false; } public async initLocalCallFeed(): Promise { @@ -275,12 +338,12 @@ export class GroupCall extends TypedEventEmitter< throw new Error(`Cannot initialize local call feed in the "${this.state}" state.`); } - this.setState(GroupCallState.InitializingLocalCallFeed); + this.state = GroupCallState.InitializingLocalCallFeed; let stream: MediaStream; let disposed = false; - const onState = (state: GroupCallState) => { + const onState = (state: GroupCallState): void => { if (state === GroupCallState.LocalCallFeedUninitialized) { disposed = true; } @@ -290,7 +353,7 @@ export class GroupCall extends TypedEventEmitter< try { stream = await this.client.getMediaHandler().getUserMediaStream(true, this.type === GroupCallType.Video); } catch (error) { - this.setState(GroupCallState.LocalCallFeedUninitialized); + this.state = GroupCallState.LocalCallFeedUninitialized; throw error; } finally { this.off(GroupCallEvent.GroupCallStateChanged, onState); @@ -299,12 +362,11 @@ export class GroupCall extends TypedEventEmitter< // The call could've been disposed while we were waiting if (disposed) throw new Error("Group call disposed"); - const userId = this.client.getUserId()!; - const callFeed = new CallFeed({ client: this.client, roomId: this.room.roomId, - userId, + userId: this.client.getUserId()!, + deviceId: this.client.getDeviceId()!, stream, purpose: SDPStreamMetadataPurpose.Usermedia, audioMuted: this.initWithAudioMuted || stream.getAudioTracks().length === 0 || this.isPtt, @@ -317,12 +379,12 @@ export class GroupCall extends TypedEventEmitter< this.localCallFeed = callFeed; this.addUserMediaFeed(callFeed); - this.setState(GroupCallState.LocalCallFeedInitialized); + this.state = GroupCallState.LocalCallFeedInitialized; return callFeed; } - public async updateLocalUsermediaStream(stream: MediaStream) { + public async updateLocalUsermediaStream(stream: MediaStream): Promise { if (this.localCallFeed) { const oldStream = this.localCallFeed.stream; this.localCallFeed.setNewStream(stream); @@ -337,9 +399,10 @@ export class GroupCall extends TypedEventEmitter< } } - public async enter() { - if (!(this.state === GroupCallState.LocalCallFeedUninitialized || - this.state === GroupCallState.LocalCallFeedInitialized)) { + public async enter(): Promise { + if (this.state === GroupCallState.LocalCallFeedUninitialized) { + await this.initLocalCallFeed(); + } else if (this.state !== GroupCallState.LocalCallFeedInitialized) { throw new Error(`Cannot enter call in the "${this.state}" state`); } @@ -347,8 +410,6 @@ export class GroupCall extends TypedEventEmitter< await this.initLocalCallFeed(); } - this.addParticipant(this.room.getMember(this.client.getUserId()!)!); - // TODO: Call preferred foci // This needs to be done before we set the state to entered. With the @@ -356,31 +417,29 @@ export class GroupCall extends TypedEventEmitter< // which we don't want, if we have a focus this.chooseFocus(); - await this.sendMemberStateEvent(); + await this.updateMemberState(); this.activeSpeaker = undefined; - this.setState(GroupCallState.Entered); + this.state = GroupCallState.Entered; logger.log(`Entered group call ${this.groupCallId}`); + this.state = GroupCallState.Entered; this.client.on(CallEventHandlerEvent.Incoming, this.onIncomingCall); - const calls = this.client.callEventHandler!.calls.values(); - - for (const call of calls) { + for (const call of this.client.callEventHandler!.calls.values()) { this.onIncomingCall(call); } - // Set up participants for the members currently in the room. - // Other members will be picked up by the RoomState.members event. - for (const stateEvent of this.getMemberStateEvents()) { - this.onMemberStateChanged(stateEvent); - } - - this.retryCallLoopTimeout = setTimeout(this.onRetryCallLoop, this.retryCallInterval); + this.retryCallLoopInterval = setInterval(this.onRetryCallLoop, this.retryCallInterval); + this.activeSpeaker = undefined; this.onActiveSpeakerLoop(); + this.activeSpeakerLoopInterval = setInterval( + this.onActiveSpeakerLoop, + this.activeSpeakerInterval, + ); if (this.foci[0]) { const opponentDevice = { @@ -426,7 +485,8 @@ export class GroupCall extends TypedEventEmitter< return; } - this.addCall(sfuCall); + this.calls.set(this.foci[0], new Map([[opponentDevice.device_id, sfuCall]])); + this.initCall(sfuCall); } } @@ -449,7 +509,7 @@ export class GroupCall extends TypedEventEmitter< this.foci = [focusOfAnotherMember ?? this.client.getFoci()[0]]; } - private dispose() { + private dispose(): void { if (this.localCallFeed) { this.removeUserMediaFeed(this.localCallFeed); this.localCallFeed = undefined; @@ -464,83 +524,67 @@ export class GroupCall extends TypedEventEmitter< this.client.getMediaHandler().stopAllStreams(); - if (this.state !== GroupCallState.Entered) { - return; + if (this.transmitTimer !== null) { + clearTimeout(this.transmitTimer); + this.transmitTimer = null; } - this.removeParticipant(this.room.getMember(this.client.getUserId()!)!); - - this.removeMemberStateEvent(); + if (this.retryCallLoopInterval !== undefined) { + clearInterval(this.retryCallLoopInterval); + this.retryCallLoopInterval = undefined; + } - while (this.calls.length > 0) { - this.removeCall(this.calls[this.calls.length - 1], CallErrorCode.UserHangup); + if (this.state !== GroupCallState.Entered) { + return; } + this.forEachCall(call => this.disposeCall(call, CallErrorCode.UserHangup)); + this.calls.clear(); + this.activeSpeaker = undefined; - clearTimeout(this.activeSpeakerLoopTimeout); + clearInterval(this.activeSpeakerLoopInterval); this.retryCallCounts.clear(); - clearTimeout(this.retryCallLoopTimeout); - - for (const [userId] of this.memberStateExpirationTimers) { - clearTimeout(this.memberStateExpirationTimers.get(userId)); - this.memberStateExpirationTimers.delete(userId); - } - - if (this.transmitTimer !== null) { - clearTimeout(this.transmitTimer); - this.transmitTimer = null; - } + clearInterval(this.retryCallLoopInterval); this.client.removeListener(CallEventHandlerEvent.Incoming, this.onIncomingCall); } - public leave() { - if (this.transmitTimer !== null) { - clearTimeout(this.transmitTimer); - this.transmitTimer = null; - } - + public leave(): void { this.dispose(); - this.setState(GroupCallState.LocalCallFeedUninitialized); + this.state = GroupCallState.LocalCallFeedUninitialized; } - public async terminate(emitStateEvent = true) { + public async terminate(emitStateEvent = true): Promise { this.dispose(); - if (this.transmitTimer !== null) { - clearTimeout(this.transmitTimer); - this.transmitTimer = null; - } - - this.participants = []; + this.room.off(RoomStateEvent.Update, this.onRoomState); this.client.groupCallEventHandler!.groupCalls.delete(this.room.roomId); + this.client.emit(GroupCallEventHandlerEvent.Ended, this); + this.state = GroupCallState.Ended; if (emitStateEvent) { const existingStateEvent = this.room.currentState.getStateEvents( EventType.GroupCallPrefix, this.groupCallId, - ); + )!; await this.client.sendStateEvent( this.room.roomId, EventType.GroupCallPrefix, { ...existingStateEvent.getContent(), - ["m.terminated"]: GroupCallTerminationReason.CallEnded, + "m.terminated": GroupCallTerminationReason.CallEnded, }, this.groupCallId, ); } - - this.client.emit(GroupCallEventHandlerEvent.Ended, this); - this.setState(GroupCallState.Ended); } - /** + /* * Local Usermedia */ - public isLocalVideoMuted() { + public isLocalVideoMuted(): boolean { if (this.localCallFeed) { return this.localCallFeed.isVideoMuted(); } @@ -548,7 +592,7 @@ export class GroupCall extends TypedEventEmitter< return true; } - public isMicrophoneMuted() { + public isMicrophoneMuted(): boolean { if (this.localCallFeed) { return this.localCallFeed.isAudioMuted(); } @@ -584,17 +628,18 @@ export class GroupCall extends TypedEventEmitter< } } - for (const call of this.calls) { - call.localUsermediaFeed?.setAudioVideoMuted(muted, null); - } + this.forEachCall(call => call.localUsermediaFeed?.setAudioVideoMuted(muted, null)); - if (sendUpdatesBefore) { - try { - await Promise.all(this.calls.map(c => c.sendMetadataUpdate())); - } catch (e) { - logger.info("Failed to send one or more metadata updates", e); - } - } + const sendUpdates = async (): Promise => { + const updates: Promise[] = []; + this.forEachCall(call => updates.push(call.sendMetadataUpdate())); + + await Promise.all(updates).catch( + e => logger.info("Failed to send some metadata updates", e), + ); + }; + + if (sendUpdatesBefore) await sendUpdates(); if (this.localCallFeed) { logger.log(`groupCall ${this.groupCallId} setMicrophoneMuted stream ${ @@ -610,22 +655,13 @@ export class GroupCall extends TypedEventEmitter< this.initWithAudioMuted = muted; } - for (const call of this.calls) { - setTracksEnabled(call.localUsermediaFeed!.stream.getAudioTracks(), !muted); - } - + this.forEachCall(call => setTracksEnabled(call.localUsermediaFeed!.stream.getAudioTracks(), !muted)); this.emit(GroupCallEvent.LocalMuteStateChanged, muted, this.isLocalVideoMuted()); - if (!sendUpdatesBefore) { - try { - await Promise.all(this.calls.map(c => c.sendMetadataUpdate())); - } catch (e) { - logger.info("Failed to send one or more metadata updates", e); - } - } + if (!sendUpdatesBefore) await sendUpdates(); this.emit(GroupCallEvent.LocalMuteStateChanged, muted, this.isLocalVideoMuted()); - this.sendMemberStateEvent(); + this.updateMemberState(); return true; } @@ -652,12 +688,13 @@ export class GroupCall extends TypedEventEmitter< this.initWithVideoMuted = muted; } - for (const call of this.calls) { - call.setLocalVideoMuted(muted); - } + const updates: Promise[] = []; + this.forEachCall(call => updates.push(call.setLocalVideoMuted(muted))); + await Promise.all(updates); this.emit(GroupCallEvent.LocalMuteStateChanged, this.isMicrophoneMuted(), muted); - this.sendMemberStateEvent(); + this.updateMemberState(); + return true; } @@ -674,7 +711,7 @@ export class GroupCall extends TypedEventEmitter< const stream = await this.client.getMediaHandler().getScreensharingStream(opts); for (const track of stream.getTracks()) { - const onTrackEnded = () => { + const onTrackEnded = (): void => { this.setScreensharingEnabled(false); track.removeEventListener("ended", onTrackEnded); }; @@ -689,6 +726,7 @@ export class GroupCall extends TypedEventEmitter< client: this.client, roomId: this.room.roomId, userId: this.client.getUserId()!, + deviceId: this.client.getDeviceId()!, stream, purpose: SDPStreamMetadataPurpose.Screenshare, audioMuted: false, @@ -704,10 +742,10 @@ export class GroupCall extends TypedEventEmitter< ); // TODO: handle errors - await Promise.all(this.calls.map(call => call.pushLocalFeed(call.isFocus + this.forEachCall(call => call.pushLocalFeed(call.isFocus ? this.localScreenshareFeed! : this.localScreenshareFeed!.clone(), - ))); + )); return true; } catch (error) { @@ -722,9 +760,9 @@ export class GroupCall extends TypedEventEmitter< return false; } } else { - await Promise.all(this.calls.map(call => { + this.forEachCall(call => { if (call.localScreensharingFeed) call.removeLocalFeed(call.localScreensharingFeed); - })); + }); this.client.getMediaHandler().stopScreensharingStream(this.localScreenshareFeed!.stream); // We have to remove the feed manually as MatrixCall has its clone, // so it won't be removed automatically @@ -742,7 +780,7 @@ export class GroupCall extends TypedEventEmitter< return !!this.localScreenshareFeed; } - /** + /* * Call Setup * * There are two different paths for calls to be created: @@ -751,7 +789,7 @@ export class GroupCall extends TypedEventEmitter< * as they are observed by the RoomState.members event. */ - private onIncomingCall = (newCall: MatrixCall) => { + private onIncomingCall = (newCall: MatrixCall): void => { // The incoming calls may be for another room, which we will ignore. if (newCall.roomId !== this.room.roomId) { return; @@ -769,371 +807,196 @@ export class GroupCall extends TypedEventEmitter< return; } - const opponentMemberId = newCall.getOpponentMember()?.userId; - const existingCall = opponentMemberId ? this.getCallByUserId(opponentMemberId) : null; - - if (existingCall && existingCall.callId === newCall.callId) { + const opponent = newCall.getOpponentMember(); + if (opponent === undefined) { + logger.warn("Incoming call with no member. Ignoring."); return; } - logger.log(`GroupCall: incoming call from: ${opponentMemberId} with ID ${newCall.callId}`); + const deviceMap = this.calls.get(opponent) ?? new Map(); + const prevCall = deviceMap.get(newCall.getOpponentDeviceId()!); - // we are handlng this call as a PTT call, so enable PTT semantics - newCall.isPtt = this.isPtt; + if (prevCall?.callId === newCall.callId) return; - // Check if the user calling has an existing call and use this call instead. - if (existingCall) { - this.replaceCall(existingCall, newCall); - } else { - this.addCall(newCall); - } + logger.log(`GroupCall: incoming call from ${opponent.userId} with ID ${newCall.callId}`); + + if (prevCall) this.disposeCall(prevCall, CallErrorCode.Replaced); + this.initCall(newCall); newCall.answerWithCallFeeds(this.getLocalFeeds().map((feed) => feed.clone())); + + deviceMap.set(newCall.getOpponentDeviceId()!, newCall); + this.calls.set(opponent, deviceMap); + this.emit(GroupCallEvent.CallsChanged, this.calls); }; /** - * Room Member State + * Determines whether a given participant expects us to call them (versus + * them calling us). + * @param userId The participant's user ID. + * @param deviceId The participant's device ID. + * @returns Whether we need to place an outgoing call to the participant. */ - - private getMemberStateEvents(): MatrixEvent[]; - private getMemberStateEvents(userId: string): MatrixEvent | null; - private getMemberStateEvents(userId?: string): MatrixEvent[] | MatrixEvent | null { - if (userId != null) { - const event = this.room.currentState.getStateEvents(EventType.GroupCallMemberPrefix, userId); - return callMemberStateIsExpired(event) ? null : event; - } else { - return this.room.currentState.getStateEvents(EventType.GroupCallMemberPrefix) - .filter(event => !callMemberStateIsExpired(event)); - } - } - - private async sendMemberStateEvent(): Promise { - if (!this.participants.includes(this.room.getMember(this.client.getUserId()!)!)) return null; - - const send = (): Promise => this.updateMemberCallState({ - "m.call_id": this.groupCallId, - "m.devices": [ - { - "device_id": this.client.getDeviceId()!, - "session_id": this.client.getSessionId(), - "feeds": this.getLocalFeeds().map((feed) => ({ - purpose: feed.purpose, - })), - "m.foci.active": this.foci, - "m.foci.preferred": this.getPreferredFoci(), - // TODO: Add data channels - }, - ], - // TODO "m.foci" - }); - - const res = await send(); - - // Clear the old interval first, so that it isn't forgot - if (this.resendMemberStateTimer !== null) clearInterval(this.resendMemberStateTimer); - // Resend the state event every so often so it doesn't become stale - this.resendMemberStateTimer = setInterval(async () => { - logger.log("Resending call member state"); - await send(); - }, CALL_MEMBER_STATE_TIMEOUT * 3 / 4); - - return res; - } - - private async removeMemberStateEvent(): Promise { - if (this.resendMemberStateTimer !== null) clearInterval(this.resendMemberStateTimer); - this.resendMemberStateTimer = null; - return await this.updateMemberCallState(undefined, true); - } - - private async updateMemberCallState( - memberCallState?: IGroupCallRoomMemberCallState, - keepAlive = false, - ): Promise { + private wantsOutgoingCall(userId: string, deviceId: string): boolean { const localUserId = this.client.getUserId()!; - - const memberState = this.getMemberStateEvents(localUserId)?.getContent(); - - let calls: IGroupCallRoomMemberCallState[] = []; - - // Sanitize existing member state event - if (memberState && Array.isArray(memberState["m.calls"])) { - calls = memberState["m.calls"].filter((call) => !!call); - } - - const existingCallIndex = calls.findIndex((call) => call && call["m.call_id"] === this.groupCallId); - - if (existingCallIndex !== -1) { - if (memberCallState) { - calls.splice(existingCallIndex, 1, memberCallState); - } else { - calls.splice(existingCallIndex, 1); - } - } else if (memberCallState) { - calls.push(memberCallState); - } - - const content = { - "m.calls": calls, - "m.expires_ts": Date.now() + CALL_MEMBER_STATE_TIMEOUT, - }; - - return this.client.sendStateEvent( - this.room.roomId, EventType.GroupCallMemberPrefix, content, localUserId, { keepAlive }, + const localDeviceId = this.client.getDeviceId()!; + return ( + // If a user's ID is less than our own, they'll call us + userId >= localUserId + // If this is another one of our devices, compare device IDs to tell whether it'll call us + && (userId !== localUserId || deviceId > localDeviceId) ); } - public onMemberStateChanged = async (event: MatrixEvent) => { - if (this.foci[0]) { - // TODO: Check if someone switched to one of our preferred foci - return; - } - - // If we haven't entered the call yet, we don't care - if (this.state !== GroupCallState.Entered) { - return; - } - - // The member events may be received for another room, which we will ignore. - if (event.getRoomId() !== this.room.roomId) return; - - const member = this.room.getMember(event.getStateKey()!); - if (!member) { - logger.warn(`Couldn't find room member for ${event.getStateKey()}: ignoring member state event!`); - return; - } - - // Don't process your own member. - const localUserId = this.client.getUserId()!; - if (member.userId === localUserId) return; - - logger.debug(`Processing member state event for ${member.userId}`); - - const ignore = () => { - this.removeParticipant(member); - clearTimeout(this.memberStateExpirationTimers.get(member.userId)); - this.memberStateExpirationTimers.delete(member.userId); - }; - - const content = event.getContent(); - const callsState = !callMemberStateIsExpired(event) && Array.isArray(content["m.calls"]) - ? content["m.calls"].filter((call) => call) - : []; // Ignore expired device data - - if (callsState.length === 0) { - logger.info(`Ignoring member state from ${member.userId} member not in any calls.`); - ignore(); - return; - } - - // Currently we only support a single call per room. So grab the first call. - const callState = callsState[0]; - const callId = callState["m.call_id"]; - - if (!callId) { - logger.warn(`Room member ${member.userId} does not have a valid m.call_id set. Ignoring.`); - ignore(); - return; - } - - if (callId !== this.groupCallId) { - logger.warn(`Call id ${callId} does not match group call id ${this.groupCallId}, ignoring.`); - ignore(); - return; - } - - this.addParticipant(member); - - clearTimeout(this.memberStateExpirationTimers.get(member.userId)); - this.memberStateExpirationTimers.set(member.userId, setTimeout(() => { - logger.warn(`Call member state for ${member.userId} has expired`); - this.removeParticipant(member); - }, content["m.expires_ts"] - Date.now())); - - // Only initiate a call with a user who has a userId that is lexicographically - // less than your own. Otherwise, that user will call you. - if (member.userId < localUserId) { - logger.debug(`Waiting for ${member.userId} to send call invite.`); - return; - } - - const opponentDevice = this.getDeviceForMember(member.userId); - - if (!opponentDevice) { - logger.warn(`No opponent device found for ${member.userId}, ignoring.`); - this.emit( - GroupCallEvent.Error, - new GroupCallUnknownDeviceError(member.userId), - ); - return; - } - - const existingCall = this.getCallByUserId(member.userId); - - if ( - existingCall && - existingCall.getOpponentSessionId() === opponentDevice.session_id - ) { - return; - } - - const newCall = createNewMatrixCall( - this.client, - this.room.roomId, - { - invitee: member.userId, - opponentDeviceId: opponentDevice.device_id, - opponentSessionId: opponentDevice.session_id, - groupCallId: this.groupCallId, - }, - ); - - if (!newCall) { - logger.error("Failed to create call!"); - return; - } - - if (existingCall) { - logger.debug(`Replacing call ${existingCall.callId} to ${member.userId} with ${newCall.callId}`); - this.replaceCall(existingCall, newCall, CallErrorCode.NewSession); - } else { - logger.debug(`Adding call ${newCall.callId} to ${member.userId}`); - this.addCall(newCall); - } - - newCall.isPtt = this.isPtt; - - const requestScreenshareFeed = opponentDevice.feeds.some( - (feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); - - logger.debug( - `Placing call to ${member.userId}/${opponentDevice.device_id} session ID ${opponentDevice.session_id}.`, - ); + /** + * Places calls to all participants that we're responsible for calling. + */ + private placeOutgoingCalls(): void { + if (this.foci[0]) return; + + let callsChanged = false; + + for (const [member, participantMap] of this.participants) { + const callMap = this.calls.get(member) ?? new Map(); + + for (const [deviceId, participant] of participantMap) { + const prevCall = callMap.get(deviceId); + + if ( + prevCall?.getOpponentSessionId() !== participant.sessionId + && this.wantsOutgoingCall(member.userId, deviceId) + ) { + callsChanged = true; + + if (prevCall !== undefined) { + logger.debug(`Replacing call ${prevCall.callId} to ${member.userId} ${deviceId}`); + this.disposeCall(prevCall, CallErrorCode.NewSession); + } + + const newCall = createNewMatrixCall( + this.client, + this.room.roomId, + { + invitee: member.userId, + opponentDeviceId: deviceId, + opponentSessionId: participant.sessionId, + groupCallId: this.groupCallId, + }, + ); + + if (newCall === null) { + logger.error(`Failed to create call with ${member.userId} ${deviceId}`); + callMap.delete(deviceId); + } else { + this.initCall(newCall); + callMap.set(deviceId, newCall); + + logger.debug( + `Placing call to ${member.userId} ${deviceId} (session ${participant.sessionId})`, + ); + + newCall.placeCallWithCallFeeds( + this.getLocalFeeds().map(feed => feed.clone()), + participant.screensharing, + ).then(() => { + if (this.dataChannelsEnabled) { + newCall.createDataChannel("datachannel", this.dataChannelOptions); + } + }).catch(e => { + logger.warn(`Failed to place call to ${member.userId}`, e); + + if (e instanceof CallError && e.code === GroupCallErrorCode.UnknownDevice) { + this.emit(GroupCallEvent.Error, e); + } else { + this.emit( + GroupCallEvent.Error, + new GroupCallError( + GroupCallErrorCode.PlaceCallFailed, + `Failed to place call to ${member.userId}`, + ), + ); + } + + this.disposeCall(newCall, CallErrorCode.SignallingFailed); + if (callMap.get(deviceId) === newCall) callMap.delete(deviceId); + }); + } + } + } - try { - await newCall.placeCallWithCallFeeds( - this.getLocalFeeds().map(feed => feed.clone()), - requestScreenshareFeed, - ); - } catch (e) { - logger.warn(`Failed to place call to ${member.userId}!`, e); - if (e instanceof CallError && e.code === GroupCallErrorCode.UnknownDevice) { - this.emit(GroupCallEvent.Error, e); + if (callMap.size > 0) { + this.calls.set(member, callMap); } else { - this.emit( - GroupCallEvent.Error, - new GroupCallError( - GroupCallErrorCode.PlaceCallFailed, - `Failed to place call to ${member.userId}.`, - ), - ); + this.calls.delete(member); } - this.removeCall(newCall, CallErrorCode.SignallingFailed); - return; - } - - if (this.dataChannelsEnabled) { - newCall.createDataChannel("datachannel", this.dataChannelOptions); - } - }; - - public getDeviceForMember(userId: string): IGroupCallRoomMemberDevice | undefined { - const memberStateEvent = this.getMemberStateEvents(userId); - - if (!memberStateEvent) { - return undefined; - } - - const memberState = memberStateEvent.getContent(); - const memberGroupCallState = memberState["m.calls"]?.find( - (call) => call && call["m.call_id"] === this.groupCallId); - - if (!memberGroupCallState) { - return undefined; } - const memberDevices = memberGroupCallState["m.devices"]; - - if (!memberDevices || memberDevices.length === 0) { - return undefined; - } - - // NOTE: For now we only support one device so we use the device id in the first source. - return memberDevices[0]; + if (callsChanged) this.emit(GroupCallEvent.CallsChanged, this.calls); } - private onRetryCallLoop = () => { - for (const event of this.getMemberStateEvents()) { - const memberId = event.getStateKey()!; - const existingCall = this.calls.find((call) => getCallUserId(call) === memberId); - const retryCallCount = this.retryCallCounts.get(memberId) || 0; - - if (!existingCall && retryCallCount < 3) { - this.retryCallCounts.set(memberId, retryCallCount + 1); - this.onMemberStateChanged(event); - } - } - - this.retryCallLoopTimeout = setTimeout(this.onRetryCallLoop, this.retryCallInterval); - }; - - /** - * Call Event Handlers + /* + * Room Member State */ - public getCallByUserId(userId: string): MatrixCall | undefined { - return this.calls.find((call) => getCallUserId(call) === userId); - } - - private addCall(call: MatrixCall) { - this.calls.push(call); - this.initCall(call); - this.emit(GroupCallEvent.CallsChanged, this.calls); - } - - private replaceCall(existingCall: MatrixCall, replacementCall: MatrixCall, hangupReason = CallErrorCode.Replaced) { - const existingCallIndex = this.calls.indexOf(existingCall); - - if (existingCallIndex === -1) { - throw new Error("Couldn't find call to replace"); - } - - this.calls.splice(existingCallIndex, 1, replacementCall); - - this.disposeCall(existingCall, hangupReason); - this.initCall(replacementCall); - - this.emit(GroupCallEvent.CallsChanged, this.calls); + private getMemberStateEvents(): MatrixEvent[]; + private getMemberStateEvents(userId: string): MatrixEvent | null; + private getMemberStateEvents(userId?: string): MatrixEvent[] | MatrixEvent | null { + return userId === undefined + ? this.room.currentState.getStateEvents(EventType.GroupCallMemberPrefix) + : this.room.currentState.getStateEvents(EventType.GroupCallMemberPrefix, userId); } - private removeCall(call: MatrixCall, hangupReason: CallErrorCode) { - this.disposeCall(call, hangupReason); - - const callIndex = this.calls.indexOf(call); - - if (callIndex === -1) { - throw new Error("Couldn't find call to remove"); + private onRetryCallLoop = (): void => { + let needsRetry = false; + + for (const [member, participantMap] of this.participants) { + const callMap = this.calls.get(member); + let retriesMap = this.retryCallCounts.get(member); + + for (const [deviceId, participant] of participantMap) { + const call = callMap?.get(deviceId); + const retries = retriesMap?.get(deviceId) ?? 0; + + if ( + call?.getOpponentSessionId() !== participant.sessionId + && this.wantsOutgoingCall(member.userId, deviceId) + && retries < 3 + ) { + if (retriesMap === undefined) { + retriesMap = new Map(); + this.retryCallCounts.set(member, retriesMap); + } + retriesMap.set(deviceId, retries + 1); + needsRetry = true; + } + } } - this.calls.splice(callIndex, 1); - - this.emit(GroupCallEvent.CallsChanged, this.calls); - } + if (needsRetry) this.placeOutgoingCalls(); + }; - private initCall(call: MatrixCall) { + private initCall(call: MatrixCall): void { const opponentMemberId = getCallUserId(call); if (!opponentMemberId) { throw new Error("Cannot init call without user id"); } - const onCallFeedsChanged = () => this.onCallFeedsChanged(call); - const onCallStateChanged = - (state: CallState, oldState: CallState | undefined) => this.onCallStateChanged(call, state, oldState); + const onCallFeedsChanged = (): void => this.onCallFeedsChanged(call); + const onCallStateChanged = ( + state: CallState, + oldState?: CallState, + ): void => this.onCallStateChanged(call, state, oldState); const onCallHangup = this.onCallHangup; - const onCallReplaced = (newCall: MatrixCall) => this.replaceCall(call, newCall); + const onCallReplaced = (newCall: MatrixCall): void => this.onCallReplaced(call, newCall); - this.callHandlers.set(opponentMemberId, { + let deviceMap = this.callHandlers.get(opponentMemberId); + if (deviceMap === undefined) { + deviceMap = new Map(); + this.callHandlers.set(opponentMemberId, deviceMap); + } + + deviceMap.set(call.getOpponentDeviceId()!, { onCallFeedsChanged, onCallStateChanged, onCallHangup, @@ -1145,31 +1008,36 @@ export class GroupCall extends TypedEventEmitter< call.on(CallEvent.Hangup, onCallHangup); call.on(CallEvent.Replaced, onCallReplaced); + call.isPtt = this.isPtt; + this.reEmitter.reEmit(call, Object.values(CallEvent)); onCallFeedsChanged(); } - private disposeCall(call: MatrixCall, hangupReason: CallErrorCode) { + private disposeCall(call: MatrixCall, hangupReason: CallErrorCode): void { const opponentMemberId = getCallUserId(call); + const opponentDeviceId = call.getOpponentDeviceId()!; if (!opponentMemberId) { throw new Error("Cannot dispose call without user id"); } + const deviceMap = this.callHandlers.get(opponentMemberId)!; const { onCallFeedsChanged, onCallStateChanged, onCallHangup, onCallReplaced, - } = this.callHandlers.get(opponentMemberId)!; + } = deviceMap.get(opponentDeviceId)!; call.removeListener(CallEvent.FeedsChanged, onCallFeedsChanged); call.removeListener(CallEvent.State, onCallStateChanged); call.removeListener(CallEvent.Hangup, onCallHangup); call.removeListener(CallEvent.Replaced, onCallReplaced); - this.callHandlers.delete(opponentMemberId); + deviceMap.delete(opponentMemberId); + if (deviceMap.size === 0) this.callHandlers.delete(opponentMemberId); if (call.hangupReason === CallErrorCode.Replaced) { return; @@ -1179,13 +1047,13 @@ export class GroupCall extends TypedEventEmitter< call.hangup(hangupReason, false); } - const usermediaFeed = this.getUserMediaFeedByUserId(opponentMemberId); + const usermediaFeed = this.getUserMediaFeed(opponentMemberId, opponentDeviceId); if (usermediaFeed) { this.removeUserMediaFeed(usermediaFeed); } - const screenshareFeed = this.getScreenshareFeedByUserId(opponentMemberId); + const screenshareFeed = this.getScreenshareFeed(opponentMemberId, opponentDeviceId); if (screenshareFeed) { this.removeScreenshareFeed(screenshareFeed); @@ -1193,18 +1061,26 @@ export class GroupCall extends TypedEventEmitter< } private onCallFeedsChanged = (call: MatrixCall) => { - this.sendMemberStateEvent(); + this.updateMemberState(); // Find removed feeds [...this.userMediaFeeds, ...this.screenshareFeeds].filter((gf) => gf.disposed).forEach((feed) => { - // Only remove the feed if the other feed array doesn't have a feed - // from the given user + // Only remove the participant if the other feed array doesn't have a feed + // from the given participant if ( !(feed.purpose === SDPStreamMetadataPurpose.Usermedia ? this.screenshareFeeds - : this.userMediaFeeds).some((f) => f.userId === feed.getMember()!.userId) + : this.userMediaFeeds).some((f) => + ( + f.userId === feed.userId + && f.deviceId === feed.deviceId + ), + ) ) { - this.removeParticipant(feed.getMember()!); + this.participants.get(feed.getMember()!)?.delete(feed.deviceId!); + if (this.participants.get(feed.getMember()!)?.size === 0) { + this.participants.delete(feed.getMember()!); + } } if (feed.purpose === SDPStreamMetadataPurpose.Usermedia) this.removeUserMediaFeed(feed); @@ -1215,14 +1091,34 @@ export class GroupCall extends TypedEventEmitter< call.getRemoteFeeds().filter((cf) => { return !this.userMediaFeeds.find((gf) => gf.stream.id === cf.stream.id); }).forEach((feed) => { - this.addParticipant(feed.getMember()!); + const participant = this.participants.get(feed.getMember()!); + if (participant) { + participant.set( + feed.deviceId!, + { + sessionId: call.getOpponentSessionId()!, + screensharing: false, + }, + ); + } else { + this.participants.set( + feed.getMember()!, + new Map([[ + feed.deviceId!, + { + sessionId: call.getOpponentSessionId()!, + screensharing: false, + }, + ]]), + ); + } if (feed.purpose === SDPStreamMetadataPurpose.Usermedia) this.addUserMediaFeed(feed); else if (feed.purpose === SDPStreamMetadataPurpose.Screenshare) this.addScreenshareFeed(feed); }); }; - private onCallStateChanged = (call: MatrixCall, state: CallState, _oldState: CallState | undefined) => { + private onCallStateChanged = (call: MatrixCall, state: CallState, _oldState: CallState | undefined): void => { const audioMuted = this.localCallFeed!.isAudioMuted(); if ( @@ -1242,39 +1138,66 @@ export class GroupCall extends TypedEventEmitter< } if (state === CallState.Connected) { - this.retryCallCounts.delete(getCallUserId(call)!); - // if we're calling an SFU, subscribe to its feeds if (call.isFocus) { call.subscribeToSFU(); } + + const opponent = call.getOpponentMember()!; + const retriesMap = this.retryCallCounts.get(opponent); + retriesMap?.delete(call.getOpponentDeviceId()!); + if (retriesMap?.size === 0) this.retryCallCounts.delete(opponent); } }; - private onCallHangup = (call: MatrixCall) => { - if (call.hangupReason === CallErrorCode.Replaced) { - return; + private onCallHangup = (call: MatrixCall): void => { + if (call.hangupReason === CallErrorCode.Replaced) return; + + const opponent = call.getOpponentMember() ?? this.room.getMember(call.invitee!)!; + const deviceMap = this.calls.get(opponent); + + // Sanity check that this call is in fact in the map + if (deviceMap?.get(call.getOpponentDeviceId()!) === call) { + this.disposeCall(call, call.hangupReason as CallErrorCode); + deviceMap.delete(call.getOpponentDeviceId()!); + if (deviceMap.size === 0) this.calls.delete(opponent); + this.emit(GroupCallEvent.CallsChanged, this.calls); + } + }; + + private onCallReplaced = (prevCall: MatrixCall, newCall: MatrixCall): void => { + const opponent = prevCall.getOpponentMember()!; + + let deviceMap = this.calls.get(opponent); + if (deviceMap === undefined) { + deviceMap = new Map(); + this.calls.set(opponent, deviceMap); } - this.removeCall(call, call.hangupReason as CallErrorCode); + this.disposeCall(prevCall, CallErrorCode.Replaced); + this.initCall(newCall); + deviceMap.set(prevCall.getOpponentDeviceId()!, newCall); + this.emit(GroupCallEvent.CallsChanged, this.calls); }; - /** + /* * UserMedia CallFeed Event Handlers */ - public getUserMediaFeedByUserId(userId: string) { - return this.userMediaFeeds.find((feed) => feed.userId === userId); + public getUserMediaFeed(userId: string, deviceId: string): CallFeed | undefined { + return this.userMediaFeeds.find(f => f.userId === userId && f.deviceId! === deviceId); } - private addUserMediaFeed(callFeed: CallFeed) { + private addUserMediaFeed(callFeed: CallFeed): void { this.userMediaFeeds.push(callFeed); callFeed.measureVolumeActivity(true); this.emit(GroupCallEvent.UserMediaFeedsChanged, this.userMediaFeeds); } - private removeUserMediaFeed(callFeed: CallFeed) { - const feedIndex = this.userMediaFeeds.findIndex((feed) => feed.userId === callFeed.userId); + private removeUserMediaFeed(callFeed: CallFeed): void { + const feedIndex = this.userMediaFeeds.findIndex( + f => f.userId === callFeed.userId && f.deviceId! === callFeed.deviceId, + ); if (feedIndex === -1) { throw new Error("Couldn't find user media feed to remove"); @@ -1285,36 +1208,27 @@ export class GroupCall extends TypedEventEmitter< callFeed.dispose(); this.emit(GroupCallEvent.UserMediaFeedsChanged, this.userMediaFeeds); - if ( - this.activeSpeaker === callFeed.userId && - this.userMediaFeeds.length > 0 - ) { - this.activeSpeaker = this.userMediaFeeds[0].userId; + if (this.activeSpeaker === callFeed) { + this.activeSpeaker = this.userMediaFeeds[0]; this.emit(GroupCallEvent.ActiveSpeakerChanged, this.activeSpeaker); } } - private onActiveSpeakerLoop = () => { + private onActiveSpeakerLoop = (): void => { let topAvg: number | undefined = undefined; - let nextActiveSpeaker: string | undefined = undefined; + let nextActiveSpeaker: CallFeed | undefined = undefined; for (const callFeed of this.userMediaFeeds) { - if (callFeed.userId === this.client.getUserId() && this.userMediaFeeds.length > 1) { - continue; - } - - let total = 0; - - for (let i = 0; i < callFeed.speakingVolumeSamples.length; i++) { - const volume = callFeed.speakingVolumeSamples[i]; - total += Math.max(volume, SPEAKING_THRESHOLD); - } + if (callFeed.isLocal() && this.userMediaFeeds.length > 1) continue; + const total = callFeed.speakingVolumeSamples.reduce( + (acc, volume) => acc + Math.max(volume, SPEAKING_THRESHOLD), + ); const avg = total / callFeed.speakingVolumeSamples.length; if (!topAvg || avg > topAvg) { topAvg = avg; - nextActiveSpeaker = callFeed.userId; + nextActiveSpeaker = callFeed; } } @@ -1322,28 +1236,25 @@ export class GroupCall extends TypedEventEmitter< this.activeSpeaker = nextActiveSpeaker; this.emit(GroupCallEvent.ActiveSpeakerChanged, this.activeSpeaker); } - - this.activeSpeakerLoopTimeout = setTimeout( - this.onActiveSpeakerLoop, - this.activeSpeakerInterval, - ); }; - /** + /* * Screenshare Call Feed Event Handlers */ - public getScreenshareFeedByUserId(userId: string) { - return this.screenshareFeeds.find((feed) => feed.userId === userId); + public getScreenshareFeed(userId: string, deviceId: string): CallFeed | undefined { + return this.screenshareFeeds.find(f => f.userId === userId && f.deviceId! === deviceId); } - private addScreenshareFeed(callFeed: CallFeed) { + private addScreenshareFeed(callFeed: CallFeed): void { this.screenshareFeeds.push(callFeed); this.emit(GroupCallEvent.ScreenshareFeedsChanged, this.screenshareFeeds); } - private removeScreenshareFeed(callFeed: CallFeed) { - const feedIndex = this.screenshareFeeds.findIndex((feed) => feed.userId === callFeed.userId); + private removeScreenshareFeed(callFeed: CallFeed): void { + const feedIndex = this.screenshareFeeds.findIndex( + f => f.userId === callFeed.userId && f.deviceId! === callFeed.deviceId, + ); if (feedIndex === -1) { throw new Error("Couldn't find screenshare feed to remove"); @@ -1356,30 +1267,227 @@ export class GroupCall extends TypedEventEmitter< } /** - * Participant Management + * Recalculates and updates the participant map to match the room state. */ + private updateParticipants(): void { + if (this.participantsExpirationTimer !== null) { + clearTimeout(this.participantsExpirationTimer); + this.participantsExpirationTimer = null; + } - private addParticipant(member: RoomMember) { - if (this.participants.find((m) => m.userId === member.userId)) { + if (this.state === GroupCallState.Ended) { + this.participants = new Map(); return; } - this.participants.push(member); + const participants = new Map>(); + const now = Date.now(); + const entered = this.state === GroupCallState.Entered; + let nextExpiration = Infinity; + + for (const e of this.getMemberStateEvents()) { + const member = this.room.getMember(e.getStateKey()!); + const content = e.getContent>(); + const calls: Record[] = Array.isArray(content["m.calls"]) ? content["m.calls"] : []; + const call = calls.find(call => call["m.call_id"] === this.groupCallId); + const devices: Record[] = Array.isArray(call?.["m.devices"]) ? call!["m.devices"] : []; + + // Filter out invalid and expired devices + let validDevices = devices.filter(d => ( + typeof d.device_id === "string" + && typeof d.session_id === "string" + && typeof d.expires_ts === "number" + && d.expires_ts > now + && Array.isArray(d.feeds) + )) as unknown as IGroupCallRoomMemberDevice[]; + + // Apply local echo for the unentered case + if (!entered && member?.userId === this.client.getUserId()!) { + validDevices = validDevices.filter(d => d.device_id !== this.client.getDeviceId()!); + } + + // Must have a connected device and be joined to the room + if (validDevices.length > 0 && member?.membership === "join") { + const deviceMap = new Map(); + participants.set(member, deviceMap); + + for (const d of validDevices) { + deviceMap.set(d.device_id, { + sessionId: d.session_id, + screensharing: d.feeds.some(f => f.purpose === SDPStreamMetadataPurpose.Screenshare), + }); + if (d.expires_ts < nextExpiration) nextExpiration = d.expires_ts; + } + } + } + + // Apply local echo for the entered case + if (entered) { + const localMember = this.room.getMember(this.client.getUserId()!)!; + let deviceMap = participants.get(localMember); + if (deviceMap === undefined) { + deviceMap = new Map(); + participants.set(localMember, deviceMap); + } + + if (!deviceMap.has(this.client.getDeviceId()!)) { + deviceMap.set(this.client.getDeviceId()!, { + sessionId: this.client.getSessionId(), + screensharing: this.getLocalFeeds().some(f => f.purpose === SDPStreamMetadataPurpose.Screenshare), + }); + } + } + + this.participants = participants; + if (nextExpiration < Infinity) { + this.participantsExpirationTimer = setTimeout(() => this.updateParticipants(), nextExpiration - now); + } + } + + /** + * Updates the local user's member state with the devices returned by the given function. + * @param fn A function from the current devices to the new devices. If it + * returns null, the update will be skipped. + * @param keepAlive Whether the request should outlive the window. + */ + private async updateDevices( + fn: (devices: IGroupCallRoomMemberDevice[]) => IGroupCallRoomMemberDevice[] | null, + keepAlive = false, + ): Promise { + const now = Date.now(); + const localUserId = this.client.getUserId()!; + + const event = this.getMemberStateEvents(localUserId); + const content = event?.getContent>() ?? {}; + const calls: Record[] = Array.isArray(content["m.calls"]) ? content["m.calls"] : []; + + let call: Record | null = null; + const otherCalls: Record[] = []; + for (const c of calls) { + if (c["m.call_id"] === this.groupCallId) { + call = c; + } else { + otherCalls.push(c); + } + } + if (call === null) call = {}; + + const devices: Record[] = Array.isArray(call["m.devices"]) ? call["m.devices"] : []; - this.emit(GroupCallEvent.ParticipantsChanged, this.participants); - this.client.emit(GroupCallEventHandlerEvent.Participants, this.participants, this); + // Filter out invalid and expired devices + const validDevices = devices.filter(d => ( + typeof d.device_id === "string" + && typeof d.session_id === "string" + && typeof d.expires_ts === "number" + && d.expires_ts > now + && Array.isArray(d.feeds) + )) as unknown as IGroupCallRoomMemberDevice[]; + + const newDevices = fn(validDevices); + if (newDevices === null) return; + + const newCalls = [...otherCalls as unknown as IGroupCallRoomMemberCallState[]]; + if (newDevices.length > 0) { + newCalls.push({ + ...call, + "m.call_id": this.groupCallId, + "m.devices": newDevices, + }); + } + + const newContent: IGroupCallRoomMemberState = { "m.calls": newCalls }; + + await this.client.sendStateEvent( + this.room.roomId, EventType.GroupCallMemberPrefix, newContent, localUserId, { keepAlive }, + ); } - private removeParticipant(member: RoomMember) { - const index = this.participants.findIndex((m) => m.userId === member.userId); + private async addDeviceToMemberState(): Promise { + await this.updateDevices(devices => [ + ...devices.filter(d => d.device_id !== this.client.getDeviceId()!), + { + "device_id": this.client.getDeviceId()!, + "session_id": this.client.getSessionId(), + "expires_ts": Date.now() + DEVICE_TIMEOUT, + "feeds": this.getLocalFeeds().map(feed => ({ purpose: feed.purpose })), + "m.foci.active": this.foci, + "m.foci.preferred": this.getPreferredFoci(), + // TODO: Add data channels + }, + ]); + } - if (index === -1) { - return; + private async updateMemberState(): Promise { + // Clear the old update interval before proceeding + if (this.resendMemberStateTimer !== null) { + clearInterval(this.resendMemberStateTimer); + this.resendMemberStateTimer = null; } - this.participants.splice(index, 1); + if (this.state === GroupCallState.Entered) { + // Add the local device + await this.addDeviceToMemberState(); - this.emit(GroupCallEvent.ParticipantsChanged, this.participants); - this.client.emit(GroupCallEventHandlerEvent.Participants, this.participants, this); + // Resend the state event every so often so it doesn't become stale + this.resendMemberStateTimer = setInterval(async () => { + logger.log("Resending call member state"); + try { + await this.addDeviceToMemberState(); + } catch (e) { + logger.error("Failed to resend call member state", e); + } + }, DEVICE_TIMEOUT * 3 / 4); + } else { + // Remove the local device + await this.updateDevices( + devices => devices.filter(d => d.device_id !== this.client.getDeviceId()!), + true, + ); + } } + + /** + * Cleans up our member state by filtering out logged out devices, inactive + * devices, and our own device (if we know we haven't entered). + */ + public async cleanMemberState(): Promise { + const { devices: myDevices } = await this.client.getDevices(); + const deviceMap = new Map(myDevices.map(d => [d.device_id, d])); + + // updateDevices takes care of filtering out inactive devices for us + await this.updateDevices(devices => { + const newDevices = devices.filter(d => { + const device = deviceMap.get(d.device_id); + return device?.last_seen_ts !== undefined + && !(d.device_id === this.client.getDeviceId()! && this.state !== GroupCallState.Entered); + }); + + // Skip the update if the devices are unchanged + return newDevices.length === devices.length ? null : newDevices; + }); + } + + private onRoomState = (): void => this.updateParticipants(); + + private onParticipantsChanged = (): void => { + if (this.state === GroupCallState.Entered) this.placeOutgoingCalls(); + }; + + private onStateChanged = (newState: GroupCallState, oldState: GroupCallState): void => { + if ( + newState === GroupCallState.Entered + || oldState === GroupCallState.Entered + || newState === GroupCallState.Ended + ) { + // We either entered, left, or ended the call + this.updateParticipants(); + this.updateMemberState().catch(e => logger.error("Failed to update member state devices", e)); + } + }; + + private onLocalFeedsChanged = (): void => { + if (this.state === GroupCallState.Entered) { + this.updateMemberState().catch(e => logger.error("Failed to update member state feeds", e)); + } + }; } diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts index 4074cab5a9d..592f34edc95 100644 --- a/src/webrtc/groupCallEventHandler.ts +++ b/src/webrtc/groupCallEventHandler.ts @@ -32,12 +32,14 @@ import { SyncState } from '../sync'; export enum GroupCallEventHandlerEvent { Incoming = "GroupCall.incoming", + Outgoing = "GroupCall.outgoing", Ended = "GroupCall.ended", Participants = "GroupCall.participants", } export type GroupCallEventHandlerEventHandlerMap = { [GroupCallEventHandlerEvent.Incoming]: (call: GroupCall) => void; + [GroupCallEventHandlerEvent.Outgoing]: (call: GroupCall) => void; [GroupCallEventHandlerEvent.Ended]: (call: GroupCall) => void; [GroupCallEventHandlerEvent.Participants]: (participants: RoomMember[], call: GroupCall) => void; }; @@ -56,7 +58,7 @@ export class GroupCallEventHandler { // and get private roomDeferreds = new Map(); - constructor(private client: MatrixClient) { } + public constructor(private client: MatrixClient) { } public async start(): Promise { // We wait until the client has started syncing for real. @@ -67,7 +69,7 @@ export class GroupCallEventHandler { if (this.client.getSyncState() !== SyncState.Syncing) { logger.debug("Waiting for client to start syncing..."); await new Promise(resolve => { - const onSync = () => { + const onSync = (): void => { if (this.client.getSyncState() === SyncState.Syncing) { this.client.off(ClientEvent.Sync, onSync); return resolve(); @@ -193,7 +195,7 @@ export class GroupCallEventHandler { return groupCall; } - private onRoomsChanged = (room: Room) => { + private onRoomsChanged = (room: Room): void => { this.createGroupCallForRoom(room); }; @@ -221,14 +223,6 @@ export class GroupCallEventHandler { logger.warn(`Multiple group calls detected for room: ${ state.roomId}. Multiple group calls are currently unsupported.`); } - } else if (eventType === EventType.GroupCallMemberPrefix) { - const groupCall = this.groupCalls.get(state.roomId); - - if (!groupCall) { - return; - } - - groupCall.onMemberStateChanged(event); } }; } diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index c4b912ef38a..c7c84876bf9 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -56,11 +56,11 @@ export class MediaHandler extends TypedEventEmitter< public userMediaStreams: MediaStream[] = []; public screensharingStreams: MediaStream[] = []; - constructor(private client: MatrixClient) { + public constructor(private client: MatrixClient) { super(); } - public restoreMediaSettings(audioInput: string, videoInput: string) { + public restoreMediaSettings(audioInput: string, videoInput: string): void { this.audioInput = audioInput; this.videoInput = videoInput; } @@ -275,7 +275,7 @@ export class MediaHandler extends TypedEventEmitter< /** * Stops all tracks on the provided usermedia stream */ - public stopUserMediaStream(mediaStream: MediaStream) { + public stopUserMediaStream(mediaStream: MediaStream): void { logger.log(`mediaHandler stopUserMediaStream stopping stream ${mediaStream.id}`); for (const track of mediaStream.getTracks()) { track.stop(); @@ -333,7 +333,7 @@ export class MediaHandler extends TypedEventEmitter< /** * Stops all tracks on the provided screensharing stream */ - public stopScreensharingStream(mediaStream: MediaStream) { + public stopScreensharingStream(mediaStream: MediaStream): void { logger.debug("Stopping screensharing stream", mediaStream.id); for (const track of mediaStream.getTracks()) { track.stop(); @@ -352,7 +352,7 @@ export class MediaHandler extends TypedEventEmitter< /** * Stops all local media tracks */ - public stopAllStreams() { + public stopAllStreams(): void { for (const stream of this.userMediaStreams) { logger.log(`mediaHandler stopAllStreams stopping stream ${stream.id}`); for (const track of stream.getTracks()) { diff --git a/yarn.lock b/yarn.lock index 5df6f84a311..77cca92702e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -58,26 +58,31 @@ dependencies: "@babel/highlight" "^7.18.6" -"@babel/compat-data@^7.17.7", "@babel/compat-data@^7.20.0", "@babel/compat-data@^7.20.1": +"@babel/compat-data@^7.17.7", "@babel/compat-data@^7.20.1": version "7.20.1" resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.20.1.tgz#f2e6ef7790d8c8dbf03d379502dcc246dcce0b30" integrity sha512-EWZ4mE2diW3QALKvDMiXnbZpRvlj+nayZ112nK93SnhqOtpdsbVD4W+2tEoT3YNBAG9RBR0ISY758ZkOgsn6pQ== +"@babel/compat-data@^7.20.0": + version "7.20.5" + resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.20.5.tgz#86f172690b093373a933223b4745deeb6049e733" + integrity sha512-KZXo2t10+/jxmkhNXc7pZTqRvSOIvVv/+lJwHS+B2rErwOyjuVRh60yVpb7liQ1U5t7lLJ1bz+t8tSypUZdm0g== + "@babel/core@^7.11.6", "@babel/core@^7.12.10", "@babel/core@^7.12.3", "@babel/core@^7.7.5": - version "7.20.2" - resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.20.2.tgz#8dc9b1620a673f92d3624bd926dc49a52cf25b92" - integrity sha512-w7DbG8DtMrJcFOi4VrLm+8QM4az8Mo+PuLBKLp2zrYRCow8W/f9xiXm5sN53C8HksCyDQwCKha9JiDoIyPjT2g== + version "7.20.5" + resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.20.5.tgz#45e2114dc6cd4ab167f81daf7820e8fa1250d113" + integrity sha512-UdOWmk4pNWTm/4DlPUl/Pt4Gz4rcEMb7CY0Y3eJl5Yz1vI8ZJGmHWaVE55LoxRjdpx0z259GE9U5STA9atUinQ== dependencies: "@ampproject/remapping" "^2.1.0" "@babel/code-frame" "^7.18.6" - "@babel/generator" "^7.20.2" + "@babel/generator" "^7.20.5" "@babel/helper-compilation-targets" "^7.20.0" "@babel/helper-module-transforms" "^7.20.2" - "@babel/helpers" "^7.20.1" - "@babel/parser" "^7.20.2" + "@babel/helpers" "^7.20.5" + "@babel/parser" "^7.20.5" "@babel/template" "^7.18.10" - "@babel/traverse" "^7.20.1" - "@babel/types" "^7.20.2" + "@babel/traverse" "^7.20.5" + "@babel/types" "^7.20.5" convert-source-map "^1.7.0" debug "^4.1.0" gensync "^1.0.0-beta.2" @@ -100,7 +105,7 @@ dependencies: eslint-rule-composer "^0.3.0" -"@babel/generator@^7.12.11", "@babel/generator@^7.20.1", "@babel/generator@^7.20.2", "@babel/generator@^7.7.2": +"@babel/generator@^7.12.11": version "7.20.3" resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.20.3.tgz#e58c9ae2f7bf7fdf4899160cf1e04400a82cd641" integrity sha512-Wl5ilw2UD1+ZYprHVprxHZJCFeBWlzZYOovE4SDYLZnqCOD11j+0QzNeEWKLLTWM7nixrZEh7vNIyb76MyJg3A== @@ -109,6 +114,24 @@ "@jridgewell/gen-mapping" "^0.3.2" jsesc "^2.5.1" +"@babel/generator@^7.20.1", "@babel/generator@^7.20.5": + version "7.20.5" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.20.5.tgz#cb25abee3178adf58d6814b68517c62bdbfdda95" + integrity sha512-jl7JY2Ykn9S0yj4DQP82sYvPU+T3g0HFcWTqDLqiuA9tGRNIj9VfbtXGAYTTkyNEnQk1jkMGOdYka8aG/lulCA== + dependencies: + "@babel/types" "^7.20.5" + "@jridgewell/gen-mapping" "^0.3.2" + jsesc "^2.5.1" + +"@babel/generator@^7.7.2": + version "7.20.4" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.20.4.tgz#4d9f8f0c30be75fd90a0562099a26e5839602ab8" + integrity sha512-luCf7yk/cm7yab6CAW1aiFnmEfBJplb/JojV56MYEK7ziWfGmFlTfmL9Ehwfy4gFhbjBfWO1wj7/TuSbVNEEtA== + dependencies: + "@babel/types" "^7.20.2" + "@jridgewell/gen-mapping" "^0.3.2" + jsesc "^2.5.1" + "@babel/helper-annotate-as-pure@^7.18.6": version "7.18.6" resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.18.6.tgz#eaa49f6f80d5a33f9a5dd2276e6d6e451be0a6bb" @@ -301,14 +324,14 @@ "@babel/traverse" "^7.19.0" "@babel/types" "^7.19.0" -"@babel/helpers@^7.20.1": - version "7.20.1" - resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.20.1.tgz#2ab7a0fcb0a03b5bf76629196ed63c2d7311f4c9" - integrity sha512-J77mUVaDTUJFZ5BpP6mMn6OIl3rEWymk2ZxDBQJUG3P+PbmyMcF3bYWvz0ma69Af1oobDqT/iAsvzhB58xhQUg== +"@babel/helpers@^7.20.5": + version "7.20.6" + resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.20.6.tgz#e64778046b70e04779dfbdf924e7ebb45992c763" + integrity sha512-Pf/OjgfgFRW5bApskEz5pvidpim7tEDPlFtKcNRXWmfHGn9IEI2W2flqRQXTFb7gIPTyK++N6rVHuwKut4XK6w== dependencies: "@babel/template" "^7.18.10" - "@babel/traverse" "^7.20.1" - "@babel/types" "^7.20.0" + "@babel/traverse" "^7.20.5" + "@babel/types" "^7.20.5" "@babel/highlight@^7.18.6": version "7.18.6" @@ -319,11 +342,16 @@ chalk "^2.0.0" js-tokens "^4.0.0" -"@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.18.10", "@babel/parser@^7.2.3", "@babel/parser@^7.20.1", "@babel/parser@^7.20.2": +"@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.2.3": version "7.20.3" resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.20.3.tgz#5358cf62e380cf69efcb87a7bb922ff88bfac6e2" integrity sha512-OP/s5a94frIPXwjzEcv5S/tpQfc6XhxYUnmWpgdqMWGgYCuErA3SzozaRAMQgSZWKeTJxht9aWAkUY+0UzvOFg== +"@babel/parser@^7.18.10", "@babel/parser@^7.20.1", "@babel/parser@^7.20.5": + version "7.20.5" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.20.5.tgz#7f3c7335fe417665d929f34ae5dceae4c04015e8" + integrity sha512-r27t/cy/m9uKLXQNWWebeCUHgnAZq0CpG1OwKRxzJMP1vpSU4bSIK2hq+/cp0bQxetkXx38n09rNu8jVkcK/zA== + "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@^7.18.6": version "7.18.6" resolved "https://registry.yarnpkg.com/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/-/plugin-bugfix-safari-id-destructuring-collision-in-function-expression-7.18.6.tgz#da5b8f9a580acdfbe53494dba45ea389fb09a4d2" @@ -986,11 +1014,11 @@ source-map-support "^0.5.16" "@babel/runtime@^7.12.5", "@babel/runtime@^7.7.6", "@babel/runtime@^7.8.4": - version "7.20.1" - resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.20.1.tgz#1148bb33ab252b165a06698fde7576092a78b4a9" - integrity sha512-mrzLkl6U9YLF8qpqI7TB82PESyEGjm/0Ly91jG575eVxMMlb8fYfOXFZIJ8XfLrJZQbm7dlKry2bJmXBUEkdFg== + version "7.20.6" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.20.6.tgz#facf4879bfed9b5326326273a64220f099b0fce3" + integrity sha512-Q+8MqP7TiHMWzSfwiJwXCjyf4GYA4Dgw3emg/7xmwsdLJOZUp+nMqcOwOzzYheuM1rhDu8FSj2l0aoMygEuXuA== dependencies: - regenerator-runtime "^0.13.10" + regenerator-runtime "^0.13.11" "@babel/template@^7.18.10", "@babel/template@^7.3.3": version "7.18.10" @@ -1001,7 +1029,7 @@ "@babel/parser" "^7.18.10" "@babel/types" "^7.18.10" -"@babel/traverse@^7.1.6", "@babel/traverse@^7.19.0", "@babel/traverse@^7.19.1", "@babel/traverse@^7.20.1", "@babel/traverse@^7.7.2": +"@babel/traverse@^7.1.6", "@babel/traverse@^7.19.0", "@babel/traverse@^7.19.1", "@babel/traverse@^7.7.2": version "7.20.1" resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.20.1.tgz#9b15ccbf882f6d107eeeecf263fbcdd208777ec8" integrity sha512-d3tN8fkVJwFLkHkBN479SOsw4DMZnz8cdbL/gvuDuzy3TS6Nfw80HuQqhw1pITbIruHyh7d1fMA47kWzmcUEGA== @@ -1017,7 +1045,23 @@ debug "^4.1.0" globals "^11.1.0" -"@babel/types@^7.0.0", "@babel/types@^7.18.10", "@babel/types@^7.18.6", "@babel/types@^7.18.9", "@babel/types@^7.19.0", "@babel/types@^7.2.0", "@babel/types@^7.20.0", "@babel/types@^7.20.2", "@babel/types@^7.3.0", "@babel/types@^7.3.3", "@babel/types@^7.4.4": +"@babel/traverse@^7.20.1", "@babel/traverse@^7.20.5": + version "7.20.5" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.20.5.tgz#78eb244bea8270fdda1ef9af22a5d5e5b7e57133" + integrity sha512-WM5ZNN3JITQIq9tFZaw1ojLU3WgWdtkxnhM1AegMS+PvHjkM5IXjmYEGY7yukz5XS4sJyEf2VzWjI8uAavhxBQ== + dependencies: + "@babel/code-frame" "^7.18.6" + "@babel/generator" "^7.20.5" + "@babel/helper-environment-visitor" "^7.18.9" + "@babel/helper-function-name" "^7.19.0" + "@babel/helper-hoist-variables" "^7.18.6" + "@babel/helper-split-export-declaration" "^7.18.6" + "@babel/parser" "^7.20.5" + "@babel/types" "^7.20.5" + debug "^4.1.0" + globals "^11.1.0" + +"@babel/types@^7.0.0", "@babel/types@^7.18.9", "@babel/types@^7.2.0", "@babel/types@^7.3.0", "@babel/types@^7.3.3", "@babel/types@^7.4.4": version "7.20.2" resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.20.2.tgz#67ac09266606190f496322dbaff360fdaa5e7842" integrity sha512-FnnvsNWgZCr232sqtXggapvlkk/tuwR/qhGzcmxI0GXLCjmPYQPzio2FbdlWuY6y1sHFfQKk+rRbUZ9VStQMog== @@ -1026,6 +1070,15 @@ "@babel/helper-validator-identifier" "^7.19.1" to-fast-properties "^2.0.0" +"@babel/types@^7.18.10", "@babel/types@^7.18.6", "@babel/types@^7.19.0", "@babel/types@^7.20.0", "@babel/types@^7.20.2", "@babel/types@^7.20.5": + version "7.20.5" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.20.5.tgz#e206ae370b5393d94dfd1d04cd687cace53efa84" + integrity sha512-c9fst/h2/dcF7H+MJKZ2T0KjEQ8hY/BNnDk/H3XY8C4Aw/eWQXWn/lWntHF9ooUBnGmEvbfGrTgLWc+um0YDUg== + dependencies: + "@babel/helper-string-parser" "^7.19.4" + "@babel/helper-validator-identifier" "^7.19.1" + to-fast-properties "^2.0.0" + "@bcoe/v8-coverage@^0.2.3": version "0.2.3" resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" @@ -1040,6 +1093,13 @@ uuid "8.3.2" xml "1.0.1" +"@eslint-community/eslint-utils@^4.1.0": + version "4.1.2" + resolved "https://registry.yarnpkg.com/@eslint-community/eslint-utils/-/eslint-utils-4.1.2.tgz#14ca568ddaa291dd19a4a54498badc18c6cfab78" + integrity sha512-7qELuQWWjVDdVsFQ5+beUl+KPczrEDA7S3zM4QUd/bJl7oXgsmpXaEVqrRTnOBqenOV4rWf2kVZk2Ot085zPWA== + dependencies: + eslint-visitor-keys "^3.3.0" + "@eslint/eslintrc@^1.3.3": version "1.3.3" resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-1.3.3.tgz#2b044ab39fdfa75b4688184f9e573ce3c5b0ff95" @@ -1090,28 +1150,28 @@ resolved "https://registry.yarnpkg.com/@istanbuljs/schema/-/schema-0.1.3.tgz#e45e384e4b8ec16bce2fd903af78450f6bf7ec98" integrity sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA== -"@jest/console@^29.2.1": - version "29.2.1" - resolved "https://registry.yarnpkg.com/@jest/console/-/console-29.2.1.tgz#5f2c62dcdd5ce66e94b6d6729e021758bceea090" - integrity sha512-MF8Adcw+WPLZGBiNxn76DOuczG3BhODTcMlDCA4+cFi41OkaY/lyI0XUUhi73F88Y+7IHoGmD80pN5CtxQUdSw== +"@jest/console@^29.3.1": + version "29.3.1" + resolved "https://registry.yarnpkg.com/@jest/console/-/console-29.3.1.tgz#3e3f876e4e47616ea3b1464b9fbda981872e9583" + integrity sha512-IRE6GD47KwcqA09RIWrabKdHPiKDGgtAL31xDxbi/RjQMsr+lY+ppxmHwY0dUEV3qvvxZzoe5Hl0RXZJOjQNUg== dependencies: - "@jest/types" "^29.2.1" + "@jest/types" "^29.3.1" "@types/node" "*" chalk "^4.0.0" - jest-message-util "^29.2.1" - jest-util "^29.2.1" + jest-message-util "^29.3.1" + jest-util "^29.3.1" slash "^3.0.0" -"@jest/core@^29.3.0": - version "29.3.0" - resolved "https://registry.yarnpkg.com/@jest/core/-/core-29.3.0.tgz#7042d3fd673b51d89d6f6bf8b9f85fb65573629e" - integrity sha512-5DyNvV8452bwqcYyXHCYaAD8UrTiWosrhBY+rc0MBMyXyDzcIL+w5gdlCYhlHbNsHoWnf4nUbRmg++LWfWVtMQ== +"@jest/core@^29.3.1": + version "29.3.1" + resolved "https://registry.yarnpkg.com/@jest/core/-/core-29.3.1.tgz#bff00f413ff0128f4debec1099ba7dcd649774a1" + integrity sha512-0ohVjjRex985w5MmO5L3u5GR1O30DexhBSpuwx2P+9ftyqHdJXnk7IUWiP80oHMvt7ubHCJHxV0a0vlKVuZirw== dependencies: - "@jest/console" "^29.2.1" - "@jest/reporters" "^29.3.0" - "@jest/test-result" "^29.2.1" - "@jest/transform" "^29.3.0" - "@jest/types" "^29.2.1" + "@jest/console" "^29.3.1" + "@jest/reporters" "^29.3.1" + "@jest/test-result" "^29.3.1" + "@jest/transform" "^29.3.1" + "@jest/types" "^29.3.1" "@types/node" "*" ansi-escapes "^4.2.1" chalk "^4.0.0" @@ -1119,42 +1179,32 @@ exit "^0.1.2" graceful-fs "^4.2.9" jest-changed-files "^29.2.0" - jest-config "^29.3.0" - jest-haste-map "^29.3.0" - jest-message-util "^29.2.1" + jest-config "^29.3.1" + jest-haste-map "^29.3.1" + jest-message-util "^29.3.1" jest-regex-util "^29.2.0" - jest-resolve "^29.3.0" - jest-resolve-dependencies "^29.3.0" - jest-runner "^29.3.0" - jest-runtime "^29.3.0" - jest-snapshot "^29.3.0" - jest-util "^29.2.1" - jest-validate "^29.2.2" - jest-watcher "^29.2.2" + jest-resolve "^29.3.1" + jest-resolve-dependencies "^29.3.1" + jest-runner "^29.3.1" + jest-runtime "^29.3.1" + jest-snapshot "^29.3.1" + jest-util "^29.3.1" + jest-validate "^29.3.1" + jest-watcher "^29.3.1" micromatch "^4.0.4" - pretty-format "^29.2.1" + pretty-format "^29.3.1" slash "^3.0.0" strip-ansi "^6.0.0" -"@jest/environment@^28.1.3": - version "28.1.3" - resolved "https://registry.yarnpkg.com/@jest/environment/-/environment-28.1.3.tgz#abed43a6b040a4c24fdcb69eab1f97589b2d663e" - integrity sha512-1bf40cMFTEkKyEf585R9Iz1WayDjHoHqvts0XFYEqyKM3cFWDpeMoqKKTAF9LSYQModPUlh8FKptoM2YcMWAXA== +"@jest/environment@^29.3.1": + version "29.3.1" + resolved "https://registry.yarnpkg.com/@jest/environment/-/environment-29.3.1.tgz#eb039f726d5fcd14698acd072ac6576d41cfcaa6" + integrity sha512-pMmvfOPmoa1c1QpfFW0nXYtNLpofqo4BrCIk6f2kW4JFeNlHV2t3vd+3iDLf31e2ot2Mec0uqZfmI+U0K2CFag== dependencies: - "@jest/fake-timers" "^28.1.3" - "@jest/types" "^28.1.3" + "@jest/fake-timers" "^29.3.1" + "@jest/types" "^29.3.1" "@types/node" "*" - jest-mock "^28.1.3" - -"@jest/environment@^29.3.0": - version "29.3.0" - resolved "https://registry.yarnpkg.com/@jest/environment/-/environment-29.3.0.tgz#e7bfe22531813f86040feb75c046faab32fd534d" - integrity sha512-8wgn3br51bx+7rgC8FOKmAD62Q39iswdiy5/p6acoekp/9Bb/IQbh3zydOrnGp74LwStSrKgpQSKBlOKlAQq0g== - dependencies: - "@jest/fake-timers" "^29.3.0" - "@jest/types" "^29.2.1" - "@types/node" "*" - jest-mock "^29.3.0" + jest-mock "^29.3.1" "@jest/expect-utils@^28.1.3": version "28.1.3" @@ -1163,65 +1213,53 @@ dependencies: jest-get-type "^28.0.2" -"@jest/expect-utils@^29.2.2": - version "29.2.2" - resolved "https://registry.yarnpkg.com/@jest/expect-utils/-/expect-utils-29.2.2.tgz#460a5b5a3caf84d4feb2668677393dd66ff98665" - integrity sha512-vwnVmrVhTmGgQzyvcpze08br91OL61t9O0lJMDyb6Y/D8EKQ9V7rGUb/p7PDt0GPzK0zFYqXWFo4EO2legXmkg== +"@jest/expect-utils@^29.3.1": + version "29.3.1" + resolved "https://registry.yarnpkg.com/@jest/expect-utils/-/expect-utils-29.3.1.tgz#531f737039e9b9e27c42449798acb5bba01935b6" + integrity sha512-wlrznINZI5sMjwvUoLVk617ll/UYfGIZNxmbU+Pa7wmkL4vYzhV9R2pwVqUh4NWWuLQWkI8+8mOkxs//prKQ3g== dependencies: jest-get-type "^29.2.0" -"@jest/expect@^29.3.0": - version "29.3.0" - resolved "https://registry.yarnpkg.com/@jest/expect/-/expect-29.3.0.tgz#06907ffc02541c8d5186e8324765a72f391f3125" - integrity sha512-Lz/3x4Se5g6nBuLjTO+xE8D4OXY9fFmosZPwkXXZUJUsp9r9seN81cJa54wOGr1QjCQnhngMqclblhM4X/hcCg== - dependencies: - expect "^29.3.0" - jest-snapshot "^29.3.0" - -"@jest/fake-timers@^28.1.3": - version "28.1.3" - resolved "https://registry.yarnpkg.com/@jest/fake-timers/-/fake-timers-28.1.3.tgz#230255b3ad0a3d4978f1d06f70685baea91c640e" - integrity sha512-D/wOkL2POHv52h+ok5Oj/1gOG9HSywdoPtFsRCUmlCILXNn5eIWmcnd3DIiWlJnpGvQtmajqBP95Ei0EimxfLw== +"@jest/expect@^29.3.1": + version "29.3.1" + resolved "https://registry.yarnpkg.com/@jest/expect/-/expect-29.3.1.tgz#456385b62894349c1d196f2d183e3716d4c6a6cd" + integrity sha512-QivM7GlSHSsIAWzgfyP8dgeExPRZ9BIe2LsdPyEhCGkZkoyA+kGsoIzbKAfZCvvRzfZioKwPtCZIt5SaoxYCvg== dependencies: - "@jest/types" "^28.1.3" - "@sinonjs/fake-timers" "^9.1.2" - "@types/node" "*" - jest-message-util "^28.1.3" - jest-mock "^28.1.3" - jest-util "^28.1.3" + expect "^29.3.1" + jest-snapshot "^29.3.1" -"@jest/fake-timers@^29.3.0": - version "29.3.0" - resolved "https://registry.yarnpkg.com/@jest/fake-timers/-/fake-timers-29.3.0.tgz#ffa74e5b2937676849866cac79cac6a742697f00" - integrity sha512-SzmWtN6Rld+xebMRGuWeMGhytc7qHnYfFk1Zd/1QavQWsFOmA9SgtvGHCBue1wXQhdDMaSIm1aPGj2Zmyrr1Zg== +"@jest/fake-timers@^29.3.1": + version "29.3.1" + resolved "https://registry.yarnpkg.com/@jest/fake-timers/-/fake-timers-29.3.1.tgz#b140625095b60a44de820876d4c14da1aa963f67" + integrity sha512-iHTL/XpnDlFki9Tq0Q1GGuVeQ8BHZGIYsvCO5eN/O/oJaRzofG9Xndd9HuSDBI/0ZS79pg0iwn07OMTQ7ngF2A== dependencies: - "@jest/types" "^29.2.1" + "@jest/types" "^29.3.1" "@sinonjs/fake-timers" "^9.1.2" "@types/node" "*" - jest-message-util "^29.2.1" - jest-mock "^29.3.0" - jest-util "^29.2.1" + jest-message-util "^29.3.1" + jest-mock "^29.3.1" + jest-util "^29.3.1" -"@jest/globals@^29.3.0": - version "29.3.0" - resolved "https://registry.yarnpkg.com/@jest/globals/-/globals-29.3.0.tgz#f58c14d727fd7d02d7851bc03fc0445eefa2dbe2" - integrity sha512-okYDVzYNrt/4ysR8XnX6u0I1bGG4kmfdXtUu7kwWHZ9OP13RCjmphgve0tfOrNluwksWvOPYS1f/HOrFTHLygQ== +"@jest/globals@^29.3.1": + version "29.3.1" + resolved "https://registry.yarnpkg.com/@jest/globals/-/globals-29.3.1.tgz#92be078228e82d629df40c3656d45328f134a0c6" + integrity sha512-cTicd134vOcwO59OPaB6AmdHQMCtWOe+/DitpTZVxWgMJ+YvXL1HNAmPyiGbSHmF/mXVBkvlm8YYtQhyHPnV6Q== dependencies: - "@jest/environment" "^29.3.0" - "@jest/expect" "^29.3.0" - "@jest/types" "^29.2.1" - jest-mock "^29.3.0" + "@jest/environment" "^29.3.1" + "@jest/expect" "^29.3.1" + "@jest/types" "^29.3.1" + jest-mock "^29.3.1" -"@jest/reporters@^29.3.0": - version "29.3.0" - resolved "https://registry.yarnpkg.com/@jest/reporters/-/reporters-29.3.0.tgz#e5e2af97d7754510393d7c04084744841cce2eaf" - integrity sha512-MV76tB3Kd80vcv2yMDZfQpMkwkHaY9hlvVhCtHXkVRCWwN+SX3EOmCdX8pT/X4Xh+NusA7l2Rc3yhx4q5p3+Fg== +"@jest/reporters@^29.3.1": + version "29.3.1" + resolved "https://registry.yarnpkg.com/@jest/reporters/-/reporters-29.3.1.tgz#9a6d78c109608e677c25ddb34f907b90e07b4310" + integrity sha512-GhBu3YFuDrcAYW/UESz1JphEAbvUjaY2vShRZRoRY1mxpCMB3yGSJ4j9n0GxVlEOdCf7qjvUfBCrTUUqhVfbRA== dependencies: "@bcoe/v8-coverage" "^0.2.3" - "@jest/console" "^29.2.1" - "@jest/test-result" "^29.2.1" - "@jest/transform" "^29.3.0" - "@jest/types" "^29.2.1" + "@jest/console" "^29.3.1" + "@jest/test-result" "^29.3.1" + "@jest/transform" "^29.3.1" + "@jest/types" "^29.3.1" "@jridgewell/trace-mapping" "^0.3.15" "@types/node" "*" chalk "^4.0.0" @@ -1234,9 +1272,9 @@ istanbul-lib-report "^3.0.0" istanbul-lib-source-maps "^4.0.0" istanbul-reports "^3.1.3" - jest-message-util "^29.2.1" - jest-util "^29.2.1" - jest-worker "^29.3.0" + jest-message-util "^29.3.1" + jest-util "^29.3.1" + jest-worker "^29.3.1" slash "^3.0.0" string-length "^4.0.1" strip-ansi "^6.0.0" @@ -1265,42 +1303,42 @@ callsites "^3.0.0" graceful-fs "^4.2.9" -"@jest/test-result@^29.2.1": - version "29.2.1" - resolved "https://registry.yarnpkg.com/@jest/test-result/-/test-result-29.2.1.tgz#f42dbf7b9ae465d0a93eee6131473b8bb3bd2edb" - integrity sha512-lS4+H+VkhbX6z64tZP7PAUwPqhwj3kbuEHcaLuaBuB+riyaX7oa1txe0tXgrFj5hRWvZKvqO7LZDlNWeJ7VTPA== +"@jest/test-result@^29.3.1": + version "29.3.1" + resolved "https://registry.yarnpkg.com/@jest/test-result/-/test-result-29.3.1.tgz#92cd5099aa94be947560a24610aa76606de78f50" + integrity sha512-qeLa6qc0ddB0kuOZyZIhfN5q0e2htngokyTWsGriedsDhItisW7SDYZ7ceOe57Ii03sL988/03wAcBh3TChMGw== dependencies: - "@jest/console" "^29.2.1" - "@jest/types" "^29.2.1" + "@jest/console" "^29.3.1" + "@jest/types" "^29.3.1" "@types/istanbul-lib-coverage" "^2.0.0" collect-v8-coverage "^1.0.0" -"@jest/test-sequencer@^29.3.0": - version "29.3.0" - resolved "https://registry.yarnpkg.com/@jest/test-sequencer/-/test-sequencer-29.3.0.tgz#0c2198fe482c26d763abbcb183992ae769bb7978" - integrity sha512-XQlTP/S6Yf6NKV0Mt4oopFKyDxiEkDMD7hIFcCTeltKQszE0Z+LI5KLukwNW6Qxr1YzaZ/s6PlKJusiCLJNTcw== +"@jest/test-sequencer@^29.3.1": + version "29.3.1" + resolved "https://registry.yarnpkg.com/@jest/test-sequencer/-/test-sequencer-29.3.1.tgz#fa24b3b050f7a59d48f7ef9e0b782ab65123090d" + integrity sha512-IqYvLbieTv20ArgKoAMyhLHNrVHJfzO6ARZAbQRlY4UGWfdDnLlZEF0BvKOMd77uIiIjSZRwq3Jb3Fa3I8+2UA== dependencies: - "@jest/test-result" "^29.2.1" + "@jest/test-result" "^29.3.1" graceful-fs "^4.2.9" - jest-haste-map "^29.3.0" + jest-haste-map "^29.3.1" slash "^3.0.0" -"@jest/transform@^29.3.0": - version "29.3.0" - resolved "https://registry.yarnpkg.com/@jest/transform/-/transform-29.3.0.tgz#7f71c9596d5bad1613a3a5eb26729dd84fc71a5a" - integrity sha512-4T8h61ItCakAlJkdYa7XVWP3r39QldlCeOSNmRpiJisi5PrrlzwZdpJDIH13ZZjh+MlSPQ2cq8YbUs3TuH+tRA== +"@jest/transform@^29.3.1": + version "29.3.1" + resolved "https://registry.yarnpkg.com/@jest/transform/-/transform-29.3.1.tgz#1e6bd3da4af50b5c82a539b7b1f3770568d6e36d" + integrity sha512-8wmCFBTVGYqFNLWfcOWoVuMuKYPUBTnTMDkdvFtAYELwDOl9RGwOsvQWGPFxDJ8AWY9xM/8xCXdqmPK3+Q5Lug== dependencies: "@babel/core" "^7.11.6" - "@jest/types" "^29.2.1" + "@jest/types" "^29.3.1" "@jridgewell/trace-mapping" "^0.3.15" babel-plugin-istanbul "^6.1.1" chalk "^4.0.0" convert-source-map "^2.0.0" fast-json-stable-stringify "^2.1.0" graceful-fs "^4.2.9" - jest-haste-map "^29.3.0" + jest-haste-map "^29.3.1" jest-regex-util "^29.2.0" - jest-util "^29.2.1" + jest-util "^29.3.1" micromatch "^4.0.4" pirates "^4.0.4" slash "^3.0.0" @@ -1318,10 +1356,10 @@ "@types/yargs" "^17.0.8" chalk "^4.0.0" -"@jest/types@^29.2.1": - version "29.2.1" - resolved "https://registry.yarnpkg.com/@jest/types/-/types-29.2.1.tgz#ec9c683094d4eb754e41e2119d8bdaef01cf6da0" - integrity sha512-O/QNDQODLnINEPAI0cl9U6zUIDXEWXt6IC1o2N2QENuos7hlGUIthlKyV4p6ki3TvXFX071blj8HUhgLGquPjw== +"@jest/types@^29.3.1": + version "29.3.1" + resolved "https://registry.yarnpkg.com/@jest/types/-/types-29.3.1.tgz#7c5a80777cb13e703aeec6788d044150341147e3" + integrity sha512-d0S0jmmTpjnhCmNpApgX3jrUZgZ22ivKJRvL2lli5hpCRoNnp1f85r2/wpKfXuYu8E7Jjh1hGfhPyup1NM5AmA== dependencies: "@jest/schemas" "^29.0.0" "@types/istanbul-lib-coverage" "^2.0.0" @@ -1378,6 +1416,13 @@ "@jridgewell/resolve-uri" "3.1.0" "@jridgewell/sourcemap-codec" "1.4.14" +"@jsdoc/salty@^0.2.1": + version "0.2.1" + resolved "https://registry.yarnpkg.com/@jsdoc/salty/-/salty-0.2.1.tgz#815c487c859eca81ad3dfea540f830e1ff9d3392" + integrity sha512-JXwylDNSHa549N9uceDYu8D4GMXwSo3H8CCPYEQqxhhHpxD28+lRl2b3bS/caaPj5w1YD3SWtrficJNTnUjGpg== + dependencies: + lodash "^4.17.21" + "@matrix-org/olm@https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.13.tgz": version "3.2.13" resolved "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.13.tgz#0109fde93bcc61def851f79826c9384c073b5175" @@ -1648,21 +1693,21 @@ "@types/istanbul-lib-report" "*" "@types/jest@^29.0.0": - version "29.2.2" - resolved "https://registry.yarnpkg.com/@types/jest/-/jest-29.2.2.tgz#874e7dc6702fa6a3fe6107792aa98636dcc480b4" - integrity sha512-og1wAmdxKoS71K2ZwSVqWPX6OVn3ihZ6ZT2qvZvZQm90lJVDyXIjYcu4Khx2CNIeaFv12rOU/YObOsI3VOkzog== + version "29.2.3" + resolved "https://registry.yarnpkg.com/@types/jest/-/jest-29.2.3.tgz#f5fd88e43e5a9e4221ca361e23790d48fcf0a211" + integrity sha512-6XwoEbmatfyoCjWRX7z0fKMmgYKe9+/HrviJ5k0X/tjJWHGAezZOfYaxqQKuzG/TvQyr+ktjm4jgbk0s4/oF2w== dependencies: expect "^29.0.0" pretty-format "^29.0.0" -"@types/jsdom@^16.2.4": - version "16.2.15" - resolved "https://registry.yarnpkg.com/@types/jsdom/-/jsdom-16.2.15.tgz#6c09990ec43b054e49636cba4d11d54367fc90d6" - integrity sha512-nwF87yjBKuX/roqGYerZZM0Nv1pZDMAT5YhOHYeM/72Fic+VEqJh4nyoqoapzJnW3pUlfxPY5FhgsJtM+dRnQQ== +"@types/jsdom@^20.0.0": + version "20.0.1" + resolved "https://registry.yarnpkg.com/@types/jsdom/-/jsdom-20.0.1.tgz#07c14bc19bd2f918c1929541cdaacae894744808" + integrity sha512-d0r18sZPmMQr1eG35u12FZfhIXNrnsPU/g5wvRKCUf/tOGilKKwYMYGqh33BNR6ba+2gkHw1EUiHoN3mn7E5IQ== dependencies: "@types/node" "*" - "@types/parse5" "^6.0.3" "@types/tough-cookie" "*" + parse5 "^7.0.0" "@types/json-schema@^7.0.9": version "7.0.11" @@ -1674,26 +1719,16 @@ resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" integrity sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ== -"@types/node@*": +"@types/node@*", "@types/node@18": version "18.11.9" resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.9.tgz#02d013de7058cea16d36168ef2fc653464cfbad4" integrity sha512-CRpX21/kGdzjOpFsZSkcrXMGIBWMGNIHXXBVFSH+ggkftxg+XYP20TESbh+zFvFj3EQOl5byk0HTRn1IL6hbqg== -"@types/node@16": - version "16.18.3" - resolved "https://registry.yarnpkg.com/@types/node/-/node-16.18.3.tgz#d7f7ba828ad9e540270f01ce00d391c54e6e0abc" - integrity sha512-jh6m0QUhIRcZpNv7Z/rpN+ZWXOicUUQbSoWks7Htkbb9IjFQj4kzcX/xFCkjstCj5flMsN8FiSvt+q+Tcs4Llg== - "@types/normalize-package-data@^2.4.0": version "2.4.1" resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz#d3357479a0fdfdd5907fe67e17e0a85c906e1301" integrity sha512-Gj7cI7z+98M282Tqmp2K5EIsoouUEzbBJhQQzDE3jSIRk6r9gsz0oUokqIUR4u1R3dMHo0pDHM7sNOHyhulypw== -"@types/parse5@^6.0.3": - version "6.0.3" - resolved "https://registry.yarnpkg.com/@types/parse5/-/parse5-6.0.3.tgz#705bb349e789efa06f43f128cef51240753424cb" - integrity sha512-SuT16Q1K51EAVPz1K29DJ/sXjhSQ0zjvsypYJ6tlwVsRV9jwW5Adq2ch8Dq8kDBCkYnELS7N7VNCSB5nC56t/g== - "@types/prettier@^2.1.5": version "2.7.1" resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-2.7.1.tgz#dfd20e2dc35f027cdd6c1908e80a5ddc7499670e" @@ -1742,13 +1777,13 @@ "@types/yargs-parser" "*" "@typescript-eslint/eslint-plugin@^5.6.0": - version "5.42.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.42.1.tgz#696b9cc21dfd4749c1c8ad1307f76a36a00aa0e3" - integrity sha512-LyR6x784JCiJ1j6sH5Y0K6cdExqCCm8DJUTcwG5ThNXJj/G8o5E56u5EdG4SLy+bZAwZBswC+GYn3eGdttBVCg== + version "5.45.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.45.0.tgz#ffa505cf961d4844d38cfa19dcec4973a6039e41" + integrity sha512-CXXHNlf0oL+Yg021cxgOdMHNTXD17rHkq7iW6RFHoybdFgQBjU3yIXhhcPpGwr1CjZlo6ET8C6tzX5juQoXeGA== dependencies: - "@typescript-eslint/scope-manager" "5.42.1" - "@typescript-eslint/type-utils" "5.42.1" - "@typescript-eslint/utils" "5.42.1" + "@typescript-eslint/scope-manager" "5.45.0" + "@typescript-eslint/type-utils" "5.45.0" + "@typescript-eslint/utils" "5.45.0" debug "^4.3.4" ignore "^5.2.0" natural-compare-lite "^1.4.0" @@ -1757,71 +1792,71 @@ tsutils "^3.21.0" "@typescript-eslint/parser@^5.6.0": - version "5.42.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-5.42.1.tgz#3e66156f2f74b11690b45950d8f5f28a62751d35" - integrity sha512-kAV+NiNBWVQDY9gDJDToTE/NO8BHi4f6b7zTsVAJoTkmB/zlfOpiEVBzHOKtlgTndCKe8vj9F/PuolemZSh50Q== + version "5.45.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-5.45.0.tgz#b18a5f6b3cf1c2b3e399e9d2df4be40d6b0ddd0e" + integrity sha512-brvs/WSM4fKUmF5Ot/gEve6qYiCMjm6w4HkHPfS6ZNmxTS0m0iNN4yOChImaCkqc1hRwFGqUyanMXuGal6oyyQ== dependencies: - "@typescript-eslint/scope-manager" "5.42.1" - "@typescript-eslint/types" "5.42.1" - "@typescript-eslint/typescript-estree" "5.42.1" + "@typescript-eslint/scope-manager" "5.45.0" + "@typescript-eslint/types" "5.45.0" + "@typescript-eslint/typescript-estree" "5.45.0" debug "^4.3.4" -"@typescript-eslint/scope-manager@5.42.1": - version "5.42.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-5.42.1.tgz#05e5e1351485637d466464237e5259b49f609b18" - integrity sha512-QAZY/CBP1Emx4rzxurgqj3rUinfsh/6mvuKbLNMfJMMKYLRBfweus8brgXF8f64ABkIZ3zdj2/rYYtF8eiuksQ== +"@typescript-eslint/scope-manager@5.45.0": + version "5.45.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-5.45.0.tgz#7a4ac1bfa9544bff3f620ab85947945938319a96" + integrity sha512-noDMjr87Arp/PuVrtvN3dXiJstQR1+XlQ4R1EvzG+NMgXi8CuMCXpb8JqNtFHKceVSQ985BZhfRdowJzbv4yKw== dependencies: - "@typescript-eslint/types" "5.42.1" - "@typescript-eslint/visitor-keys" "5.42.1" + "@typescript-eslint/types" "5.45.0" + "@typescript-eslint/visitor-keys" "5.45.0" -"@typescript-eslint/type-utils@5.42.1": - version "5.42.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-5.42.1.tgz#21328feb2d4b193c5852b35aabd241ccc1449daa" - integrity sha512-WWiMChneex5w4xPIX56SSnQQo0tEOy5ZV2dqmj8Z371LJ0E+aymWD25JQ/l4FOuuX+Q49A7pzh/CGIQflxMVXg== +"@typescript-eslint/type-utils@5.45.0": + version "5.45.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-5.45.0.tgz#aefbc954c40878fcebeabfb77d20d84a3da3a8b2" + integrity sha512-DY7BXVFSIGRGFZ574hTEyLPRiQIvI/9oGcN8t1A7f6zIs6ftbrU0nhyV26ZW//6f85avkwrLag424n+fkuoJ1Q== dependencies: - "@typescript-eslint/typescript-estree" "5.42.1" - "@typescript-eslint/utils" "5.42.1" + "@typescript-eslint/typescript-estree" "5.45.0" + "@typescript-eslint/utils" "5.45.0" debug "^4.3.4" tsutils "^3.21.0" -"@typescript-eslint/types@5.42.1": - version "5.42.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-5.42.1.tgz#0d4283c30e9b70d2aa2391c36294413de9106df2" - integrity sha512-Qrco9dsFF5lhalz+lLFtxs3ui1/YfC6NdXu+RAGBa8uSfn01cjO7ssCsjIsUs484vny9Xm699FSKwpkCcqwWwA== +"@typescript-eslint/types@5.45.0": + version "5.45.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-5.45.0.tgz#794760b9037ee4154c09549ef5a96599621109c5" + integrity sha512-QQij+u/vgskA66azc9dCmx+rev79PzX8uDHpsqSjEFtfF2gBUTRCpvYMh2gw2ghkJabNkPlSUCimsyBEQZd1DA== -"@typescript-eslint/typescript-estree@5.42.1": - version "5.42.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-5.42.1.tgz#f9a223ecb547a781d37e07a5ac6ba9ff681eaef0" - integrity sha512-qElc0bDOuO0B8wDhhW4mYVgi/LZL+igPwXtV87n69/kYC/7NG3MES0jHxJNCr4EP7kY1XVsRy8C/u3DYeTKQmw== +"@typescript-eslint/typescript-estree@5.45.0": + version "5.45.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-5.45.0.tgz#f70a0d646d7f38c0dfd6936a5e171a77f1e5291d" + integrity sha512-maRhLGSzqUpFcZgXxg1qc/+H0bT36lHK4APhp0AEUVrpSwXiRAomm/JGjSG+kNUio5kAa3uekCYu/47cnGn5EQ== dependencies: - "@typescript-eslint/types" "5.42.1" - "@typescript-eslint/visitor-keys" "5.42.1" + "@typescript-eslint/types" "5.45.0" + "@typescript-eslint/visitor-keys" "5.45.0" debug "^4.3.4" globby "^11.1.0" is-glob "^4.0.3" semver "^7.3.7" tsutils "^3.21.0" -"@typescript-eslint/utils@5.42.1": - version "5.42.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-5.42.1.tgz#2789b1cd990f0c07aaa3e462dbe0f18d736d5071" - integrity sha512-Gxvf12xSp3iYZd/fLqiQRD4uKZjDNR01bQ+j8zvhPjpsZ4HmvEFL/tC4amGNyxN9Rq+iqvpHLhlqx6KTxz9ZyQ== +"@typescript-eslint/utils@5.45.0": + version "5.45.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-5.45.0.tgz#9cca2996eee1b8615485a6918a5c763629c7acf5" + integrity sha512-OUg2JvsVI1oIee/SwiejTot2OxwU8a7UfTFMOdlhD2y+Hl6memUSL4s98bpUTo8EpVEr0lmwlU7JSu/p2QpSvA== dependencies: "@types/json-schema" "^7.0.9" "@types/semver" "^7.3.12" - "@typescript-eslint/scope-manager" "5.42.1" - "@typescript-eslint/types" "5.42.1" - "@typescript-eslint/typescript-estree" "5.42.1" + "@typescript-eslint/scope-manager" "5.45.0" + "@typescript-eslint/types" "5.45.0" + "@typescript-eslint/typescript-estree" "5.45.0" eslint-scope "^5.1.1" eslint-utils "^3.0.0" semver "^7.3.7" -"@typescript-eslint/visitor-keys@5.42.1": - version "5.42.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-5.42.1.tgz#df10839adf6605e1cdb79174cf21e46df9be4872" - integrity sha512-LOQtSF4z+hejmpUvitPlc4hA7ERGoj2BVkesOcG91HCn8edLGUXbTrErmutmPbl8Bo9HjAvOO/zBKQHExXNA2A== +"@typescript-eslint/visitor-keys@5.45.0": + version "5.45.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-5.45.0.tgz#e0d160e9e7fdb7f8da697a5b78e7a14a22a70528" + integrity sha512-jc6Eccbn2RtQPr1s7th6jJWQHBHI6GBVQkCHoJFQ5UreaKm59Vxw+ynQUPPY2u2Amquc+7tmEoC2G52ApsGNNg== dependencies: - "@typescript-eslint/types" "5.42.1" + "@typescript-eslint/types" "5.45.0" eslint-visitor-keys "^3.3.0" JSONStream@^1.0.3: @@ -1832,7 +1867,7 @@ JSONStream@^1.0.3: jsonparse "^1.2.0" through ">=2.2.7 <3" -abab@^2.0.5, abab@^2.0.6: +abab@^2.0.6: version "2.0.6" resolved "https://registry.yarnpkg.com/abab/-/abab-2.0.6.tgz#41b80f2c871d19686216b82309231cfd3cb3d291" integrity sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA== @@ -1849,13 +1884,13 @@ acorn-globals@^3.0.0: dependencies: acorn "^4.0.4" -acorn-globals@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/acorn-globals/-/acorn-globals-6.0.0.tgz#46cdd39f0f8ff08a876619b55f5ac8a6dc770b45" - integrity sha512-ZQl7LOWaF5ePqqcX4hLuv/bLXYQNfNWw2c0/yX/TsPRKamzHcTGQnlCjHT3TsmkOUVEPS3crCxiPfdzE/Trlhg== +acorn-globals@^7.0.0: + version "7.0.1" + resolved "https://registry.yarnpkg.com/acorn-globals/-/acorn-globals-7.0.1.tgz#0dbf05c44fa7c94332914c02066d5beff62c40c3" + integrity sha512-umOSDSDrfHbTNPuNpC2NSnnA3LUrqpevPb4T9jRx4MagXNS0rs+gwiTcAvqCRmsD6utzsrzNt+ebm00SNWiC3Q== dependencies: - acorn "^7.1.1" - acorn-walk "^7.1.1" + acorn "^8.1.0" + acorn-walk "^8.0.2" acorn-jsx@^5.3.2: version "5.3.2" @@ -1871,11 +1906,16 @@ acorn-node@^1.2.0, acorn-node@^1.3.0, acorn-node@^1.5.2, acorn-node@^1.8.2: acorn-walk "^7.0.0" xtend "^4.0.2" -acorn-walk@^7.0.0, acorn-walk@^7.1.1: +acorn-walk@^7.0.0: version "7.2.0" resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-7.2.0.tgz#0de889a601203909b0fbe07b8938dc21d2e967bc" integrity sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA== +acorn-walk@^8.0.2: + version "8.2.0" + resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.2.0.tgz#741210f2e2426454508853a2f44d0ab83b7f69c1" + integrity sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA== + acorn@^3.1.0: version "3.3.0" resolved "https://registry.yarnpkg.com/acorn/-/acorn-3.3.0.tgz#45e37fb39e8da3f25baee3ff5369e2bb5f22017a" @@ -1886,12 +1926,12 @@ acorn@^4.0.4, acorn@~4.0.2: resolved "https://registry.yarnpkg.com/acorn/-/acorn-4.0.13.tgz#105495ae5361d697bd195c825192e1ad7f253787" integrity sha512-fu2ygVGuMmlzG8ZeRJ0bvR41nsAkxxhbyk8bZ1SS521Z7vmgJFTQQlfz/Mp/nJexGBz+v8sC9bM6+lNgskt4Ug== -acorn@^7.0.0, acorn@^7.1.1: +acorn@^7.0.0: version "7.4.1" resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa" integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A== -acorn@^8.5.0, acorn@^8.8.0: +acorn@^8.1.0, acorn@^8.5.0, acorn@^8.8.0: version "8.8.1" resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.1.tgz#0a3f9cbecc4ec3bea6f0a80b66ae8dd2da250b73" integrity sha512-7zFpHzhnqYKrkYdUjF1HI1bzd0VygEGX8lFk4k5zVMqHEoES+P+7TKI+EvLO9WVMJ8eekdO0aDEK044xTXwPPA== @@ -2068,12 +2108,12 @@ available-typed-arrays@^1.0.5: resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz#92f95616501069d07d10edb2fc37d3e1c65123b7" integrity sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw== -babel-jest@^29.0.0, babel-jest@^29.3.0: - version "29.3.0" - resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-29.3.0.tgz#31fd7ef97dd6a77ddd1b4ab1f8cf77c29e20bb5c" - integrity sha512-LzQWdGm6hUugVeyGpIKI/T4SVT+PgAA5WFPqBDbneK7C/PqfckNb0tc4KvcKXq/PLA1yY6wTvB8Bc/REQdUxFg== +babel-jest@^29.0.0, babel-jest@^29.3.1: + version "29.3.1" + resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-29.3.1.tgz#05c83e0d128cd48c453eea851482a38782249f44" + integrity sha512-aard+xnMoxgjwV70t0L6wkW/3HQQtV+O0PEimxKgzNqCJnbYmroPojdP2tqKSOAt8QAKV/uSZU8851M7B5+fcA== dependencies: - "@jest/transform" "^29.3.0" + "@jest/transform" "^29.3.1" "@types/babel__core" "^7.1.14" babel-plugin-istanbul "^6.1.1" babel-preset-jest "^29.2.0" @@ -2285,11 +2325,6 @@ browser-pack@^6.0.1: through2 "^2.0.0" umd "^3.0.0" -browser-process-hrtime@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz#3c9b4b7d782c8121e56f10106d84c0d0ffc94626" - integrity sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow== - browser-resolve@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/browser-resolve/-/browser-resolve-2.0.0.tgz#99b7304cb392f8d73dba741bb2d7da28c6d7842b" @@ -2516,9 +2551,9 @@ camelcase@^6.2.0: integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== caniuse-lite@^1.0.30001400: - version "1.0.30001431" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001431.tgz#e7c59bd1bc518fae03a4656be442ce6c4887a795" - integrity sha512-zBUoFU0ZcxpvSt9IU66dXVT/3ctO1cy4y9cscs1szkPlcWb6pasYM144GqrUygUbT+k7cmUCW61cvskjcv0enQ== + version "1.0.30001435" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001435.tgz#502c93dbd2f493bee73a408fe98e98fb1dad10b2" + integrity sha512-kdCkUTjR+v4YAJelyiDTqiu82BDr4W4CP5sgTA0ZBmqn30XfS2ZghPLMowik9TPhS+psWJiUNxsqLyurDbmutA== center-align@^0.1.1: version "0.1.3" @@ -2572,10 +2607,15 @@ chokidar@^3.4.0: optionalDependencies: fsevents "~2.3.2" -ci-info@^3.2.0, ci-info@^3.4.0: - version "3.5.0" - resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-3.5.0.tgz#bfac2a29263de4c829d806b1ab478e35091e171f" - integrity sha512-yH4RezKOGlOhxkmhbeNuC4eYZKAUsEaGtBuBzDDP1eFUKiccDWzBABxBfOx31IDwDIXMTxWuwAxUGModvkbuVw== +ci-info@^3.2.0: + version "3.6.1" + resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-3.6.1.tgz#7594f1c95cb7fdfddee7af95a13af7dbc67afdcf" + integrity sha512-up5ggbaDqOqJ4UqLKZ2naVkyqSJQgJi5lwD6b6mM748ysrghDBX0bx/qJTUHzw7zu6Mq4gycviSF5hJnwceD8w== + +ci-info@^3.6.1: + version "3.7.0" + resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-3.7.0.tgz#6d01b3696c59915b6ce057e4aa4adfc2fa25f5ef" + integrity sha512-2CpRNYmImPx+RXKLq6jko/L07phmS9I02TyqkcNU20GCF/GgaWvc58hPtjxDX8lPpkdwc9sNh72V9k00S7ezog== cipher-base@^1.0.0, cipher-base@^1.0.1, cipher-base@^1.0.3: version "1.0.4" @@ -2876,7 +2916,7 @@ dash-ast@^1.0.0: resolved "https://registry.yarnpkg.com/dash-ast/-/dash-ast-1.0.0.tgz#12029ba5fb2f8aa6f0a861795b23c1b4b6c27d37" integrity sha512-Vy4dx7gquTeMcQR/hDkYLGUnwVil6vk4FOOct+djUnHOUWt+zJPJAaRIXaAFkPXtJjvlY7o3rfRu0/3hpnwoUA== -data-urls@^3.0.1: +data-urls@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/data-urls/-/data-urls-3.0.2.tgz#9cf24a477ae22bcef5cd5f6f0bfbc1d2d3be9143" integrity sha512-Jy/tj3ldjZJo63sVAvg6LHt2mHvl4V6AgRAmNDtLdm7faqtsx+aJG42rsyCo9JCoRVKwPFzKlIPx3DIibwSIaQ== @@ -2916,7 +2956,7 @@ decamelize@^1.0.0: resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" integrity sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA== -decimal.js@^10.3.1: +decimal.js@^10.4.1: version "10.4.2" resolved "https://registry.yarnpkg.com/decimal.js/-/decimal.js-10.4.2.tgz#0341651d1d997d86065a2ce3a441fbd0d8e8b98e" integrity sha512-ic1yEvwT6GuvaYwBLLY6/aFFgjZdySKTE8en/fkU3QICTmRtgtSlFn0u0BXN06InZwtfCelR7j8LRiDI/02iGA== @@ -3006,10 +3046,10 @@ diff-sequences@^28.1.1: resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-28.1.1.tgz#9989dc731266dc2903457a70e996f3a041913ac6" integrity sha512-FU0iFaH/E23a+a718l8Qa/19bF9p06kgE0KipMOMadwa3SjnaElKzPaUC0vnibs6/B/9ni97s61mcejk8W1fQw== -diff-sequences@^29.2.0: - version "29.2.0" - resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-29.2.0.tgz#4c55b5b40706c7b5d2c5c75999a50c56d214e8f6" - integrity sha512-413SY5JpYeSBZxmenGEmCVQ8mCgtFJF0w9PROdaS6z987XC2Pd2GOKqOITLtMftmyFZqgtCOb/QA7/Z3ZXfzIw== +diff-sequences@^29.3.1: + version "29.3.1" + resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-29.3.1.tgz#104b5b95fe725932421a9c6e5b4bef84c3f2249e" + integrity sha512-hlM3QR272NXCi4pq+N4Kok4kOp6EsgOM3ZSpJI7Da3UAs+Ttsi8MRmB6trM/lhyzUxGfOgnpkHtgqm5Q/CTcfQ== diffie-hellman@^5.0.0: version "5.0.3" @@ -3027,10 +3067,12 @@ dir-glob@^3.0.1: dependencies: path-type "^4.0.0" -docdash@^1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/docdash/-/docdash-1.2.0.tgz#f99dde5b8a89aa4ed083a3150383e042d06c7f49" - integrity sha512-IYZbgYthPTspgqYeciRJNPhSwL51yer7HAwDXhF5p+H7mTDbPvY3PCk/QDjNxdPCpWkaJVFC4t7iCNB/t9E5Kw== +docdash@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/docdash/-/docdash-2.0.0.tgz#1db24c934f9d4feb5576c638e8c061deb5ae1832" + integrity sha512-AxxZwrMLmiArEHJirdyZmW6YTBKxkd/Z5V9U2EU1crIMtpgoU/cH7Hnc9n1E0lzB1ZSam+VVMSnvlc+9+GUGVg== + dependencies: + "@jsdoc/salty" "^0.2.1" doctrine@^2.1.0: version "2.1.0" @@ -3113,6 +3155,11 @@ enhanced-resolve@^5.10.0: graceful-fs "^4.2.4" tapable "^2.2.0" +entities@^4.4.0: + version "4.4.0" + resolved "https://registry.yarnpkg.com/entities/-/entities-4.4.0.tgz#97bdaba170339446495e653cfd2db78962900174" + integrity sha512-oYp7156SP8LkeGD0GF85ad1X9Ai79WtRsZ2gxJqtBuzH+98YUV6jkHEKlZkMbcrjJjIVJNIDP/3WL9wQkoPbWA== + error-ex@^1.2.0, error-ex@^1.3.1: version "1.3.2" resolved "https://registry.yarnpkg.com/error-ex/-/error-ex-1.3.2.tgz#b4ac40648107fdcdcfae242f428bea8a14d4f1bf" @@ -3286,29 +3333,31 @@ eslint-plugin-import@^2.26.0: resolve "^1.22.0" tsconfig-paths "^3.14.1" -eslint-plugin-matrix-org@^0.7.0: - version "0.7.0" - resolved "https://registry.yarnpkg.com/eslint-plugin-matrix-org/-/eslint-plugin-matrix-org-0.7.0.tgz#4b7456b31e30e7575b62c2aada91915478829f88" - integrity sha512-FLmwE4/cRalB7J+J1BBuTccaXvKtRgAoHlbqSCbdsRqhh27xpxEWXe08KlNiET7drEnnz+xMHXdmvW469gch7g== +eslint-plugin-matrix-org@^0.8.0: + version "0.8.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-matrix-org/-/eslint-plugin-matrix-org-0.8.0.tgz#daa1396900a8cb1c1d88f1a370e45fc32482cd9e" + integrity sha512-/Poz/F8lXYDsmQa29iPSt+kO+Jn7ArvRdq10g0CCk8wbRS0sb2zb6fvd9xL1BgR5UDQL771V0l8X32etvY5yKA== -eslint-plugin-unicorn@^44.0.2: - version "44.0.2" - resolved "https://registry.yarnpkg.com/eslint-plugin-unicorn/-/eslint-plugin-unicorn-44.0.2.tgz#6324a001c0a5e2ac00fb51b30db27d14c6c36ab3" - integrity sha512-GLIDX1wmeEqpGaKcnMcqRvMVsoabeF0Ton0EX4Th5u6Kmf7RM9WBl705AXFEsns56ESkEs0uyelLuUTvz9Tr0w== +eslint-plugin-unicorn@^45.0.0: + version "45.0.1" + resolved "https://registry.yarnpkg.com/eslint-plugin-unicorn/-/eslint-plugin-unicorn-45.0.1.tgz#2307f4620502fd955c819733ce1276bed705b736" + integrity sha512-tLnIw5oDJJc3ILYtlKtqOxPP64FZLTkZkgeuoN6e7x6zw+rhBjOxyvq2c7577LGxXuIhBYrwisZuKNqOOHp3BA== dependencies: "@babel/helper-validator-identifier" "^7.19.1" - ci-info "^3.4.0" + "@eslint-community/eslint-utils" "^4.1.0" + ci-info "^3.6.1" clean-regexp "^1.0.0" - eslint-utils "^3.0.0" esquery "^1.4.0" indent-string "^4.0.0" is-builtin-module "^3.2.0" + jsesc "^3.0.2" lodash "^4.17.21" pluralize "^8.0.0" read-pkg-up "^7.0.1" regexp-tree "^0.1.24" + regjsparser "^0.9.1" safe-regex "^2.1.1" - semver "^7.3.7" + semver "^7.3.8" strip-indent "^3.0.0" eslint-rule-composer@^0.3.0: @@ -3349,10 +3398,10 @@ eslint-visitor-keys@^3.3.0: resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.3.0.tgz#f6480fa6b1f30efe2d1968aa8ac745b862469826" integrity sha512-mQ+suqKJVyeuwGYHAdjMFqjCyfl8+Ldnxuyp3ldiMBFKkvytrXUZWaiPCEav8qDHKty44bD+qV1IP4T+w+xXRA== -eslint@8.26.0: - version "8.26.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.26.0.tgz#2bcc8836e6c424c4ac26a5674a70d44d84f2181d" - integrity sha512-kzJkpaw1Bfwheq4VXUezFriD1GxszX6dUekM7Z3aC2o4hju+tsR/XyTC3RcoSD7jmy9VkPU3+N6YjVU2e96Oyg== +eslint@8.28.0: + version "8.28.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.28.0.tgz#81a680732634677cc890134bcdd9fdfea8e63d6e" + integrity sha512-S27Di+EVyMxcHiwDrFzk8dJYAaD+/5SoWKxL1ri/71CRHsnJnRDPNt2Kzj24+MT9FDupf4aqqyqPrvI8MvQ4VQ== dependencies: "@eslint/eslintrc" "^1.3.3" "@humanwhocodes/config-array" "^0.11.6" @@ -3508,16 +3557,16 @@ expect@^28.1.0: jest-message-util "^28.1.3" jest-util "^28.1.3" -expect@^29.0.0, expect@^29.3.0: - version "29.3.0" - resolved "https://registry.yarnpkg.com/expect/-/expect-29.3.0.tgz#2dad3a73ac837dd8074ff91d25cf1614c3e91504" - integrity sha512-bms139btnQNZh4uxCPmzbWz46YOjtEpYIZ847OfY9GCeSBEfzedHWH0CkdR20Sy+XBs8/FI2lFJPZiuH0NGv+w== +expect@^29.0.0, expect@^29.3.1: + version "29.3.1" + resolved "https://registry.yarnpkg.com/expect/-/expect-29.3.1.tgz#92877aad3f7deefc2e3f6430dd195b92295554a6" + integrity sha512-gGb1yTgU30Q0O/tQq+z30KBWv24ApkMgFUpvKBkyLUBL68Wv8dHdJxTBZFl/iT8K/bqDHvUYRH6IIN3rToopPA== dependencies: - "@jest/expect-utils" "^29.2.2" + "@jest/expect-utils" "^29.3.1" jest-get-type "^29.2.0" - jest-matcher-utils "^29.2.2" - jest-message-util "^29.2.1" - jest-util "^29.2.1" + jest-matcher-utils "^29.3.1" + jest-message-util "^29.3.1" + jest-util "^29.3.1" ext@^1.1.2: version "1.7.0" @@ -3527,9 +3576,9 @@ ext@^1.1.2: type "^2.7.2" fake-indexeddb@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/fake-indexeddb/-/fake-indexeddb-4.0.0.tgz#1dfb2023a3be175e35a6d84975218b432041934d" - integrity sha512-oCfWSJ/qvQn1XPZ8SHX6kY3zr1t+bN7faZ/lltGY0SBGhFOPXnWf0+pbO/MOAgfMx6khC2gK3S/bvAgQpuQHDQ== + version "4.0.1" + resolved "https://registry.yarnpkg.com/fake-indexeddb/-/fake-indexeddb-4.0.1.tgz#09bb2468e21d0832b2177e894765fb109edac8fb" + integrity sha512-hFRyPmvEZILYgdcLBxVdHLik4Tj3gDTu/g7s9ZDOiU3sTNiGx+vEu1ri/AMsFJUZ/1sdRbAVrEcKndh3sViBcA== dependencies: realistic-structured-clone "^3.0.0" @@ -3944,7 +3993,7 @@ https-browserify@^1.0.0: resolved "https://registry.yarnpkg.com/https-browserify/-/https-browserify-1.0.0.tgz#ec06c10e0a34c0f2faf199f7fd7fc78fffd03c73" integrity sha512-J+FkSdyD+0mA0N+81tMotaRMfSL9SGi+xpD3T6YApKsc3bGSXJlfXri3VyFOeYkfLRQisDk1W+jIFFKBeUBbBg== -https-proxy-agent@^5.0.0: +https-proxy-agent@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz#c59ef224a04fe8b754f3db0063a25ea30d0005d6" integrity sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA== @@ -3970,9 +4019,9 @@ ieee754@^1.1.4: integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== ignore@^5.2.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.0.tgz#6d3bac8fa7fe0d45d9f9be7bac2fc279577e345a" - integrity sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ== + version "5.2.1" + resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.1.tgz#c2b1f76cb999ede1502f3a226a9310fdfe88d46c" + integrity sha512-d2qQLzTJ9WxQftPAuEQpSPmKqzxePjzVbpAVv62AQ64NTL+wR4JkrVqR/LqFsFEUsHDAiId52mJteHDFuDkElA== import-fresh@^3.0.0, import-fresh@^3.2.1: version "3.3.0" @@ -4331,74 +4380,74 @@ jest-changed-files@^29.2.0: execa "^5.0.0" p-limit "^3.1.0" -jest-circus@^29.3.0: - version "29.3.0" - resolved "https://registry.yarnpkg.com/jest-circus/-/jest-circus-29.3.0.tgz#997354276d24706e14549cab98ac2995d63c92fd" - integrity sha512-xL1cmbUGBGy923KBZpZ2LRKspHlIhrltrwGaefJ677HXCPY5rTF758BtweamBype2ogcSEK/oqcp1SmYZ/ATig== +jest-circus@^29.3.1: + version "29.3.1" + resolved "https://registry.yarnpkg.com/jest-circus/-/jest-circus-29.3.1.tgz#177d07c5c0beae8ef2937a67de68f1e17bbf1b4a" + integrity sha512-wpr26sEvwb3qQQbdlmei+gzp6yoSSoSL6GsLPxnuayZSMrSd5Ka7IjAvatpIernBvT2+Ic6RLTg+jSebScmasg== dependencies: - "@jest/environment" "^29.3.0" - "@jest/expect" "^29.3.0" - "@jest/test-result" "^29.2.1" - "@jest/types" "^29.2.1" + "@jest/environment" "^29.3.1" + "@jest/expect" "^29.3.1" + "@jest/test-result" "^29.3.1" + "@jest/types" "^29.3.1" "@types/node" "*" chalk "^4.0.0" co "^4.6.0" dedent "^0.7.0" is-generator-fn "^2.0.0" - jest-each "^29.2.1" - jest-matcher-utils "^29.2.2" - jest-message-util "^29.2.1" - jest-runtime "^29.3.0" - jest-snapshot "^29.3.0" - jest-util "^29.2.1" + jest-each "^29.3.1" + jest-matcher-utils "^29.3.1" + jest-message-util "^29.3.1" + jest-runtime "^29.3.1" + jest-snapshot "^29.3.1" + jest-util "^29.3.1" p-limit "^3.1.0" - pretty-format "^29.2.1" + pretty-format "^29.3.1" slash "^3.0.0" stack-utils "^2.0.3" -jest-cli@^29.3.0: - version "29.3.0" - resolved "https://registry.yarnpkg.com/jest-cli/-/jest-cli-29.3.0.tgz#7ca1913f6088570ae58bbb312d70500e00120d41" - integrity sha512-rDb9iasZvqTkgrlwzVGemR5i20T0/XN1ug46Ch2vxTRa0zS5PHaVXQXYzYbuLFHs1xpc+XsB9xPfEkkwbnLJBg== +jest-cli@^29.3.1: + version "29.3.1" + resolved "https://registry.yarnpkg.com/jest-cli/-/jest-cli-29.3.1.tgz#e89dff427db3b1df50cea9a393ebd8640790416d" + integrity sha512-TO/ewvwyvPOiBBuWZ0gm04z3WWP8TIK8acgPzE4IxgsLKQgb377NYGrQLc3Wl/7ndWzIH2CDNNsUjGxwLL43VQ== dependencies: - "@jest/core" "^29.3.0" - "@jest/test-result" "^29.2.1" - "@jest/types" "^29.2.1" + "@jest/core" "^29.3.1" + "@jest/test-result" "^29.3.1" + "@jest/types" "^29.3.1" chalk "^4.0.0" exit "^0.1.2" graceful-fs "^4.2.9" import-local "^3.0.2" - jest-config "^29.3.0" - jest-util "^29.2.1" - jest-validate "^29.2.2" + jest-config "^29.3.1" + jest-util "^29.3.1" + jest-validate "^29.3.1" prompts "^2.0.1" yargs "^17.3.1" -jest-config@^29.3.0: - version "29.3.0" - resolved "https://registry.yarnpkg.com/jest-config/-/jest-config-29.3.0.tgz#4b30188390636106414ea6b43aa6b2fb30d1a54d" - integrity sha512-sTSDs/M+//njznsytxiBxwfDnSWRb6OqiNSlO/B2iw1HUaa1YLsdWmV4AWLXss1XKzv1F0yVK+kA4XOhZ0I1qQ== +jest-config@^29.3.1: + version "29.3.1" + resolved "https://registry.yarnpkg.com/jest-config/-/jest-config-29.3.1.tgz#0bc3dcb0959ff8662957f1259947aedaefb7f3c6" + integrity sha512-y0tFHdj2WnTEhxmGUK1T7fgLen7YK4RtfvpLFBXfQkh2eMJAQq24Vx9472lvn5wg0MAO6B+iPfJfzdR9hJYalg== dependencies: "@babel/core" "^7.11.6" - "@jest/test-sequencer" "^29.3.0" - "@jest/types" "^29.2.1" - babel-jest "^29.3.0" + "@jest/test-sequencer" "^29.3.1" + "@jest/types" "^29.3.1" + babel-jest "^29.3.1" chalk "^4.0.0" ci-info "^3.2.0" deepmerge "^4.2.2" glob "^7.1.3" graceful-fs "^4.2.9" - jest-circus "^29.3.0" - jest-environment-node "^29.3.0" + jest-circus "^29.3.1" + jest-environment-node "^29.3.1" jest-get-type "^29.2.0" jest-regex-util "^29.2.0" - jest-resolve "^29.3.0" - jest-runner "^29.3.0" - jest-util "^29.2.1" - jest-validate "^29.2.2" + jest-resolve "^29.3.1" + jest-runner "^29.3.1" + jest-util "^29.3.1" + jest-validate "^29.3.1" micromatch "^4.0.4" parse-json "^5.2.0" - pretty-format "^29.2.1" + pretty-format "^29.3.1" slash "^3.0.0" strip-json-comments "^3.1.1" @@ -4412,15 +4461,15 @@ jest-diff@^28.1.3: jest-get-type "^28.0.2" pretty-format "^28.1.3" -jest-diff@^29.2.1: - version "29.2.1" - resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-29.2.1.tgz#027e42f5a18b693fb2e88f81b0ccab533c08faee" - integrity sha512-gfh/SMNlQmP3MOUgdzxPOd4XETDJifADpT937fN1iUGz+9DgOu2eUPHH25JDkLVcLwwqxv3GzVyK4VBUr9fjfA== +jest-diff@^29.3.1: + version "29.3.1" + resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-29.3.1.tgz#d8215b72fed8f1e647aed2cae6c752a89e757527" + integrity sha512-vU8vyiO7568tmin2lA3r2DP8oRvzhvRcD4DjpXc6uGveQodyk7CKLhQlCSiwgx3g0pFaE88/KLZ0yaTWMc4Uiw== dependencies: chalk "^4.0.0" - diff-sequences "^29.2.0" + diff-sequences "^29.3.1" jest-get-type "^29.2.0" - pretty-format "^29.2.1" + pretty-format "^29.3.1" jest-docblock@^29.2.0: version "29.2.0" @@ -4429,42 +4478,42 @@ jest-docblock@^29.2.0: dependencies: detect-newline "^3.0.0" -jest-each@^29.2.1: - version "29.2.1" - resolved "https://registry.yarnpkg.com/jest-each/-/jest-each-29.2.1.tgz#6b0a88ee85c2ba27b571a6010c2e0c674f5c9b29" - integrity sha512-sGP86H/CpWHMyK3qGIGFCgP6mt+o5tu9qG4+tobl0LNdgny0aitLXs9/EBacLy3Bwqy+v4uXClqJgASJWcruYw== +jest-each@^29.3.1: + version "29.3.1" + resolved "https://registry.yarnpkg.com/jest-each/-/jest-each-29.3.1.tgz#bc375c8734f1bb96625d83d1ca03ef508379e132" + integrity sha512-qrZH7PmFB9rEzCSl00BWjZYuS1BSOH8lLuC0azQE9lQrAx3PWGKHTDudQiOSwIy5dGAJh7KA0ScYlCP7JxvFYA== dependencies: - "@jest/types" "^29.2.1" + "@jest/types" "^29.3.1" chalk "^4.0.0" jest-get-type "^29.2.0" - jest-util "^29.2.1" - pretty-format "^29.2.1" - -jest-environment-jsdom@^28.1.3: - version "28.1.3" - resolved "https://registry.yarnpkg.com/jest-environment-jsdom/-/jest-environment-jsdom-28.1.3.tgz#2d4e5d61b7f1d94c3bddfbb21f0308ee506c09fb" - integrity sha512-HnlGUmZRdxfCByd3GM2F100DgQOajUBzEitjGqIREcb45kGjZvRrKUdlaF6escXBdcXNl0OBh+1ZrfeZT3GnAg== - dependencies: - "@jest/environment" "^28.1.3" - "@jest/fake-timers" "^28.1.3" - "@jest/types" "^28.1.3" - "@types/jsdom" "^16.2.4" + jest-util "^29.3.1" + pretty-format "^29.3.1" + +jest-environment-jsdom@^29.0.0: + version "29.3.1" + resolved "https://registry.yarnpkg.com/jest-environment-jsdom/-/jest-environment-jsdom-29.3.1.tgz#14ca63c3e0ef5c63c5bcb46033e50bc649e3b639" + integrity sha512-G46nKgiez2Gy4zvYNhayfMEAFlVHhWfncqvqS6yCd0i+a4NsSUD2WtrKSaYQrYiLQaupHXxCRi8xxVL2M9PbhA== + dependencies: + "@jest/environment" "^29.3.1" + "@jest/fake-timers" "^29.3.1" + "@jest/types" "^29.3.1" + "@types/jsdom" "^20.0.0" "@types/node" "*" - jest-mock "^28.1.3" - jest-util "^28.1.3" - jsdom "^19.0.0" - -jest-environment-node@^29.3.0: - version "29.3.0" - resolved "https://registry.yarnpkg.com/jest-environment-node/-/jest-environment-node-29.3.0.tgz#aed95a8e566d80f9f8564fba01ca5caa9d0aa59a" - integrity sha512-oikVE5pyiBUMrqi7J/kFGd1zeT14+EnJulyqzopDNijLX13ygwjiOF/GVpVKSGyBrrAwSkaj/ohEQJCcjkCtOA== - dependencies: - "@jest/environment" "^29.3.0" - "@jest/fake-timers" "^29.3.0" - "@jest/types" "^29.2.1" + jest-mock "^29.3.1" + jest-util "^29.3.1" + jsdom "^20.0.0" + +jest-environment-node@^29.3.1: + version "29.3.1" + resolved "https://registry.yarnpkg.com/jest-environment-node/-/jest-environment-node-29.3.1.tgz#5023b32472b3fba91db5c799a0d5624ad4803e74" + integrity sha512-xm2THL18Xf5sIHoU7OThBPtuH6Lerd+Y1NLYiZJlkE3hbE+7N7r8uvHIl/FkZ5ymKXJe/11SQuf3fv4v6rUMag== + dependencies: + "@jest/environment" "^29.3.1" + "@jest/fake-timers" "^29.3.1" + "@jest/types" "^29.3.1" "@types/node" "*" - jest-mock "^29.3.0" - jest-util "^29.2.1" + jest-mock "^29.3.1" + jest-util "^29.3.1" jest-get-type@^28.0.2: version "28.0.2" @@ -4476,32 +4525,32 @@ jest-get-type@^29.2.0: resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-29.2.0.tgz#726646f927ef61d583a3b3adb1ab13f3a5036408" integrity sha512-uXNJlg8hKFEnDgFsrCjznB+sTxdkuqiCL6zMgA75qEbAJjJYTs9XPrvDctrEig2GDow22T/LvHgO57iJhXB/UA== -jest-haste-map@^29.3.0: - version "29.3.0" - resolved "https://registry.yarnpkg.com/jest-haste-map/-/jest-haste-map-29.3.0.tgz#9786f501ed6ab1c7adc35d3f83c1c21ed4172c77" - integrity sha512-ugdLIreycMRRg3+6AjiExECmuFI2D9PS+BmNU7eGvBt3fzVMKybb9USAZXN6kw4Q6Mn8DSK+7OFCloY2rN820Q== +jest-haste-map@^29.3.1: + version "29.3.1" + resolved "https://registry.yarnpkg.com/jest-haste-map/-/jest-haste-map-29.3.1.tgz#af83b4347f1dae5ee8c2fb57368dc0bb3e5af843" + integrity sha512-/FFtvoG1xjbbPXQLFef+WSU4yrc0fc0Dds6aRPBojUid7qlPqZvxdUBA03HW0fnVHXVCnCdkuoghYItKNzc/0A== dependencies: - "@jest/types" "^29.2.1" + "@jest/types" "^29.3.1" "@types/graceful-fs" "^4.1.3" "@types/node" "*" anymatch "^3.0.3" fb-watchman "^2.0.0" graceful-fs "^4.2.9" jest-regex-util "^29.2.0" - jest-util "^29.2.1" - jest-worker "^29.3.0" + jest-util "^29.3.1" + jest-worker "^29.3.1" micromatch "^4.0.4" walker "^1.0.8" optionalDependencies: fsevents "^2.3.2" -jest-leak-detector@^29.2.1: - version "29.2.1" - resolved "https://registry.yarnpkg.com/jest-leak-detector/-/jest-leak-detector-29.2.1.tgz#ec551686b7d512ec875616c2c3534298b1ffe2fc" - integrity sha512-1YvSqYoiurxKOJtySc+CGVmw/e1v4yNY27BjWTVzp0aTduQeA7pdieLiW05wTYG/twlKOp2xS/pWuikQEmklug== +jest-leak-detector@^29.3.1: + version "29.3.1" + resolved "https://registry.yarnpkg.com/jest-leak-detector/-/jest-leak-detector-29.3.1.tgz#95336d020170671db0ee166b75cd8ef647265518" + integrity sha512-3DA/VVXj4zFOPagGkuqHnSQf1GZBmmlagpguxEERO6Pla2g84Q1MaVIB3YMxgUaFIaYag8ZnTyQgiZ35YEqAQA== dependencies: jest-get-type "^29.2.0" - pretty-format "^29.2.1" + pretty-format "^29.3.1" jest-localstorage-mock@^2.4.6: version "2.4.22" @@ -4518,15 +4567,15 @@ jest-matcher-utils@^28.1.3: jest-get-type "^28.0.2" pretty-format "^28.1.3" -jest-matcher-utils@^29.2.2: - version "29.2.2" - resolved "https://registry.yarnpkg.com/jest-matcher-utils/-/jest-matcher-utils-29.2.2.tgz#9202f8e8d3a54733266784ce7763e9a08688269c" - integrity sha512-4DkJ1sDPT+UX2MR7Y3od6KtvRi9Im1ZGLGgdLFLm4lPexbTaCgJW5NN3IOXlQHF7NSHY/VHhflQ+WoKtD/vyCw== +jest-matcher-utils@^29.3.1: + version "29.3.1" + resolved "https://registry.yarnpkg.com/jest-matcher-utils/-/jest-matcher-utils-29.3.1.tgz#6e7f53512f80e817dfa148672bd2d5d04914a572" + integrity sha512-fkRMZUAScup3txIKfMe3AIZZmPEjWEdsPJFK3AIy5qRohWqQFg1qrmKfYXR9qEkNc7OdAu2N4KPHibEmy4HPeQ== dependencies: chalk "^4.0.0" - jest-diff "^29.2.1" + jest-diff "^29.3.1" jest-get-type "^29.2.0" - pretty-format "^29.2.1" + pretty-format "^29.3.1" jest-message-util@^28.1.3: version "28.1.3" @@ -4543,130 +4592,122 @@ jest-message-util@^28.1.3: slash "^3.0.0" stack-utils "^2.0.3" -jest-message-util@^29.2.1: - version "29.2.1" - resolved "https://registry.yarnpkg.com/jest-message-util/-/jest-message-util-29.2.1.tgz#3a51357fbbe0cc34236f17a90d772746cf8d9193" - integrity sha512-Dx5nEjw9V8C1/Yj10S/8ivA8F439VS8vTq1L7hEgwHFn9ovSKNpYW/kwNh7UglaEgXO42XxzKJB+2x0nSglFVw== +jest-message-util@^29.3.1: + version "29.3.1" + resolved "https://registry.yarnpkg.com/jest-message-util/-/jest-message-util-29.3.1.tgz#37bc5c468dfe5120712053dd03faf0f053bd6adb" + integrity sha512-lMJTbgNcDm5z+6KDxWtqOFWlGQxD6XaYwBqHR8kmpkP+WWWG90I35kdtQHY67Ay5CSuydkTBbJG+tH9JShFCyA== dependencies: "@babel/code-frame" "^7.12.13" - "@jest/types" "^29.2.1" + "@jest/types" "^29.3.1" "@types/stack-utils" "^2.0.0" chalk "^4.0.0" graceful-fs "^4.2.9" micromatch "^4.0.4" - pretty-format "^29.2.1" + pretty-format "^29.3.1" slash "^3.0.0" stack-utils "^2.0.3" -jest-mock@^28.1.3: - version "28.1.3" - resolved "https://registry.yarnpkg.com/jest-mock/-/jest-mock-28.1.3.tgz#d4e9b1fc838bea595c77ab73672ebf513ab249da" - integrity sha512-o3J2jr6dMMWYVH4Lh/NKmDXdosrsJgi4AviS8oXLujcjpCMBb1FMsblDnOXKZKfSiHLxYub1eS0IHuRXsio9eA== +jest-mock@^29.0.0, jest-mock@^29.3.1: + version "29.3.1" + resolved "https://registry.yarnpkg.com/jest-mock/-/jest-mock-29.3.1.tgz#60287d92e5010979d01f218c6b215b688e0f313e" + integrity sha512-H8/qFDtDVMFvFP4X8NuOT3XRDzOUTz+FeACjufHzsOIBAxivLqkB1PoLCaJx9iPPQ8dZThHPp/G3WRWyMgA3JA== dependencies: - "@jest/types" "^28.1.3" - "@types/node" "*" - -jest-mock@^29.0.0, jest-mock@^29.3.0: - version "29.3.0" - resolved "https://registry.yarnpkg.com/jest-mock/-/jest-mock-29.3.0.tgz#aa6f1d5118fd4b9d007df782e0624e6efab6d33d" - integrity sha512-BRKfsAaeP3pTWeog+1D0ILeJF96SzB6y3k0JDxY63kssxiUy9nDLHmNUoVkBGILjMbpHULhbzVTsb3harPXuUQ== - dependencies: - "@jest/types" "^29.2.1" + "@jest/types" "^29.3.1" "@types/node" "*" - jest-util "^29.2.1" + jest-util "^29.3.1" jest-pnp-resolver@^1.2.2: - version "1.2.2" - resolved "https://registry.yarnpkg.com/jest-pnp-resolver/-/jest-pnp-resolver-1.2.2.tgz#b704ac0ae028a89108a4d040b3f919dfddc8e33c" - integrity sha512-olV41bKSMm8BdnuMsewT4jqlZ8+3TCARAXjZGT9jcoSnrfUnRCqnMoF9XEeoWjbzObpqF9dRhHQj0Xb9QdF6/w== + version "1.2.3" + resolved "https://registry.yarnpkg.com/jest-pnp-resolver/-/jest-pnp-resolver-1.2.3.tgz#930b1546164d4ad5937d5540e711d4d38d4cad2e" + integrity sha512-+3NpwQEnRoIBtx4fyhblQDPgJI0H1IEIkX7ShLUjPGA7TtUTvI1oiKi3SR4oBR0hQhQR80l4WAe5RrXBwWMA8w== jest-regex-util@^29.2.0: version "29.2.0" resolved "https://registry.yarnpkg.com/jest-regex-util/-/jest-regex-util-29.2.0.tgz#82ef3b587e8c303357728d0322d48bbfd2971f7b" integrity sha512-6yXn0kg2JXzH30cr2NlThF+70iuO/3irbaB4mh5WyqNIvLLP+B6sFdluO1/1RJmslyh/f9osnefECflHvTbwVA== -jest-resolve-dependencies@^29.3.0: - version "29.3.0" - resolved "https://registry.yarnpkg.com/jest-resolve-dependencies/-/jest-resolve-dependencies-29.3.0.tgz#9a2c3389d1a4cce95445f7c34b0b25f44fff9fad" - integrity sha512-ykSbDbWmIaHprOBig57AExw7i6Fj0y69M6baiAd75Ivx1UMQt4wsM6A+SNqIhycV6Zy8XV3L40Ac3HYSrDSq7w== +jest-resolve-dependencies@^29.3.1: + version "29.3.1" + resolved "https://registry.yarnpkg.com/jest-resolve-dependencies/-/jest-resolve-dependencies-29.3.1.tgz#a6a329708a128e68d67c49f38678a4a4a914c3bf" + integrity sha512-Vk0cYq0byRw2WluNmNWGqPeRnZ3p3hHmjJMp2dyyZeYIfiBskwq4rpiuGFR6QGAdbj58WC7HN4hQHjf2mpvrLA== dependencies: jest-regex-util "^29.2.0" - jest-snapshot "^29.3.0" + jest-snapshot "^29.3.1" -jest-resolve@^29.3.0: - version "29.3.0" - resolved "https://registry.yarnpkg.com/jest-resolve/-/jest-resolve-29.3.0.tgz#038711e9af86f4b4f55a407db14310bc57a8beb4" - integrity sha512-xH6C6loDlOWEWHdCgioLDlbpmsolNdNsV/UR35ChuK217x0ttHuhyEPdh5wa6CTQ/Eq4OGW2/EZTlh0ay5aojQ== +jest-resolve@^29.3.1: + version "29.3.1" + resolved "https://registry.yarnpkg.com/jest-resolve/-/jest-resolve-29.3.1.tgz#9a4b6b65387a3141e4a40815535c7f196f1a68a7" + integrity sha512-amXJgH/Ng712w3Uz5gqzFBBjxV8WFLSmNjoreBGMqxgCz5cH7swmBZzgBaCIOsvb0NbpJ0vgaSFdJqMdT+rADw== dependencies: chalk "^4.0.0" graceful-fs "^4.2.9" - jest-haste-map "^29.3.0" + jest-haste-map "^29.3.1" jest-pnp-resolver "^1.2.2" - jest-util "^29.2.1" - jest-validate "^29.2.2" + jest-util "^29.3.1" + jest-validate "^29.3.1" resolve "^1.20.0" resolve.exports "^1.1.0" slash "^3.0.0" -jest-runner@^29.3.0: - version "29.3.0" - resolved "https://registry.yarnpkg.com/jest-runner/-/jest-runner-29.3.0.tgz#fc255482594c1536a34e2b41f610355c4f3d2ee6" - integrity sha512-E/ROzAVj7gy44FvIe+Tbz0xGWG1sa8WLkhUg/hsXHewPC0Z48kqWySdfYRtXkB7RmMn4OcWE+hIBfsRAMVV+sQ== +jest-runner@^29.3.1: + version "29.3.1" + resolved "https://registry.yarnpkg.com/jest-runner/-/jest-runner-29.3.1.tgz#a92a879a47dd096fea46bb1517b0a99418ee9e2d" + integrity sha512-oFvcwRNrKMtE6u9+AQPMATxFcTySyKfLhvso7Sdk/rNpbhg4g2GAGCopiInk1OP4q6gz3n6MajW4+fnHWlU3bA== dependencies: - "@jest/console" "^29.2.1" - "@jest/environment" "^29.3.0" - "@jest/test-result" "^29.2.1" - "@jest/transform" "^29.3.0" - "@jest/types" "^29.2.1" + "@jest/console" "^29.3.1" + "@jest/environment" "^29.3.1" + "@jest/test-result" "^29.3.1" + "@jest/transform" "^29.3.1" + "@jest/types" "^29.3.1" "@types/node" "*" chalk "^4.0.0" emittery "^0.13.1" graceful-fs "^4.2.9" jest-docblock "^29.2.0" - jest-environment-node "^29.3.0" - jest-haste-map "^29.3.0" - jest-leak-detector "^29.2.1" - jest-message-util "^29.2.1" - jest-resolve "^29.3.0" - jest-runtime "^29.3.0" - jest-util "^29.2.1" - jest-watcher "^29.2.2" - jest-worker "^29.3.0" + jest-environment-node "^29.3.1" + jest-haste-map "^29.3.1" + jest-leak-detector "^29.3.1" + jest-message-util "^29.3.1" + jest-resolve "^29.3.1" + jest-runtime "^29.3.1" + jest-util "^29.3.1" + jest-watcher "^29.3.1" + jest-worker "^29.3.1" p-limit "^3.1.0" source-map-support "0.5.13" -jest-runtime@^29.3.0: - version "29.3.0" - resolved "https://registry.yarnpkg.com/jest-runtime/-/jest-runtime-29.3.0.tgz#e44f838f469ab1f6b524178bae6233697748eb27" - integrity sha512-ufgX/hbpa7MLnjWRW82T5mVF73FBk3W38dGCLPXWtYZ5Zr1ZFh8QnaAtITKJt0p3kGXR8ZqlIjadSiBTk/QJ/A== +jest-runtime@^29.3.1: + version "29.3.1" + resolved "https://registry.yarnpkg.com/jest-runtime/-/jest-runtime-29.3.1.tgz#21efccb1a66911d6d8591276a6182f520b86737a" + integrity sha512-jLzkIxIqXwBEOZx7wx9OO9sxoZmgT2NhmQKzHQm1xwR1kNW/dn0OjxR424VwHHf1SPN6Qwlb5pp1oGCeFTQ62A== dependencies: - "@jest/environment" "^29.3.0" - "@jest/fake-timers" "^29.3.0" - "@jest/globals" "^29.3.0" + "@jest/environment" "^29.3.1" + "@jest/fake-timers" "^29.3.1" + "@jest/globals" "^29.3.1" "@jest/source-map" "^29.2.0" - "@jest/test-result" "^29.2.1" - "@jest/transform" "^29.3.0" - "@jest/types" "^29.2.1" + "@jest/test-result" "^29.3.1" + "@jest/transform" "^29.3.1" + "@jest/types" "^29.3.1" "@types/node" "*" chalk "^4.0.0" cjs-module-lexer "^1.0.0" collect-v8-coverage "^1.0.0" glob "^7.1.3" graceful-fs "^4.2.9" - jest-haste-map "^29.3.0" - jest-message-util "^29.2.1" - jest-mock "^29.3.0" + jest-haste-map "^29.3.1" + jest-message-util "^29.3.1" + jest-mock "^29.3.1" jest-regex-util "^29.2.0" - jest-resolve "^29.3.0" - jest-snapshot "^29.3.0" - jest-util "^29.2.1" + jest-resolve "^29.3.1" + jest-snapshot "^29.3.1" + jest-util "^29.3.1" slash "^3.0.0" strip-bom "^4.0.0" -jest-snapshot@^29.3.0: - version "29.3.0" - resolved "https://registry.yarnpkg.com/jest-snapshot/-/jest-snapshot-29.3.0.tgz#e319cb98cf36640194717fb37a9bd048a3e448f8" - integrity sha512-+4mX3T8XI3ABbZFzBd/AM74mfwOb6gMpYVFNTc0Cgg2F2fGYvHii8D6jWWka99a3wyNFmni3ov8meEVTF8n13Q== +jest-snapshot@^29.3.1: + version "29.3.1" + resolved "https://registry.yarnpkg.com/jest-snapshot/-/jest-snapshot-29.3.1.tgz#17bcef71a453adc059a18a32ccbd594b8cc4e45e" + integrity sha512-+3JOc+s28upYLI2OJM4PWRGK9AgpsMs/ekNryUV0yMBClT9B1DF2u2qay8YxcQd338PPYSFNb0lsar1B49sLDA== dependencies: "@babel/core" "^7.11.6" "@babel/generator" "^7.7.2" @@ -4674,23 +4715,23 @@ jest-snapshot@^29.3.0: "@babel/plugin-syntax-typescript" "^7.7.2" "@babel/traverse" "^7.7.2" "@babel/types" "^7.3.3" - "@jest/expect-utils" "^29.2.2" - "@jest/transform" "^29.3.0" - "@jest/types" "^29.2.1" + "@jest/expect-utils" "^29.3.1" + "@jest/transform" "^29.3.1" + "@jest/types" "^29.3.1" "@types/babel__traverse" "^7.0.6" "@types/prettier" "^2.1.5" babel-preset-current-node-syntax "^1.0.0" chalk "^4.0.0" - expect "^29.3.0" + expect "^29.3.1" graceful-fs "^4.2.9" - jest-diff "^29.2.1" + jest-diff "^29.3.1" jest-get-type "^29.2.0" - jest-haste-map "^29.3.0" - jest-matcher-utils "^29.2.2" - jest-message-util "^29.2.1" - jest-util "^29.2.1" + jest-haste-map "^29.3.1" + jest-matcher-utils "^29.3.1" + jest-message-util "^29.3.1" + jest-util "^29.3.1" natural-compare "^1.4.0" - pretty-format "^29.2.1" + pretty-format "^29.3.1" semver "^7.3.5" jest-util@^28.1.3: @@ -4705,63 +4746,63 @@ jest-util@^28.1.3: graceful-fs "^4.2.9" picomatch "^2.2.3" -jest-util@^29.2.1: - version "29.2.1" - resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-29.2.1.tgz#f26872ba0dc8cbefaba32c34f98935f6cf5fc747" - integrity sha512-P5VWDj25r7kj7kl4pN2rG/RN2c1TLfYYYZYULnS/35nFDjBai+hBeo3MDrYZS7p6IoY3YHZnt2vq4L6mKnLk0g== +jest-util@^29.3.1: + version "29.3.1" + resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-29.3.1.tgz#1dda51e378bbcb7e3bc9d8ab651445591ed373e1" + integrity sha512-7YOVZaiX7RJLv76ZfHt4nbNEzzTRiMW/IiOG7ZOKmTXmoGBxUDefgMAxQubu6WPVqP5zSzAdZG0FfLcC7HOIFQ== dependencies: - "@jest/types" "^29.2.1" + "@jest/types" "^29.3.1" "@types/node" "*" chalk "^4.0.0" ci-info "^3.2.0" graceful-fs "^4.2.9" picomatch "^2.2.3" -jest-validate@^29.2.2: - version "29.2.2" - resolved "https://registry.yarnpkg.com/jest-validate/-/jest-validate-29.2.2.tgz#e43ce1931292dfc052562a11bc681af3805eadce" - integrity sha512-eJXATaKaSnOuxNfs8CLHgdABFgUrd0TtWS8QckiJ4L/QVDF4KVbZFBBOwCBZHOS0Rc5fOxqngXeGXE3nGQkpQA== +jest-validate@^29.3.1: + version "29.3.1" + resolved "https://registry.yarnpkg.com/jest-validate/-/jest-validate-29.3.1.tgz#d56fefaa2e7d1fde3ecdc973c7f7f8f25eea704a" + integrity sha512-N9Lr3oYR2Mpzuelp1F8negJR3YE+L1ebk1rYA5qYo9TTY3f9OWdptLoNSPP9itOCBIRBqjt/S5XHlzYglLN67g== dependencies: - "@jest/types" "^29.2.1" + "@jest/types" "^29.3.1" camelcase "^6.2.0" chalk "^4.0.0" jest-get-type "^29.2.0" leven "^3.1.0" - pretty-format "^29.2.1" + pretty-format "^29.3.1" -jest-watcher@^29.2.2: - version "29.2.2" - resolved "https://registry.yarnpkg.com/jest-watcher/-/jest-watcher-29.2.2.tgz#7093d4ea8177e0a0da87681a9e7b09a258b9daf7" - integrity sha512-j2otfqh7mOvMgN2WlJ0n7gIx9XCMWntheYGlBK7+5g3b1Su13/UAK7pdKGyd4kDlrLwtH2QPvRv5oNIxWvsJ1w== +jest-watcher@^29.3.1: + version "29.3.1" + resolved "https://registry.yarnpkg.com/jest-watcher/-/jest-watcher-29.3.1.tgz#3341547e14fe3c0f79f9c3a4c62dbc3fc977fd4a" + integrity sha512-RspXG2BQFDsZSRKGCT/NiNa8RkQ1iKAjrO0//soTMWx/QUt+OcxMqMSBxz23PYGqUuWm2+m2mNNsmj0eIoOaFg== dependencies: - "@jest/test-result" "^29.2.1" - "@jest/types" "^29.2.1" + "@jest/test-result" "^29.3.1" + "@jest/types" "^29.3.1" "@types/node" "*" ansi-escapes "^4.2.1" chalk "^4.0.0" emittery "^0.13.1" - jest-util "^29.2.1" + jest-util "^29.3.1" string-length "^4.0.1" -jest-worker@^29.3.0: - version "29.3.0" - resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-29.3.0.tgz#240a1cd731c7d6645e8bcc37a3d584f122afb44a" - integrity sha512-rP8LYClB5NCWW0p8GdQT9vRmZNrDmjypklEYZuGCIU5iNviVWCZK5MILS3rQwD0FY1u96bY7b+KoU17DdZy6Ww== +jest-worker@^29.3.1: + version "29.3.1" + resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-29.3.1.tgz#e9462161017a9bb176380d721cab022661da3d6b" + integrity sha512-lY4AnnmsEWeiXirAIA0c9SDPbuCBq8IYuDVL8PMm0MZ2PEs2yPvRA/J64QBXuZp7CYKrDM/rmNrc9/i3KJQncw== dependencies: "@types/node" "*" - jest-util "^29.2.1" + jest-util "^29.3.1" merge-stream "^2.0.0" supports-color "^8.0.0" jest@^29.0.0: - version "29.3.0" - resolved "https://registry.yarnpkg.com/jest/-/jest-29.3.0.tgz#edb9ef5b308e90e1c717c8accc04b28f133ac66d" - integrity sha512-lWmHtOcJSjR6FYRw+4oo7456QUe6LN73Lw6HLwOWKTPLcyQF60cMh0EoIHi67dV74SY5tw/kL+jYC+Ji43ScUg== + version "29.3.1" + resolved "https://registry.yarnpkg.com/jest/-/jest-29.3.1.tgz#c130c0d551ae6b5459b8963747fed392ddbde122" + integrity sha512-6iWfL5DTT0Np6UYs/y5Niu7WIfNv/wRTtN5RSXt2DIEft3dx3zPuw/3WJQBCJfmEzvDiEKwoqMbGD9n49+qLSA== dependencies: - "@jest/core" "^29.3.0" - "@jest/types" "^29.2.1" + "@jest/core" "^29.3.1" + "@jest/types" "^29.3.1" import-local "^3.0.2" - jest-cli "^29.3.0" + jest-cli "^29.3.1" js-sdsl@^4.1.4: version "4.1.5" @@ -4793,37 +4834,36 @@ js-yaml@^4.1.0: dependencies: argparse "^2.0.1" -jsdom@^19.0.0: - version "19.0.0" - resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-19.0.0.tgz#93e67c149fe26816d38a849ea30ac93677e16b6a" - integrity sha512-RYAyjCbxy/vri/CfnjUWJQQtZ3LKlLnDqj+9XLNnJPgEGeirZs3hllKR20re8LUZ6o1b1X4Jat+Qd26zmP41+A== +jsdom@^20.0.0: + version "20.0.2" + resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-20.0.2.tgz#65ccbed81d5e877c433f353c58bb91ff374127db" + integrity sha512-AHWa+QO/cgRg4N+DsmHg1Y7xnz+8KU3EflM0LVDTdmrYOc1WWTSkOjtpUveQH+1Bqd5rtcVnb/DuxV/UjDO4rA== dependencies: - abab "^2.0.5" - acorn "^8.5.0" - acorn-globals "^6.0.0" + abab "^2.0.6" + acorn "^8.8.0" + acorn-globals "^7.0.0" cssom "^0.5.0" cssstyle "^2.3.0" - data-urls "^3.0.1" - decimal.js "^10.3.1" + data-urls "^3.0.2" + decimal.js "^10.4.1" domexception "^4.0.0" escodegen "^2.0.0" form-data "^4.0.0" html-encoding-sniffer "^3.0.0" http-proxy-agent "^5.0.0" - https-proxy-agent "^5.0.0" + https-proxy-agent "^5.0.1" is-potential-custom-element-name "^1.0.1" - nwsapi "^2.2.0" - parse5 "6.0.1" - saxes "^5.0.1" + nwsapi "^2.2.2" + parse5 "^7.1.1" + saxes "^6.0.0" symbol-tree "^3.2.4" - tough-cookie "^4.0.0" - w3c-hr-time "^1.0.2" + tough-cookie "^4.1.2" w3c-xmlserializer "^3.0.0" webidl-conversions "^7.0.0" whatwg-encoding "^2.0.0" whatwg-mimetype "^3.0.0" - whatwg-url "^10.0.0" - ws "^8.2.3" + whatwg-url "^11.0.0" + ws "^8.9.0" xml-name-validator "^4.0.0" jsesc@^2.5.1: @@ -4831,6 +4871,11 @@ jsesc@^2.5.1: resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4" integrity sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA== +jsesc@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-3.0.2.tgz#bb8b09a6597ba426425f2e4a07245c3d00b9343e" + integrity sha512-xKqzzWXDttJuOcawBt4KnKHHIf5oQ/Cxax+0PWFG+DFDgHNAdi+TXECADI+RYiFUMmx8792xsMbbgXj4CwnP4g== + jsesc@~0.5.0: version "0.5.0" resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-0.5.0.tgz#e7dee66e35d6fc16f710fe91d5cf69f70f08911d" @@ -4990,9 +5035,9 @@ lodash@^4.17.21, lodash@^4.17.4, lodash@^4.7.0: integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== loglevel@^1.7.1: - version "1.8.0" - resolved "https://registry.yarnpkg.com/loglevel/-/loglevel-1.8.0.tgz#e7ec73a57e1e7b419cb6c6ac06bf050b67356114" - integrity sha512-G6A/nJLRgWOuuwdNuA6koovfEV1YpqqAG4pRUlFaz3jj2QNZ8M4vBqnVA+HBTmU/AMNUtlOsMmSpF6NyOjztbA== + version "1.8.1" + resolved "https://registry.yarnpkg.com/loglevel/-/loglevel-1.8.1.tgz#5c621f83d5b48c54ae93b6156353f555963377b4" + integrity sha512-tCRIJM51SHjAayKwC+QAg8hT8vg6z7GSgLJKGvzuPb1Wc+hLzqtuVLxp6/HzSPOozuK+8ErAhy7U/sVzw8Dgfg== longest@^1.0.1: version "1.0.1" @@ -5300,7 +5345,7 @@ npm-run-path@^4.0.1: dependencies: path-key "^3.0.0" -nwsapi@^2.2.0: +nwsapi@^2.2.2: version "2.2.2" resolved "https://registry.yarnpkg.com/nwsapi/-/nwsapi-2.2.2.tgz#e5418863e7905df67d51ec95938d67bf801f0bb0" integrity sha512-90yv+6538zuvUMnN+zCr8LuV6bPFdq50304114vJYJ8RDyK8D5O9Phpbd6SZWgI7PwzmmfN1upeOJlvybDSgCw== @@ -5486,10 +5531,12 @@ parse-json@^5.0.0, parse-json@^5.2.0: json-parse-even-better-errors "^2.3.0" lines-and-columns "^1.1.6" -parse5@6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/parse5/-/parse5-6.0.1.tgz#e1a1c085c569b3dc08321184f19a39cc27f7c30b" - integrity sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw== +parse5@^7.0.0, parse5@^7.1.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/parse5/-/parse5-7.1.1.tgz#4649f940ccfb95d8754f37f73078ea20afe0c746" + integrity sha512-kwpuwzB+px5WUg9pyK0IcK/shltJN5/OVhQagxhCQNtT9Y9QRZqNY2e1cmbu/paRh5LMnz/oVTVLBpjFmMZhSg== + dependencies: + entities "^4.4.0" path-browserify@^1.0.0: version "1.0.1" @@ -5601,10 +5648,10 @@ pretty-format@^28.1.3: ansi-styles "^5.0.0" react-is "^18.0.0" -pretty-format@^29.0.0, pretty-format@^29.2.1: - version "29.2.1" - resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-29.2.1.tgz#86e7748fe8bbc96a6a4e04fa99172630907a9611" - integrity sha512-Y41Sa4aLCtKAXvwuIpTvcFBkyeYp2gdFWzXGA+ZNES3VwURIB165XO/z7CjETwzCCS53MjW/rLMyyqEnTtaOfA== +pretty-format@^29.0.0, pretty-format@^29.3.1: + version "29.3.1" + resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-29.3.1.tgz#1841cac822b02b4da8971dacb03e8a871b4722da" + integrity sha512-FyLnmb1cYJV8biEIiRyzRFvs2lry7PPIvOqKVe1GCUEYg4YGmlx1qG9EJNMxArYm7piII4qb8UV1Pncq5dxmcg== dependencies: "@jest/schemas" "^29.0.0" ansi-styles "^5.0.0" @@ -5966,10 +6013,10 @@ regenerator-runtime@^0.11.0: resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz#be05ad7f9bf7d22e056f9726cee5017fbf19e2e9" integrity sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg== -regenerator-runtime@^0.13.10: - version "0.13.10" - resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.10.tgz#ed07b19616bcbec5da6274ebc75ae95634bfc2ee" - integrity sha512-KepLsg4dU12hryUO7bp/axHAKvwGOCV0sGloQtpagJ12ai+ojVDqkeGSiRX1zlq+kjIMZ1t7gpze+26QqtdGqw== +regenerator-runtime@^0.13.11: + version "0.13.11" + resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz#f6dca3e7ceec20590d07ada785636a90cdca17f9" + integrity sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg== regenerator-transform@^0.15.0: version "0.15.0" @@ -6137,10 +6184,10 @@ safe-regex@^2.1.1: resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== -saxes@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/saxes/-/saxes-5.0.1.tgz#eebab953fa3b7608dbe94e5dadb15c888fa6696d" - integrity sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw== +saxes@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/saxes/-/saxes-6.0.0.tgz#fe5b4a4768df4f14a201b1ba6a65c1f3d9988cc5" + integrity sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA== dependencies: xmlchars "^2.2.0" @@ -6159,7 +6206,7 @@ semver@^6.0.0, semver@^6.1.0, semver@^6.1.1, semver@^6.1.2, semver@^6.3.0: resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== -semver@^7.3.5, semver@^7.3.7: +semver@^7.3.5, semver@^7.3.7, semver@^7.3.8: version "7.3.8" resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.8.tgz#07a78feafb3f7b32347d725e33de7e2a2df67798" integrity sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A== @@ -6311,9 +6358,9 @@ sprintf-js@~1.0.2: integrity sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g== stack-utils@^2.0.3: - version "2.0.5" - resolved "https://registry.yarnpkg.com/stack-utils/-/stack-utils-2.0.5.tgz#d25265fca995154659dbbfba3b49254778d2fdd5" - integrity sha512-xrQcmYhOsn/1kX+Vraq+7j4oE2j/6BFscZ0etmYg81xuM8Gq0022Pxb8+IqgOFUIaxHs0KaSb7T1+OegiNrNFA== + version "2.0.6" + resolved "https://registry.yarnpkg.com/stack-utils/-/stack-utils-2.0.6.tgz#aaf0748169c02fc33c8232abccf933f54a1cc34f" + integrity sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ== dependencies: escape-string-regexp "^2.0.0" @@ -6505,9 +6552,9 @@ tapable@^2.2.0: integrity sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ== terser@^5.5.1: - version "5.15.1" - resolved "https://registry.yarnpkg.com/terser/-/terser-5.15.1.tgz#8561af6e0fd6d839669c73b92bdd5777d870ed6c" - integrity sha512-K1faMUvpm/FBxjBXud0LWVAGxmvoPbZbfTCYbSgaaYQaIXI3/TdI7a7ZGA73Zrou6Q8Zmz3oeUTsp/dj+ag2Xw== + version "5.16.0" + resolved "https://registry.yarnpkg.com/terser/-/terser-5.16.0.tgz#29362c6f5506e71545c73b069ccd199bb28f7f54" + integrity sha512-KjTV81QKStSfwbNiwlBXfcgMcOloyuRdb62/iLFPGBcVNF4EXjhdYBhYHmbJpiBrVxZhDvltE11j+LBQUxEEJg== dependencies: "@jridgewell/source-map" "^0.3.2" acorn "^8.5.0" @@ -6596,7 +6643,7 @@ token-stream@0.0.1: resolved "https://registry.yarnpkg.com/token-stream/-/token-stream-0.0.1.tgz#ceeefc717a76c4316f126d0b9dbaa55d7e7df01a" integrity sha512-nfjOAu/zAWmX9tgwi5NRp7O7zTDUD1miHiB40klUnAh9qnL1iXdgzcz/i5dMaL5jahcBAaSfmNOBBJBLJW8TEg== -tough-cookie@^4.0.0: +tough-cookie@^4.1.2: version "4.1.2" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.2.tgz#e53e84b85f24e0b65dd526f46628db6c85f6b874" integrity sha512-G9fqXWoYFZgTc2z8Q5zaHy/vJMjm+WV0AkAeHxVCQiEB1b+dGvWzFW6QV07cY5jQ5gRkeid2qIkzkxUnmoQZUQ== @@ -6749,9 +6796,9 @@ typedoc-plugin-missing-exports@^1.0.0: integrity sha512-7s6znXnuAj1eD9KYPyzVzR1lBF5nwAY8IKccP5sdoO9crG4lpd16RoFpLsh2PccJM+I2NASpr0+/NMka6ThwVA== typedoc@^0.23.20: - version "0.23.20" - resolved "https://registry.yarnpkg.com/typedoc/-/typedoc-0.23.20.tgz#c6fa221762322837161932990b79416afcdc895c" - integrity sha512-nfb4Mx05ZZZXux3zPcLuc7+3TVePDW3jTdEBqXdQzJUyEILxoprgPIiTChbvci9crkqNJG9YESmfCptuh9Gn3g== + version "0.23.21" + resolved "https://registry.yarnpkg.com/typedoc/-/typedoc-0.23.21.tgz#2a6b0e155f91ffa9689086706ad7e3e4bc11d241" + integrity sha512-VNE9Jv7BgclvyH9moi2mluneSviD43dCE9pY8RWkO88/DrEgJZk9KpUk7WO468c9WWs/+aG6dOnoH7ccjnErhg== dependencies: lunr "^2.3.9" marked "^4.0.19" @@ -6764,9 +6811,9 @@ typescript@^3.2.2: integrity sha512-w6fIxVE/H1PkLKcCPsFqKE7Kv7QUwhU8qQY2MueZXWx5cPZdwFupLgKK3vntcK98BtNHZtAF4LA/yl2a7k8R6Q== typescript@^4.5.3, typescript@^4.5.4: - version "4.8.4" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.8.4.tgz#c464abca159669597be5f96b8943500b238e60e6" - integrity sha512-QCh+85mCy+h0IGff8r5XWzOVSbBO+KfeYrMQh7NJ58QujwcE22u+NUSmUxqF+un70P9GXKxa2HCNiTTMJknyjQ== + version "4.9.3" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.9.3.tgz#3aea307c1746b8c384435d8ac36b8a2e580d85db" + integrity sha512-CIfGzTelbKNEnLpLdGFgdyKhG23CKdKgQPOBc+OUNrkJ2vr+KSzsSV5kq5iWhEQbok+quxgGzrAtGWCyU7tHnA== typeson-registry@^1.0.0-alpha.20: version "1.0.0-alpha.39" @@ -6993,13 +7040,6 @@ vue2-ace-editor@^0.0.15: dependencies: brace "^0.11.0" -w3c-hr-time@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz#0a89cdf5cc15822df9c360543676963e0cc308cd" - integrity sha512-z8P5DvDNjKDoFIHK7q8r8lackT6l+jo/Ye3HOle7l9nICP9lf1Ci25fy9vHd0JOWewkIFzXIEig3TdKT7JQ5fQ== - dependencies: - browser-process-hrtime "^1.0.0" - w3c-xmlserializer@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/w3c-xmlserializer/-/w3c-xmlserializer-3.0.0.tgz#06cdc3eefb7e4d0b20a560a5a3aeb0d2d9a65923" @@ -7046,14 +7086,6 @@ whatwg-mimetype@^3.0.0: resolved "https://registry.yarnpkg.com/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz#5fa1a7623867ff1af6ca3dc72ad6b8a4208beba7" integrity sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q== -whatwg-url@^10.0.0: - version "10.0.0" - resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-10.0.0.tgz#37264f720b575b4a311bd4094ed8c760caaa05da" - integrity sha512-CLxxCmdUby142H5FZzn4D8ikO1cmypvXVQktsgosNy4a4BHrDHeciBBGZhb0bNoR5/MltoCatso+vFjjGx8t0w== - dependencies: - tr46 "^3.0.0" - webidl-conversions "^7.0.0" - whatwg-url@^11.0.0: version "11.0.0" resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-11.0.0.tgz#0a849eebb5faf2119b901bb76fd795c2848d4018" @@ -7154,7 +7186,7 @@ write-file-atomic@^4.0.1: imurmurhash "^0.1.4" signal-exit "^3.0.7" -ws@^8.2.3: +ws@^8.9.0: version "8.11.0" resolved "https://registry.yarnpkg.com/ws/-/ws-8.11.0.tgz#6a0d36b8edfd9f96d8b25683db2f8d7de6e8e143" integrity sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg== From f3dcdeeebe795c6c7de2d90c117040f9791e04a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 7 Dec 2022 09:23:41 +0100 Subject: [PATCH 083/137] Remove unintentional change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/client.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/client.ts b/src/client.ts index 8ed4e8a5bed..f69fc507092 100644 --- a/src/client.ts +++ b/src/client.ts @@ -9409,7 +9409,6 @@ export function fixNotificationCountOnDecryption(cli: MatrixClient, event: Matri const isThreadEvent = !!event.threadRootId && !event.isThreadRoot; - const totalCount = room.getUnreadCountForEventContext(NotificationCountType.Total, event); const currentCount = room.getUnreadCountForEventContext(NotificationCountType.Highlight, event); // Ensure the unread counts are kept up to date if the event is encrypted @@ -9417,7 +9416,7 @@ export function fixNotificationCountOnDecryption(cli: MatrixClient, event: Matri // have encrypted events to avoid other code from resetting 'highlight' to zero. const oldHighlight = !!oldActions?.tweaks?.highlight; const newHighlight = !!actions?.tweaks?.highlight; - if ((oldHighlight !== newHighlight || currentCount > 0) && totalCount > 0) { + if (oldHighlight !== newHighlight || currentCount > 0) { // TODO: Handle mentions received while the client is offline // See also https://github.com/vector-im/element-web/issues/9069 const hasReadEvent = isThreadEvent From 218fd23efa4dd82d7a858659da521ce28e4da886 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 7 Dec 2022 09:39:53 +0100 Subject: [PATCH 084/137] Reduce diff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index f5601848e34..8e742e518a2 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2893,7 +2893,6 @@ export class MatrixCall extends TypedEventEmitter(); - const sender = ev.getSender()!; logger.debug(`Call ${this.callId} choosing opponent party ID ${msg.party_id}`); @@ -2909,7 +2908,7 @@ export class MatrixCall extends TypedEventEmitter { From d40c5d9f502fd56575614343b694c814782270f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 8 Dec 2022 16:12:36 +0100 Subject: [PATCH 085/137] Typing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 4 ++-- src/webrtc/groupCall.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 8e742e518a2..2a598ecc60d 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2044,8 +2044,8 @@ export class MatrixCall extends TypedEventEmitter { if (this.dataChannel?.readyState === 'connecting') { const p = new Promise(resolve => { - this.dataChannel!.onopen = () => resolve(); - this.dataChannel!.onclose = () => resolve(); + this.dataChannel!.onopen = (): void => resolve(); + this.dataChannel!.onclose = (): void => resolve(); }); await p; } diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 614f156c74f..b3cb4aba86e 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -1075,7 +1075,7 @@ export class GroupCall extends TypedEventEmitter< } } - private onCallFeedsChanged = (call: MatrixCall) => { + private onCallFeedsChanged = (call: MatrixCall): void => { this.updateMemberState(); // Find removed feeds From f603f5638207e31aa35c33ac5f3bdd5761cf637c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 8 Dec 2022 16:28:59 +0100 Subject: [PATCH 086/137] Remove unnecessary code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 40 ---------------------------------------- 1 file changed, 40 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index b3cb4aba86e..0d211a47ee9 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -1080,24 +1080,6 @@ export class GroupCall extends TypedEventEmitter< // Find removed feeds [...this.userMediaFeeds, ...this.screenshareFeeds].filter((gf) => gf.disposed).forEach((feed) => { - // Only remove the participant if the other feed array doesn't have a feed - // from the given participant - if ( - !(feed.purpose === SDPStreamMetadataPurpose.Usermedia - ? this.screenshareFeeds - : this.userMediaFeeds).some((f) => - ( - f.userId === feed.userId - && f.deviceId === feed.deviceId - ), - ) - ) { - this.participants.get(feed.getMember()!)?.delete(feed.deviceId!); - if (this.participants.get(feed.getMember()!)?.size === 0) { - this.participants.delete(feed.getMember()!); - } - } - if (feed.purpose === SDPStreamMetadataPurpose.Usermedia) this.removeUserMediaFeed(feed); else if (feed.purpose === SDPStreamMetadataPurpose.Screenshare) this.removeScreenshareFeed(feed); }); @@ -1106,28 +1088,6 @@ export class GroupCall extends TypedEventEmitter< call.getRemoteFeeds().filter((cf) => { return !this.userMediaFeeds.find((gf) => gf.stream.id === cf.stream.id); }).forEach((feed) => { - const participant = this.participants.get(feed.getMember()!); - if (participant) { - participant.set( - feed.deviceId!, - { - sessionId: call.getOpponentSessionId()!, - screensharing: false, - }, - ); - } else { - this.participants.set( - feed.getMember()!, - new Map([[ - feed.deviceId!, - { - sessionId: call.getOpponentSessionId()!, - screensharing: false, - }, - ]]), - ); - } - if (feed.purpose === SDPStreamMetadataPurpose.Usermedia) this.addUserMediaFeed(feed); else if (feed.purpose === SDPStreamMetadataPurpose.Screenshare) this.addScreenshareFeed(feed); }); From 4206df2c8247e1576df3ef60125c77bc1fccac70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 8 Dec 2022 16:31:30 +0100 Subject: [PATCH 087/137] Remove duplicate line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 0d211a47ee9..d96e84ef9a2 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -436,8 +436,6 @@ export class GroupCall extends TypedEventEmitter< this.activeSpeaker = undefined; - this.state = GroupCallState.Entered; - logger.log(`Entered group call ${this.groupCallId}`); this.state = GroupCallState.Entered; From 04f5c550ca6fcf007c913cf81559770dd92dcaf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 8 Dec 2022 16:32:28 +0100 Subject: [PATCH 088/137] Remove other duplicate code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index d96e84ef9a2..2380395adcf 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -421,10 +421,6 @@ export class GroupCall extends TypedEventEmitter< throw new Error(`Cannot enter call in the "${this.state}" state`); } - if (this.state === GroupCallState.LocalCallFeedUninitialized) { - await this.initLocalCallFeed(); - } - // TODO: Call preferred foci // This needs to be done before we set the state to entered. With the From cf28825ad74a541ebeb6c63eded0d74e569a7d6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 8 Dec 2022 16:48:06 +0100 Subject: [PATCH 089/137] Futher unnecessary line removal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 2380395adcf..f2189147d9c 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -669,8 +669,6 @@ export class GroupCall extends TypedEventEmitter< if (!sendUpdatesBefore) await sendUpdates(); - this.emit(GroupCallEvent.LocalMuteStateChanged, muted, this.isLocalVideoMuted()); - this.updateMemberState(); return true; } @@ -702,7 +700,6 @@ export class GroupCall extends TypedEventEmitter< await Promise.all(updates); this.emit(GroupCallEvent.LocalMuteStateChanged, this.isMicrophoneMuted(), muted); - this.updateMemberState(); return true; } From 4bdb71f89666111d5cb57406254837556b5e7af1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 8 Dec 2022 20:21:30 +0100 Subject: [PATCH 090/137] Reduce diff 1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCallEventHandler.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts index 592f34edc95..b1768ee90dd 100644 --- a/src/webrtc/groupCallEventHandler.ts +++ b/src/webrtc/groupCallEventHandler.ts @@ -21,7 +21,6 @@ import { GroupCallIntent, GroupCallType, IGroupCallDataChannelOptions, - IGroupCallRoomState, } from "./groupCall"; import { Room } from "../models/room"; import { RoomState, RoomStateEvent } from "../models/room-state"; @@ -143,7 +142,7 @@ export class GroupCallEventHandler { private createGroupCallFromRoomStateEvent(event: MatrixEvent): GroupCall | undefined { const roomId = event.getRoomId(); - const content = event.getContent(); + const content = event.getContent(); const room = this.client.getRoom(roomId); From 8f7d6dd272bd77996b379efbf9cf1d38bbe2b538 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 9 Dec 2022 14:55:12 +0100 Subject: [PATCH 091/137] Remove another unnecessary update member state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 868bdd15201..b457d4a02c7 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -1072,8 +1072,6 @@ export class GroupCall extends TypedEventEmitter< } private onCallFeedsChanged = (call: MatrixCall): void => { - this.updateMemberState(); - // Find removed feeds [...this.userMediaFeeds, ...this.screenshareFeeds].filter((gf) => gf.disposed).forEach((feed) => { if (feed.purpose === SDPStreamMetadataPurpose.Usermedia) this.removeUserMediaFeed(feed); From ca12d5af908fbc5cf58ee20a6d4a63251844db2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 12 Dec 2022 16:43:29 +0100 Subject: [PATCH 092/137] Change event shape to match MSC3898 (#2964) --- src/@types/event.ts | 5 + src/client.ts | 24 ++-- src/webrtc/call.ts | 207 +++++++++++++++++------------------ src/webrtc/callEventTypes.ts | 56 +++------- src/webrtc/groupCall.ts | 78 +++++++------ 5 files changed, 175 insertions(+), 195 deletions(-) diff --git a/src/@types/event.ts b/src/@types/event.ts index 230cb05039d..e7399a85d3e 100644 --- a/src/@types/event.ts +++ b/src/@types/event.ts @@ -42,6 +42,7 @@ export enum EventType { RoomMessage = "m.room.message", RoomMessageEncrypted = "m.room.encrypted", Sticker = "m.sticker", + CallInvite = "m.call.invite", CallCandidates = "m.call.candidates", CallAnswer = "m.call.answer", @@ -54,6 +55,10 @@ export enum EventType { CallReplaces = "m.call.replaces", CallAssertedIdentity = "m.call.asserted_identity", CallAssertedIdentityPrefix = "org.matrix.call.asserted_identity", + CallTrackSubscription = "m.call.track_subscription", + CallPing = "m.call.ping", + CallPong = "m.call.pong", + KeyVerificationRequest = "m.key.verification.request", KeyVerificationStart = "m.key.verification.start", KeyVerificationCancel = "m.key.verification.cancel", diff --git a/src/client.ts b/src/client.ts index b9de5b669ac..d3128483517 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1774,10 +1774,12 @@ export class MatrixClient extends TypedEventEmitter(); - private subscribedTracks: ISfuTrackDesc[] = []; + private subscribedTracks: FocusTrackDescription[] = []; private inviteOrAnswerSent = false; private waitForLocalAVStream = false; @@ -699,11 +695,11 @@ export class MatrixCall extends TypedEventEmitter { - await this.peerConn!.setLocalDescription(offer); - - this.sendSFUDataChannelMessage(SFUDataChannelMessageOp.Publish, { - sdp: offer.sdp, - } as ISfuPublishDataChannelMessage); - this.emit(CallEvent.FeedsChanged, this.feeds); - }); - } else { - this.emit(CallEvent.FeedsChanged, this.feeds); - } + this.emit(CallEvent.FeedsChanged, this.feeds); } /** @@ -874,7 +858,7 @@ export class MatrixCall extends TypedEventEmitter { - await this.peerConn!.setLocalDescription(offer); - - this.sendSFUDataChannelMessage(SFUDataChannelMessageOp.Unpublish, { - sdp: offer.sdp, - stop: tracksToUnpublish, - } as ISfuUnpublishDataChannelMessage); - }); - } } private deleteAllFeeds(): void { @@ -979,8 +952,9 @@ export class MatrixCall extends TypedEventEmitter { if (this.isFocus) { - this.sendSFUDataChannelMessage(SFUDataChannelMessageOp.Metadata); + this.sendFocusEvent(EventType.CallSDPStreamMetadataChanged); } else { await this.sendVoipEvent(EventType.CallSDPStreamMetadataChangedPrefix, { [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(), @@ -1975,7 +1949,7 @@ export class MatrixCall extends TypedEventEmitter => { // TODO: play nice with application layer DC listeners - let json: ISfuBaseDataChannelMessage; + let json: FocusEvent; try { json = JSON.parse(event.data); } catch (e) { @@ -1983,52 +1957,59 @@ export class MatrixCall extends TypedEventEmitter { if (this.dataChannel?.readyState === 'connecting') { - const p = new Promise(resolve => { + const p = new Promise((resolve) => { this.dataChannel!.onopen = (): void => resolve(); this.dataChannel!.onclose = (): void => resolve(); }); @@ -2050,24 +2031,24 @@ export class MatrixCall extends TypedEventEmitter Boolean(info.tracks)) // Skip trackless feeds - .reduce((a: ISfuTrackDesc[], [s, i]) => ( + .reduce((a: FocusTrackDescription[], [s, i]) => ( [...a, ...Object.keys(i.tracks).map((t) => ({ stream_id: s, track_id: t }))] ), []) // Get array of tracks from feeds .filter((track) => !this.subscribedTracks.find((subscribed) => utils.deepCompare(track, subscribed))); // Filter out already subscribed tracks if (tracks.length === 0) { - logger.warn("Failed to find any new streams to subscribe to"); + logger.info("Failed to find any new streams to subscribe to"); return; } else { this.subscribedTracks.push(...tracks); - logger.warn("Subscribing to:", tracks); } - this.sendSFUDataChannelMessage(SFUDataChannelMessageOp.Select, { - start: tracks, - } as ISfuSelectDataChannelMessage); + this.sendFocusEvent(EventType.CallTrackSubscription, { + subscribe: tracks, + unsubscribe: [], + } as FocusTrackSubscriptionEvent); } public updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { @@ -2167,6 +2148,18 @@ export class MatrixCall extends TypedEventEmitter { - this.sendSFUDataChannelMessage(SFUDataChannelMessageOp.Alive); - }, SFU_KEEP_ALIVE_INTERVAL * 3 / 4); - } - } else if (this.peerConn?.iceConnectionState == 'failed') { + } else if (this.peerConn?.iceConnectionState == "failed") { // Firefox for Android does not yet have support for restartIce() // (the types say it's always defined though, so we have to cast // to prevent typescript from warning). @@ -2332,6 +2320,15 @@ export class MatrixCall extends TypedEventEmitter { if (stream.getTracks().length === 0) { + // FIXME: We should really be doing this per-track. Šimon + // leaves this for when we switch to mids for signalling + const getIndex = (): number => this.subscribedTracks.findIndex((t) => t.stream_id === stream.id); + let indexOfTrackToRemove = getIndex(); + while (indexOfTrackToRemove !== -1) { + this.subscribedTracks.splice(indexOfTrackToRemove, 1); + indexOfTrackToRemove = getIndex(); + } + logger.info(`Call ${this.callId} removing track streamId: ${stream.id}`); this.deleteFeedByStream(stream); stream.removeEventListener("removetrack", onRemoveTrack); @@ -2388,9 +2385,6 @@ export class MatrixCall extends TypedEventEmitter => { - // Other than the initial offer, we handle negotiation manually when calling with an SFU - if (this.isFocus && ![CallState.Fledgling, CallState.CreateOffer].includes(this.state)) return; - logger.info(`Call ${this.callId} Negotiation is needed!`); if (this.state !== CallState.CreateOffer && this.opponentVersion === 0) { @@ -2508,18 +2502,19 @@ export class MatrixCall extends TypedEventEmitter ( - this.foci.find(f => pf.user_id === f.user_id && pf.device_id === pf.device_id) - ))); + const isUsingPreferredFocus = Boolean( + preferredFoci.find((pf) => + this.foci.find((f) => pf.user_id === f.user_id && pf.device_id === pf.device_id), + ), + ); return isUsingPreferredFocus ? [] : preferredFoci; } @@ -457,17 +459,14 @@ export class GroupCall extends TypedEventEmitter< this.activeSpeaker = undefined; this.onActiveSpeakerLoop(); - this.activeSpeakerLoopInterval = setInterval( - this.onActiveSpeakerLoop, - this.activeSpeakerInterval, - ); + this.activeSpeakerLoopInterval = setInterval(this.onActiveSpeakerLoop, this.activeSpeakerInterval); if (this.foci[0]) { const opponentDevice = { - "device_id": this.foci[0].device_id, + device_id: this.foci[0].device_id, // XXX: What if an SFU gets restarted? - "session_id": "sfu", - "feeds": [], + session_id: "sfu", + feeds: [], }; const onError = (e?: unknown): void => { @@ -481,17 +480,13 @@ export class GroupCall extends TypedEventEmitter< ); }; - const sfuCall = createNewMatrixCall( - this.client, - this.room.roomId, - { - invitee: this.foci[0].user_id, - opponentDeviceId: opponentDevice.device_id, - opponentSessionId: opponentDevice.session_id, - groupCallId: this.groupCallId, - isFocus: true, - }, - ); + const sfuCall = createNewMatrixCall(this.client, this.room.roomId, { + invitee: this.foci[0].user_id, + opponentDeviceId: opponentDevice.device_id, + opponentSessionId: opponentDevice.session_id, + groupCallId: this.groupCallId, + isFocus: true, + }); if (!sfuCall) { onError(); return; @@ -517,10 +512,10 @@ export class GroupCall extends TypedEventEmitter< // Try to find a focus of another user to use let focusOfAnotherMember: IFocusInfo | undefined; for (const event of this.getMemberStateEvents()) { - const focus = event.getContent() - ?.["m.calls"] ?.[0] - ?.["m.devices"]?.[0] - ?.["m.foci.active"]?.[0]; + const focus = + event.getContent()?.["m.calls"]?.[0]?.["m.devices"]?.[0]?.[ + "m.foci.active" + ]?.[0]; if (focus) { focusOfAnotherMember = focus; break; @@ -759,10 +754,9 @@ export class GroupCall extends TypedEventEmitter< ); // TODO: handle errors - this.forEachCall(call => call.pushLocalFeed(call.isFocus - ? this.localScreenshareFeed! - : this.localScreenshareFeed!.clone(), - )); + this.forEachCall((call) => + call.pushLocalFeed(call.isFocus ? this.localScreenshareFeed! : this.localScreenshareFeed!.clone()), + ); return true; } catch (error) { @@ -1073,18 +1067,22 @@ export class GroupCall extends TypedEventEmitter< private onCallFeedsChanged = (call: MatrixCall): void => { // Find removed feeds - [...this.userMediaFeeds, ...this.screenshareFeeds].filter((gf) => gf.disposed).forEach((feed) => { - if (feed.purpose === SDPStreamMetadataPurpose.Usermedia) this.removeUserMediaFeed(feed); - else if (feed.purpose === SDPStreamMetadataPurpose.Screenshare) this.removeScreenshareFeed(feed); - }); + [...this.userMediaFeeds, ...this.screenshareFeeds] + .filter((gf) => gf.disposed) + .forEach((feed) => { + if (feed.purpose === SDPStreamMetadataPurpose.Usermedia) this.removeUserMediaFeed(feed); + else if (feed.purpose === SDPStreamMetadataPurpose.Screenshare) this.removeScreenshareFeed(feed); + }); // Find new feeds - call.getRemoteFeeds().filter((cf) => { - return !this.userMediaFeeds.find((gf) => gf.stream.id === cf.stream.id); - }).forEach((feed) => { - if (feed.purpose === SDPStreamMetadataPurpose.Usermedia) this.addUserMediaFeed(feed); - else if (feed.purpose === SDPStreamMetadataPurpose.Screenshare) this.addScreenshareFeed(feed); - }); + call.getRemoteFeeds() + .filter((cf) => { + return !this.userMediaFeeds.find((gf) => gf.stream.id === cf.stream.id); + }) + .forEach((feed) => { + if (feed.purpose === SDPStreamMetadataPurpose.Usermedia) this.addUserMediaFeed(feed); + else if (feed.purpose === SDPStreamMetadataPurpose.Screenshare) this.addScreenshareFeed(feed); + }); }; private onCallStateChanged = (call: MatrixCall, state: CallState, _oldState: CallState | undefined): void => { @@ -1374,7 +1372,7 @@ export class GroupCall extends TypedEventEmitter< "device_id": this.client.getDeviceId()!, "session_id": this.client.getSessionId(), "expires_ts": Date.now() + DEVICE_TIMEOUT, - "feeds": this.getLocalFeeds().map(feed => ({ purpose: feed.purpose })), + "feeds": this.getLocalFeeds().map((feed) => ({ purpose: feed.purpose })), "m.foci.active": this.foci, "m.foci.preferred": this.getPreferredFoci(), // TODO: Add data channels From a1f763adb0ab8a2820f1b5e29a4c19ddf21055a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 13 Dec 2022 10:42:39 +0100 Subject: [PATCH 093/137] Fix `sdp_stream_metadata` key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/callEventTypes.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/callEventTypes.ts b/src/webrtc/callEventTypes.ts index 207b7c1a67f..c6a366ee952 100644 --- a/src/webrtc/callEventTypes.ts +++ b/src/webrtc/callEventTypes.ts @@ -6,7 +6,7 @@ import { CallErrorCode } from "./call"; // TODO: Change to "sdp_stream_metadata" when MSC3077 is merged export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata"; -export const SDPStreamMetadataKeyStable = "m.sdp_stream_metadata"; +export const SDPStreamMetadataKeyStable = "sdp_stream_metadata"; export enum SDPStreamMetadataPurpose { Usermedia = "m.usermedia", From e10efc197a65875ddf1fa76553829f2bd9d33db8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 13 Dec 2022 15:40:06 +0100 Subject: [PATCH 094/137] Fix prefixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 1c40cf0afa7..f0c0bc7fa43 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -24,6 +24,7 @@ import { CallEventHandlerEvent } from "./callEventHandler"; import { GroupCallEventHandlerEvent } from "./groupCallEventHandler"; import { IScreensharingOpts } from "./mediaHandler"; import { mapsEqual } from "../utils"; +import { UnstableValue } from "../NamespacedValue"; export enum GroupCallIntent { Ring = "m.ring", @@ -141,8 +142,8 @@ export interface IGroupCallRoomMemberDevice { "session_id": string; "expires_ts": number; "feeds": IGroupCallRoomMemberFeed[]; - "m.foci.active"?: IFocusInfo[]; - "m.foci.preferred"?: IFocusInfo[]; + "org.matrix.msc3898.foci.active"?: IFocusInfo[]; + "org.matrix.msc3898.foci.preferred"?: IFocusInfo[]; } export interface IGroupCallRoomMemberCallState { @@ -514,7 +515,7 @@ export class GroupCall extends TypedEventEmitter< for (const event of this.getMemberStateEvents()) { const focus = event.getContent()?.["m.calls"]?.[0]?.["m.devices"]?.[0]?.[ - "m.foci.active" + "org.matrix.msc3898.foci.active" ]?.[0]; if (focus) { focusOfAnotherMember = focus; @@ -1373,8 +1374,8 @@ export class GroupCall extends TypedEventEmitter< "session_id": this.client.getSessionId(), "expires_ts": Date.now() + DEVICE_TIMEOUT, "feeds": this.getLocalFeeds().map((feed) => ({ purpose: feed.purpose })), - "m.foci.active": this.foci, - "m.foci.preferred": this.getPreferredFoci(), + "org.matrix.msc3898.foci.active": this.foci, + "org.matrix.msc3898.foci.preferred": this.getPreferredFoci(), // TODO: Add data channels }, ]); From 41d6c9a1b2a03be4f2ec333642749d52594eacf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 14 Dec 2022 08:39:19 +0100 Subject: [PATCH 095/137] Delint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 81 ++++++++++++++++++++++------------------- src/webrtc/groupCall.ts | 1 - 2 files changed, 44 insertions(+), 38 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index bad04ef1cb8..3089bd87b63 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1965,48 +1965,51 @@ export class MatrixCall extends TypedEventEmitter { - if (this.dataChannel?.readyState === 'connecting') { + if (this.dataChannel?.readyState === "connecting") { const p = new Promise((resolve) => { this.dataChannel!.onopen = (): void => resolve(); this.dataChannel!.onclose = (): void => resolve(); @@ -2033,9 +2036,13 @@ export class MatrixCall extends TypedEventEmitter Boolean(info.tracks)) // Skip trackless feeds - .reduce((a: FocusTrackDescription[], [s, i]) => ( - [...a, ...Object.keys(i.tracks).map((t) => ({ stream_id: s, track_id: t }))] - ), []) // Get array of tracks from feeds + .reduce( + (a: FocusTrackDescription[], [s, i]) => [ + ...a, + ...Object.keys(i.tracks).map((t) => ({ stream_id: s, track_id: t })), + ], + [], + ) // Get array of tracks from feeds .filter((track) => !this.subscribedTracks.find((subscribed) => utils.deepCompare(track, subscribed))); // Filter out already subscribed tracks if (tracks.length === 0) { diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index f0c0bc7fa43..f5ad650b969 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -24,7 +24,6 @@ import { CallEventHandlerEvent } from "./callEventHandler"; import { GroupCallEventHandlerEvent } from "./groupCallEventHandler"; import { IScreensharingOpts } from "./mediaHandler"; import { mapsEqual } from "../utils"; -import { UnstableValue } from "../NamespacedValue"; export enum GroupCallIntent { Ring = "m.ring", From 01576f306f93deeb1fc9274e78e29deb8f445829 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 14 Dec 2022 08:57:38 +0100 Subject: [PATCH 096/137] Add `getFoci()` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- spec/test-utils/webrtc.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index d8e350030f1..61f703cb8be 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -22,6 +22,7 @@ import { GroupCallIntent, GroupCallType, IContent, + IFocusInfo, ISendEventResponse, MatrixClient, MatrixEvent, @@ -488,6 +489,10 @@ export class MockCallMatrixClient extends TypedEventEmitter { From 3c302737b7f8c3d1ff3ac8ac821c681bfd05e51c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 14 Dec 2022 09:35:27 +0100 Subject: [PATCH 097/137] Small improvements and diff reduction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 32 ++++++++------------------------ src/webrtc/groupCall.ts | 6 ++---- 2 files changed, 10 insertions(+), 28 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 3089bd87b63..216e5f74ddd 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -336,7 +336,7 @@ export class MatrixCall extends TypedEventEmitter(); private remoteAssertedIdentity?: AssertedIdentity; - private remoteSDPStreamMetadata?: SDPStreamMetadata; - private sfuKeepAliveInterval?: ReturnType; private callLengthInterval?: ReturnType; private callStartTime?: number; @@ -659,8 +657,10 @@ export class MatrixCall extends TypedEventEmitter>(); public readonly userMediaFeeds: CallFeed[] = []; public readonly screenshareFeeds: CallFeed[] = []; public groupCallId: string; public foci: IFocusInfo[] = []; + private readonly calls = new Map>(); // RoomMember | IFocusInfo -> device ID -> MatrixCall private callHandlers = new Map>(); // User ID -> device ID -> handlers private activeSpeakerLoopInterval?: ReturnType; private retryCallLoopInterval?: ReturnType; @@ -218,7 +218,7 @@ export class GroupCall extends TypedEventEmitter< public isPtt: boolean, public intent: GroupCallIntent, groupCallId?: string, - private dataChannelsEnabled = false, + private dataChannelsEnabled?: boolean, private dataChannelOptions?: IGroupCallDataChannelOptions, ) { super(); @@ -444,8 +444,6 @@ export class GroupCall extends TypedEventEmitter< await this.updateMemberState(); - this.activeSpeaker = undefined; - logger.log(`Entered group call ${this.groupCallId}`); this.state = GroupCallState.Entered; From 5387c73080ae6e87da8c4599daf7cb399351371c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 14 Dec 2022 09:40:35 +0100 Subject: [PATCH 098/137] Make `dataChannel` `private` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 216e5f74ddd..a4958109f97 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -393,11 +393,12 @@ export class MatrixCall extends TypedEventEmitter; private callStartTime?: number; + private dataChannel?: RTCDataChannel; + private opponentDeviceId?: string; private opponentDeviceInfo?: DeviceInfo; private opponentSessionId?: string; public groupCallId?: string; - public dataChannel?: RTCDataChannel; /** * Construct a new Matrix Call. From 272e452a4aac095f4188006cff90edc092143e8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 14 Dec 2022 09:47:03 +0100 Subject: [PATCH 099/137] Renaming and tiny improvements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 23 ++++++++++++++--------- src/webrtc/groupCall.ts | 20 ++++++++++---------- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index a4958109f97..08574bec41d 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -614,7 +614,7 @@ export class MatrixCall extends TypedEventEmitter { // XXX: We only use double equals because MediaDescription::mid is in fact a number @@ -627,7 +627,7 @@ export class MatrixCall extends TypedEventEmitter { - await this.waitForDatachannelToBeOpen(); + public subscribeToFocus(): void { if (!this.remoteSDPStreamMetadata) return; - if (this.dataChannel?.readyState !== "open") return; const tracks: FocusTrackDescription[] = Object.entries(this.remoteSDPStreamMetadata) .filter(([, info]) => Boolean(info.tracks)) // Skip trackless feeds @@ -2059,7 +2057,7 @@ export class MatrixCall extends TypedEventEmitter { + await this.waitForDatachannelToBeOpen(); + if (this.dataChannel?.readyState !== "open") { + logger.error("Failed to send focus event because data-channel is not open"); + return; + } + const event: FocusEvent = { type: type, content: content, diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 09ee7866b6b..98213d97ee1 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -462,7 +462,7 @@ export class GroupCall extends TypedEventEmitter< if (this.foci[0]) { const opponentDevice = { device_id: this.foci[0].device_id, - // XXX: What if an SFU gets restarted? + // XXX: What if a focus gets restarted? session_id: "sfu", feeds: [], }; @@ -478,29 +478,29 @@ export class GroupCall extends TypedEventEmitter< ); }; - const sfuCall = createNewMatrixCall(this.client, this.room.roomId, { + const focusCall = createNewMatrixCall(this.client, this.room.roomId, { invitee: this.foci[0].user_id, opponentDeviceId: opponentDevice.device_id, opponentSessionId: opponentDevice.session_id, groupCallId: this.groupCallId, isFocus: true, }); - if (!sfuCall) { + if (!focusCall) { onError(); return; } try { - sfuCall.isPtt = this.isPtt; - await sfuCall.placeCallWithCallFeeds(this.getLocalFeeds()); - sfuCall.createDataChannel("datachannel", this.dataChannelOptions); + focusCall.isPtt = this.isPtt; + await focusCall.placeCallWithCallFeeds(this.getLocalFeeds()); + focusCall.createDataChannel("datachannel", this.dataChannelOptions); } catch (e) { onError(e); return; } - this.calls.set(this.foci[0], new Map([[opponentDevice.device_id, sfuCall]])); - this.initCall(sfuCall); + this.calls.set(this.foci[0], new Map([[opponentDevice.device_id, focusCall]])); + this.initCall(focusCall); } } @@ -1097,9 +1097,9 @@ export class GroupCall extends TypedEventEmitter< } if (state === CallState.Connected) { - // if we're calling an SFU, subscribe to its feeds + // If we're calling a focus, subscribe to its feeds if (call.isFocus) { - call.subscribeToSFU(); + call.subscribeToFocus(); } const opponent = call.getOpponentMember()!; From a89a6208ff39173ad1ace2b4d439aba9eabdddb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 14 Dec 2022 13:07:50 +0100 Subject: [PATCH 100/137] Re-use `onNegotiateReceived()` method MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 54 ++++++++++++++++------------------------------ 1 file changed, 18 insertions(+), 36 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 08574bec41d..cb33ec00ec8 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1877,7 +1877,13 @@ export class MatrixCall extends TypedEventEmitter { const content = event.getContent(); - const description = content.description; + return this.onNegotiateContentReceived(content.description, content[SDPStreamMetadataKey]); + } + + private async onNegotiateContentReceived( + description: RTCSessionDescription, + sdpStreamMetadata: SDPStreamMetadata, + ): Promise { if (!description || !description.sdp || !description.type) { logger.info(`Call ${this.callId} Ignoring invalid m.call.negotiate event`); return; @@ -1900,7 +1906,6 @@ export class MatrixCall extends TypedEventEmitter Date: Thu, 15 Dec 2022 13:55:40 +0100 Subject: [PATCH 101/137] Clone streams even if we don't have to MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This fixes the mid-call device switch bug Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 98213d97ee1..a3ba8099e20 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -492,7 +492,7 @@ export class GroupCall extends TypedEventEmitter< try { focusCall.isPtt = this.isPtt; - await focusCall.placeCallWithCallFeeds(this.getLocalFeeds()); + await focusCall.placeCallWithCallFeeds(this.getLocalFeeds().map((feed) => feed.clone())); focusCall.createDataChannel("datachannel", this.dataChannelOptions); } catch (e) { onError(e); @@ -752,9 +752,7 @@ export class GroupCall extends TypedEventEmitter< ); // TODO: handle errors - this.forEachCall((call) => - call.pushLocalFeed(call.isFocus ? this.localScreenshareFeed! : this.localScreenshareFeed!.clone()), - ); + this.forEachCall((call) => call.pushLocalFeed(this.localScreenshareFeed!.clone())); return true; } catch (error) { @@ -777,9 +775,7 @@ export class GroupCall extends TypedEventEmitter< this.client.getMediaHandler().stopScreensharingStream(this.localScreenshareFeed!.stream); // We have to remove the feed manually as MatrixCall has its clone, // so it won't be removed automatically - if (!this.foci[0]) { - this.removeScreenshareFeed(this.localScreenshareFeed!); - } + this.removeScreenshareFeed(this.localScreenshareFeed!); this.localScreenshareFeed = undefined; this.localDesktopCapturerSourceId = undefined; this.emit(GroupCallEvent.LocalScreenshareStateChanged, false, undefined, undefined); From 7ecd10082d26f8393f9e4efe952f5d224c0cf564 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 15 Dec 2022 15:56:07 +0100 Subject: [PATCH 102/137] Improve logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index cb33ec00ec8..cb7327c9ec3 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2375,7 +2375,7 @@ export class MatrixCall extends TypedEventEmitter { - logger.debug("Hangup received for call ID " + this.callId); + logger.debug(`Hangup received for call ID ${this.callId}: ${msg}`); // party ID must match (our chosen partner hanging up the call) or be undefined (we haven't chosen // a partner yet but we're treating the hangup as a reject as per VoIP v0) From 243e0404352d9b821d058fed267254a03d1b7972 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 15 Dec 2022 15:59:58 +0100 Subject: [PATCH 103/137] Fix log line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index cb7327c9ec3..5e0a401cc2e 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2375,7 +2375,7 @@ export class MatrixCall extends TypedEventEmitter { - logger.debug(`Hangup received for call ID ${this.callId}: ${msg}`); + logger.debug(`Hangup received for call ID ${this.callId}:`, msg); // party ID must match (our chosen partner hanging up the call) or be undefined (we haven't chosen // a partner yet but we're treating the hangup as a reject as per VoIP v0) From f7cb5b35a7295c40a9047cc4af199df7684bfd42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 16 Dec 2022 10:35:15 +0100 Subject: [PATCH 104/137] Mock SDP better MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- spec/test-utils/webrtc.ts | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index 61f703cb8be..2e94708e2b8 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -116,6 +116,12 @@ export class MockAudioContext { public close() {} } +export interface MockRTCSessionDescription { + sdp: string; + type: RTCSdpType; + toJSON: () => any; +} + export class MockRTCPeerConnection { private static instances: MockRTCPeerConnection[] = []; @@ -126,7 +132,7 @@ export class MockRTCPeerConnection { public needsNegotiation = false; public readyToNegotiate: Promise; private onReadyToNegotiate?: () => void; - public localDescription: RTCSessionDescription; + public localDescription: MockRTCSessionDescription; public signalingState: RTCSignalingState = "stable"; public iceConnectionState: RTCIceConnectionState = "connected"; public transceivers: MockRTCRtpTransceiver[] = []; @@ -149,7 +155,7 @@ export class MockRTCPeerConnection { this.localDescription = { sdp: DUMMY_SDP, type: "offer", - toJSON: function () {}, + toJSON: () => {}, }; this.readyToNegotiate = new Promise((resolve) => { @@ -195,23 +201,32 @@ export class MockRTCPeerConnection { public getStats() { return []; } - public addTransceiver(track: MockMediaStreamTrack): MockRTCRtpTransceiver { + public addTransceiver(track: MockMediaStreamTrack, init?: RTCRtpTransceiverInit): MockRTCRtpTransceiver { this.needsNegotiation = true; if (this.onReadyToNegotiate) this.onReadyToNegotiate(); const newSender = new MockRTCRtpSender(track); const newReceiver = new MockRTCRtpReceiver(track); - const newTransceiver = new MockRTCRtpTransceiver(this); + const newTransceiver = new MockRTCRtpTransceiver(this, (this.transceivers.length + 1).toString()); newTransceiver.sender = newSender as unknown as RTCRtpSender; newTransceiver.receiver = newReceiver as unknown as RTCRtpReceiver; this.transceivers.push(newTransceiver); + let newSDP = this.localDescription.sdp; + init?.streams?.forEach((stream) => { + newSDP += `m=${track.kind}\r\n`; + newSDP += `a=sendrecv\r\n`; + newSDP += `a=mid:${this.transceivers.length}\r\n`; + newSDP += `a=msid:${stream.id} ${track.id}\r\n`; + }); + this.localDescription.sdp = newSDP; + return newTransceiver; } - public addTrack(track: MockMediaStreamTrack): MockRTCRtpSender { - return this.addTransceiver(track).sender as unknown as MockRTCRtpSender; + public addTrack(track: MockMediaStreamTrack, ...streams: MediaStream[]): MockRTCRtpSender { + return this.addTransceiver(track, { streams }).sender as unknown as MockRTCRtpSender; } public removeTrack() { @@ -247,7 +262,7 @@ export class MockRTCRtpReceiver { } export class MockRTCRtpTransceiver { - constructor(private peerConn: MockRTCPeerConnection) {} + constructor(private peerConn: MockRTCPeerConnection, public readonly mid: string) {} public sender?: RTCRtpSender; public receiver?: RTCRtpReceiver; From 77a98bfea54b7fcb23cf76e5347ee84e1912b271 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 16 Dec 2022 10:38:07 +0100 Subject: [PATCH 105/137] Use `expect.objectContaining()` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- spec/unit/webrtc/call.spec.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index fa99ce414d8..9e0a3116c79 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -573,16 +573,16 @@ describe("Call", function () { await call.setMicrophoneMuted(true); expect((call as any).getLocalSDPStreamMetadata()).toStrictEqual({ - local_stream1: { + local_stream1: expect.objectContaining({ purpose: SDPStreamMetadataPurpose.Usermedia, audio_muted: true, video_muted: true, - }, - local_stream2: { + }), + local_stream2: expect.objectContaining({ purpose: SDPStreamMetadataPurpose.Screenshare, audio_muted: true, video_muted: false, - }, + }), }); }); @@ -834,11 +834,11 @@ describe("Call", function () { await call.setMicrophoneMuted(true); expect(mockSendVoipEvent).toHaveBeenCalledWith(EventType.CallSDPStreamMetadataChangedPrefix, { [SDPStreamMetadataKey]: { - mock_stream_from_media_handler: { + mock_stream_from_media_handler: expect.objectContaining({ purpose: SDPStreamMetadataPurpose.Usermedia, audio_muted: true, video_muted: false, - }, + }), }, }); }); @@ -847,11 +847,11 @@ describe("Call", function () { await call.setLocalVideoMuted(true); expect(mockSendVoipEvent).toHaveBeenCalledWith(EventType.CallSDPStreamMetadataChangedPrefix, { [SDPStreamMetadataKey]: { - mock_stream_from_media_handler: { + mock_stream_from_media_handler: expect.objectContaining({ purpose: SDPStreamMetadataPurpose.Usermedia, audio_muted: false, video_muted: true, - }, + }), }, }); }); From 11be9eef2c1d71f61fa691273cef136043bc9788 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 16 Dec 2022 10:47:01 +0100 Subject: [PATCH 106/137] Mock data channel better MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- spec/test-utils/webrtc.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index 2e94708e2b8..9749fbe3995 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -116,6 +116,16 @@ export class MockAudioContext { public close() {} } +export class MockRTCDataChannel { + constructor(public readonly label: string, public readonly id?: number) {} + + public typed(): RTCDataChannel { + return this as unknown as RTCDataChannel; + } + + addEventListener() {} +} + export interface MockRTCSessionDescription { sdp: string; type: RTCSdpType; @@ -176,8 +186,8 @@ export class MockRTCPeerConnection { this.onTrackListener = listener; } } - public createDataChannel(label: string, opts: RTCDataChannelInit) { - return { label, ...opts }; + public createDataChannel(label: string, opts: RTCDataChannelInit): MockRTCDataChannel { + return new MockRTCDataChannel(label, opts.id); } public createOffer() { return Promise.resolve({ From 3e2166818781c9f665f2eeeeb28fd6b3a204d1d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 16 Dec 2022 11:26:22 +0100 Subject: [PATCH 107/137] Mock `MatrixCall` better MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- spec/test-utils/webrtc.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index 9749fbe3995..5158e636758 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -554,6 +554,13 @@ export class MockMatrixCall extends TypedEventEmitter Date: Fri, 16 Dec 2022 11:32:57 +0100 Subject: [PATCH 108/137] Fix method call order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- spec/unit/webrtc/groupCall.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index 59cdbac129d..b9fe9dceb96 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -841,6 +841,7 @@ describe("Group Call", function () { // @ts-ignore const call = groupCall.calls.get(groupCall.room.getMember(FAKE_USER_ID_2)!)!.get(FAKE_DEVICE_ID_2)!; call.getOpponentMember = () => ({ userId: call.invitee } as RoomMember); + call.onSDPStreamMetadataChangedReceived(metadataEvent); // @ts-ignore Mock call.pushRemoteFeed( // @ts-ignore Mock @@ -849,7 +850,6 @@ describe("Group Call", function () { new MockMediaStreamTrack("video_track", "video"), ]), ); - call.onSDPStreamMetadataChangedReceived(metadataEvent); const feed = groupCall.getUserMediaFeed(call.invitee!, call.getOpponentDeviceId()!); expect(feed!.isAudioMuted()).toBe(true); @@ -868,6 +868,7 @@ describe("Group Call", function () { // @ts-ignore const call = groupCall.calls.get(groupCall.room.getMember(FAKE_USER_ID_2)!)!.get(FAKE_DEVICE_ID_2)!; call.getOpponentMember = () => ({ userId: call.invitee } as RoomMember); + call.onSDPStreamMetadataChangedReceived(metadataEvent); // @ts-ignore Mock call.pushRemoteFeed( // @ts-ignore Mock @@ -876,7 +877,6 @@ describe("Group Call", function () { new MockMediaStreamTrack("video_track", "video"), ]), ); - call.onSDPStreamMetadataChangedReceived(metadataEvent); const feed = groupCall.getUserMediaFeed(call.invitee!, call.getOpponentDeviceId()!); expect(feed!.isAudioMuted()).toBe(false); From 68c4dd51481b602fbeb480dc7052bd4d1f0bcb5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 16 Dec 2022 11:33:26 +0100 Subject: [PATCH 109/137] Disable some tests while we figure out what to do MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- spec/unit/webrtc/groupCall.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index b9fe9dceb96..6413c19c4e6 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -308,7 +308,8 @@ describe("Group Call", function () { } }); - describe("call feeds changing", () => { + // FIXME: Do we need the methods for replacing feeds + describe.skip("call feeds changing", () => { let call: MockMatrixCall; const currentFeed = new MockCallFeed(FAKE_USER_ID_1, FAKE_DEVICE_ID_1, new MockMediaStream("current")); const newFeed = new MockCallFeed(FAKE_USER_ID_1, FAKE_DEVICE_ID_1, new MockMediaStream("new")); From 6221523bf93e229b6fbd2c7483268d54326d99f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 16 Dec 2022 12:19:35 +0100 Subject: [PATCH 110/137] Re-add replace call feed logic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- spec/test-utils/webrtc.ts | 31 +++++++++++++++------ spec/unit/webrtc/groupCall.spec.ts | 35 +++++++++++++++--------- src/webrtc/groupCall.ts | 43 +++++++++++++++++++++++++++++- 3 files changed, 87 insertions(+), 22 deletions(-) diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index 5158e636758..0552b3f16c7 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -37,6 +37,7 @@ import { ReEmitter } from "../../src/ReEmitter"; import { SyncState } from "../../src/sync"; import { CallEvent, CallEventHandlerMap, CallState, MatrixCall } from "../../src/webrtc/call"; import { CallEventHandlerEvent, CallEventHandlerEventHandlerMap } from "../../src/webrtc/callEventHandler"; +import { SDPStreamMetadataPurpose } from "../../src/webrtc/callEventTypes"; import { CallFeed } from "../../src/webrtc/callFeed"; import { GroupCallEventHandlerMap } from "../../src/webrtc/groupCall"; import { GroupCallEventHandlerEvent } from "../../src/webrtc/groupCallEventHandler"; @@ -534,8 +535,7 @@ export class MockMatrixCall extends TypedEventEmitter(), stream: new MockMediaStream("stream"), }; - public remoteUsermediaFeed?: CallFeed; - public remoteScreensharingFeed?: CallFeed; + public feeds: CallFeed[] = []; public reject = jest.fn(); public answerWithCallFeeds = jest.fn(); @@ -546,6 +546,14 @@ export class MockMatrixCall extends TypedEventEmitter feed.purpose === SDPStreamMetadataPurpose.Usermedia); + } + + public get remoteScreensharingFeed(): CallFeed | undefined { + return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); + } + public getOpponentMember(): Partial { return this.opponentMember; } @@ -555,10 +563,7 @@ export class MockMatrixCall extends TypedEventEmitter !feed.isLocal()); } public typed(): MatrixCall { @@ -567,10 +572,20 @@ export class MockMatrixCall extends TypedEventEmitter boolean = jest.fn(); public typed(): CallFeed { return this as unknown as CallFeed; diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index 6413c19c4e6..c30e4791fa7 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -309,7 +309,7 @@ describe("Group Call", function () { }); // FIXME: Do we need the methods for replacing feeds - describe.skip("call feeds changing", () => { + describe("call feeds changing", () => { let call: MockMatrixCall; const currentFeed = new MockCallFeed(FAKE_USER_ID_1, FAKE_DEVICE_ID_1, new MockMediaStream("current")); const newFeed = new MockCallFeed(FAKE_USER_ID_1, FAKE_DEVICE_ID_1, new MockMediaStream("new")); @@ -317,6 +317,8 @@ describe("Group Call", function () { beforeEach(async () => { jest.spyOn(currentFeed, "dispose"); jest.spyOn(newFeed, "measureVolumeActivity"); + jest.spyOn(currentFeed, "isLocal").mockReturnValue(false); + jest.spyOn(newFeed, "isLocal").mockReturnValue(false); jest.spyOn(groupCall, "emit"); @@ -325,17 +327,14 @@ describe("Group Call", function () { await groupCall.create(); }); - it("ignores changes, if we can't get user id of opponent", async () => { - const call = new MockMatrixCall(room.roomId, groupCall.groupCallId); - jest.spyOn(call, "getOpponentMember").mockReturnValue({ userId: undefined }); - - // @ts-ignore Mock - expect(() => groupCall.onCallFeedsChanged(call)).toThrowError(); - }); - describe("usermedia feeds", () => { + beforeEach(() => { + currentFeed.purpose = SDPStreamMetadataPurpose.Usermedia; + newFeed.purpose = SDPStreamMetadataPurpose.Usermedia; + }); + it("adds new usermedia feed", async () => { - call.remoteUsermediaFeed = newFeed.typed(); + call.feeds = [newFeed.typed()]; // @ts-ignore Mock groupCall.onCallFeedsChanged(call); @@ -345,7 +344,8 @@ describe("Group Call", function () { it("replaces usermedia feed", async () => { groupCall.userMediaFeeds.push(currentFeed.typed()); - call.remoteUsermediaFeed = newFeed.typed(); + call.feeds = [newFeed.typed()]; + // @ts-ignore Mock groupCall.onCallFeedsChanged(call); @@ -353,6 +353,7 @@ describe("Group Call", function () { }); it("removes usermedia feed", async () => { + currentFeed.dispose(); groupCall.userMediaFeeds.push(currentFeed.typed()); // @ts-ignore Mock @@ -363,8 +364,14 @@ describe("Group Call", function () { }); describe("screenshare feeds", () => { + beforeEach(() => { + currentFeed.purpose = SDPStreamMetadataPurpose.Screenshare; + newFeed.purpose = SDPStreamMetadataPurpose.Screenshare; + }); + it("adds new screenshare feed", async () => { - call.remoteScreensharingFeed = newFeed.typed(); + call.feeds = [newFeed.typed()]; + // @ts-ignore Mock groupCall.onCallFeedsChanged(call); @@ -374,7 +381,8 @@ describe("Group Call", function () { it("replaces screenshare feed", async () => { groupCall.screenshareFeeds.push(currentFeed.typed()); - call.remoteScreensharingFeed = newFeed.typed(); + call.feeds = [newFeed.typed()]; + // @ts-ignore Mock groupCall.onCallFeedsChanged(call); @@ -382,6 +390,7 @@ describe("Group Call", function () { }); it("removes screenshare feed", async () => { + currentFeed.dispose(); groupCall.screenshareFeeds.push(currentFeed.typed()); // @ts-ignore Mock diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index a3ba8099e20..727a65c200a 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -1060,6 +1060,16 @@ export class GroupCall extends TypedEventEmitter< } private onCallFeedsChanged = (call: MatrixCall): void => { + // Find replaced feeds + call.getRemoteFeeds().filter((cf) => { + [...this.userMediaFeeds, ...this.screenshareFeeds].forEach((gf) => { + if (gf !== cf && gf.userId === cf.userId && gf.deviceId === cf.deviceId && gf.purpose === cf.purpose) { + if (cf.purpose === SDPStreamMetadataPurpose.Usermedia) this.replaceUserMediaFeed(gf, cf); + else if (cf.purpose === SDPStreamMetadataPurpose.Screenshare) this.replaceScreenshareFeed(gf, cf); + } + }); + }); + // Find removed feeds [...this.userMediaFeeds, ...this.screenshareFeeds] .filter((gf) => gf.disposed) @@ -1071,7 +1081,7 @@ export class GroupCall extends TypedEventEmitter< // Find new feeds call.getRemoteFeeds() .filter((cf) => { - return !this.userMediaFeeds.find((gf) => gf.stream.id === cf.stream.id); + return !this.userMediaFeeds.find((gf) => gf === cf) && !this.screenshareFeeds.find((gf) => gf === cf); }) .forEach((feed) => { if (feed.purpose === SDPStreamMetadataPurpose.Usermedia) this.addUserMediaFeed(feed); @@ -1149,6 +1159,22 @@ export class GroupCall extends TypedEventEmitter< this.emit(GroupCallEvent.UserMediaFeedsChanged, this.userMediaFeeds); } + private replaceUserMediaFeed(existingFeed: CallFeed, replacementFeed: CallFeed): void { + const feedIndex = this.userMediaFeeds.findIndex( + (f) => f.userId === existingFeed.userId && f.deviceId! === existingFeed.deviceId, + ); + + if (feedIndex === -1) { + throw new Error("Couldn't find user media feed to replace"); + } + + this.userMediaFeeds.splice(feedIndex, 1, replacementFeed); + + existingFeed.dispose(); + replacementFeed.measureVolumeActivity(true); + this.emit(GroupCallEvent.UserMediaFeedsChanged, this.userMediaFeeds); + } + private removeUserMediaFeed(callFeed: CallFeed): void { const feedIndex = this.userMediaFeeds.findIndex( (f) => f.userId === callFeed.userId && f.deviceId! === callFeed.deviceId, @@ -1206,6 +1232,21 @@ export class GroupCall extends TypedEventEmitter< this.emit(GroupCallEvent.ScreenshareFeedsChanged, this.screenshareFeeds); } + private replaceScreenshareFeed(existingFeed: CallFeed, replacementFeed: CallFeed): void { + const feedIndex = this.screenshareFeeds.findIndex( + (f) => f.userId === existingFeed.userId && f.deviceId! === existingFeed.deviceId, + ); + + if (feedIndex === -1) { + throw new Error("Couldn't find screenshare feed to replace"); + } + + this.screenshareFeeds.splice(feedIndex, 1, replacementFeed); + + existingFeed.dispose(); + this.emit(GroupCallEvent.ScreenshareFeedsChanged, this.screenshareFeeds); + } + private removeScreenshareFeed(callFeed: CallFeed): void { const feedIndex = this.screenshareFeeds.findIndex( (f) => f.userId === callFeed.userId && f.deviceId! === callFeed.deviceId, From 16fed17a7ded4f696c7677f76f20696ef78ac81f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 19 Dec 2022 13:51:59 +0100 Subject: [PATCH 111/137] Use `streamId` from SDP directly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 5e0a401cc2e..1ba1cda0c49 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -614,19 +614,32 @@ export class MatrixCall extends TypedEventEmitter { + // trackIds and streamIds which the focus will see which will + // probably differ from the local trackIds and streamIds + let streamId = localFeed.sdpMetadataStreamId; + const tracks = Array.from(this.transceivers.entries()).reduce((tracks, [transceiverKey, transceiver]) => { + if ( + ![ + getTransceiverKey(localFeed.purpose, "audio"), + getTransceiverKey(localFeed.purpose, "video"), + ].includes(transceiverKey) + ) { + return tracks; + } + // XXX: We only use double equals because MediaDescription::mid is in fact a number - const trackId = sdp?.media?.find((m) => m.mid == transceiver.mid)?.msid?.split(" ")?.[1]; - if (trackId) { - tracks[trackId] = {}; + const msid = sdp?.media?.find((m) => m.mid == transceiver.mid)?.msid?.split(" "); + if (msid?.[0]) { + streamId = msid?.[0]; + } + if (msid?.[1]) { + tracks[msid?.[1]] = {}; } return tracks; }, {} as SDPStreamMetadataTracks); if (!Object.keys(tracks).length) continue; - metadata[localFeed.sdpMetadataStreamId] = { + metadata[streamId] = { // FIXME: This allows for impersonation - the focus should be // handling these user_id: this.client.getUserId()!, From 7783b246d169427eb162c70c3ee2a0961c970bd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 19 Dec 2022 14:18:07 +0100 Subject: [PATCH 112/137] Improve code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 727a65c200a..96e22f0fc6d 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -1081,7 +1081,7 @@ export class GroupCall extends TypedEventEmitter< // Find new feeds call.getRemoteFeeds() .filter((cf) => { - return !this.userMediaFeeds.find((gf) => gf === cf) && !this.screenshareFeeds.find((gf) => gf === cf); + return ![...this.userMediaFeeds, ...this.screenshareFeeds].find((gf) => gf === cf); }) .forEach((feed) => { if (feed.purpose === SDPStreamMetadataPurpose.Usermedia) this.addUserMediaFeed(feed); From d48d9821c2fb8313bd283bb2d7496b82c85bfc25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 19 Dec 2022 14:51:21 +0100 Subject: [PATCH 113/137] Clear feeds in `dispose()` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 96e22f0fc6d..2af79778568 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -536,6 +536,9 @@ export class GroupCall extends TypedEventEmitter< this.localDesktopCapturerSourceId = undefined; } + this.userMediaFeeds.splice(0, this.userMediaFeeds.length); + this.screenshareFeeds.splice(0, this.screenshareFeeds.length); + this.client.getMediaHandler().stopAllStreams(); if (this.transmitTimer !== null) { From 378cf7922e6710c534315db25485eefb42625ab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 6 Jan 2023 17:50:00 +0100 Subject: [PATCH 114/137] Make `onCallHangup` work with focus calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes https://github.com/vector-im/element-call/issues/830 Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index b88c6f0dc44..74650cbc010 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -1115,7 +1115,9 @@ export class GroupCall extends TypedEventEmitter< private onCallHangup = (call: MatrixCall): void => { if (call.hangupReason === CallErrorCode.Replaced) return; - const opponentUserId = call.getOpponentMember()?.userId ?? this.room.getMember(call.invitee!)!.userId; + const opponentUserId = call.invitee ?? call.getOpponentMember()?.userId; + if (!opponentUserId) return; + const deviceMap = this.calls.get(opponentUserId); // Sanity check that this call is in fact in the map From 451ebfde4b5cf5e7d624c9bcfa0df56763d4976e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 25 Jan 2023 17:07:31 +0100 Subject: [PATCH 115/137] Add support for simulcast (#2983) --- spec/test-utils/webrtc.ts | 6 +- spec/unit/webrtc/call.spec.ts | 85 +++--- spec/unit/webrtc/callFeed.spec.ts | 2 + spec/unit/webrtc/groupCall.spec.ts | 15 +- src/webrtc/call.ts | 399 +++++++++++++++++++---------- src/webrtc/callEventTypes.ts | 8 +- src/webrtc/callFeed.ts | 138 +++++++--- src/webrtc/groupCall.ts | 41 ++- src/webrtc/mediaHandler.ts | 4 +- 9 files changed, 463 insertions(+), 235 deletions(-) diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index 735555a6c80..1cbf0e598cd 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -266,6 +266,9 @@ export class MockRTCRtpSender { public replaceTrack(track: MockMediaStreamTrack) { this.track = track; } + + public getParameters() {} + public setParameters() {} } export class MockRTCRtpReceiver { @@ -292,7 +295,7 @@ export class MockMediaStreamTrack { public listeners: [string, (...args: any[]) => any][] = []; public isStopped = false; - public settings?: MediaTrackSettings; + public settings: MediaTrackSettings = {}; public getSettings(): MediaTrackSettings { return this.settings!; @@ -592,6 +595,7 @@ export class MockCallFeed { export function installWebRTCMocks() { global.navigator = { mediaDevices: new MockMediaDevices().typed(), + userAgent: "This is definitely a user agent string", } as unknown as Navigator; global.window = { diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 9e0a3116c79..ae6d3eb7c00 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -29,7 +29,6 @@ import { import { MCallAnswer, MCallHangupReject, - SDPStreamMetadata, SDPStreamMetadataKey, SDPStreamMetadataPurpose, } from "../../../src/webrtc/callEventTypes"; @@ -83,6 +82,7 @@ const fakeIncomingCall = async (client: TestClient, call: MatrixCall, version: s client: client.client, userId: "remote_user_id", deviceId: undefined, + feedId: "remote_stream_id", stream: new MockMediaStream("remote_stream_id", [ new MockMediaStreamTrack("remote_tack_id", "audio"), ]) as unknown as MediaStream, @@ -316,13 +316,13 @@ describe("Call", function () { }), ); - (call as any).pushRemoteFeed( + (call as any).pushRemoteStream( new MockMediaStream("remote_stream", [ new MockMediaStreamTrack("remote_audio_track", "audio"), new MockMediaStreamTrack("remote_video_track", "video"), ]), ); - const feed = call.getFeeds().find((feed) => feed.stream.id === "remote_stream"); + const feed = call.getFeeds().find((feed) => feed.stream?.id === "remote_stream"); expect(feed?.purpose).toBe(SDPStreamMetadataPurpose.Usermedia); expect(feed?.isAudioMuted()).toBeTruthy(); expect(feed?.isVideoMuted()).not.toBeTruthy(); @@ -439,8 +439,8 @@ describe("Call", function () { video_muted: false, }, }); - (call as any).pushRemoteFeed(new MockMediaStream("remote_stream", [])); - const feed = call.getFeeds().find((feed) => feed.stream.id === "remote_stream"); + (call as any).pushRemoteStream(new MockMediaStream("remote_stream", [])); + const feed = call.getFeeds().find((feed) => feed.stream?.id === "remote_stream"); call.onSDPStreamMetadataChangedReceived( makeMockEvent("@test:foo", { @@ -520,14 +520,14 @@ describe("Call", function () { it("if no video", async () => { call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); - (call as any).pushRemoteFeed(new MockMediaStream("remote_stream1", [])); + (call as any).pushRemoteStream(new MockMediaStream("remote_stream1", [])); expect(call.type).toBe(CallType.Voice); }); it("if remote video", async () => { call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); - (call as any).pushRemoteFeed( + (call as any).pushRemoteStream( new MockMediaStream("remote_stream1", [new MockMediaStreamTrack("track_id", "video")]), ); expect(call.type).toBe(CallType.Video); @@ -555,6 +555,7 @@ describe("Call", function () { roomId: call.roomId, userId: client.getUserId(), deviceId: undefined, + feedId: "local_stream1", purpose: SDPStreamMetadataPurpose.Usermedia, audioMuted: false, videoMuted: false, @@ -597,6 +598,7 @@ describe("Call", function () { client: client.client, userId: client.getUserId(), deviceId: undefined, + feedId: localUsermediaStream.id, stream: localUsermediaStream as unknown as MediaStream, purpose: SDPStreamMetadataPurpose.Usermedia, audioMuted: false, @@ -606,6 +608,7 @@ describe("Call", function () { client: client.client, userId: client.getUserId(), deviceId: undefined, + feedId: localScreensharingStream.id, stream: localScreensharingStream as unknown as MediaStream, purpose: SDPStreamMetadataPurpose.Screenshare, audioMuted: false, @@ -629,8 +632,8 @@ describe("Call", function () { video_muted: false, }, }); - (call as any).pushRemoteFeed(remoteUsermediaStream); - (call as any).pushRemoteFeed(remoteScreensharingStream); + (call as any).pushRemoteStream(remoteUsermediaStream); + (call as any).pushRemoteStream(remoteScreensharingStream); expect(call.localUsermediaFeed!.stream).toBe(localUsermediaStream); expect(call.localUsermediaStream).toBe(localUsermediaStream); @@ -762,7 +765,7 @@ describe("Call", function () { call.off(CallEvent.FeedsChanged, FEEDS_CHANGED_CALLBACK); }); - it("should ignore stream passed to pushRemoteFeed()", async () => { + it("should ignore stream passed to pushRemoteStream()", async () => { await call.onAnswerReceived( makeMockEvent("@test:foo", { version: 1, @@ -779,16 +782,16 @@ describe("Call", function () { }), ); - (call as any).pushRemoteFeed(new MockMediaStream(STREAM_ID)); - (call as any).pushRemoteFeed(new MockMediaStream(STREAM_ID)); + (call as any).pushRemoteStream(new MockMediaStream(STREAM_ID)); + (call as any).pushRemoteStream(new MockMediaStream(STREAM_ID)); expect(call.getRemoteFeeds().length).toBe(1); expect(FEEDS_CHANGED_CALLBACK).toHaveBeenCalledTimes(1); }); - it("should ignore stream passed to pushRemoteFeedWithoutMetadata()", async () => { - (call as any).pushRemoteFeedWithoutMetadata(new MockMediaStream(STREAM_ID)); - (call as any).pushRemoteFeedWithoutMetadata(new MockMediaStream(STREAM_ID)); + it("should ignore stream passed to pushRemoteStreamWithoutMetadata()", async () => { + (call as any).pushRemoteStreamWithoutMetadata(new MockMediaStream(STREAM_ID)); + (call as any).pushRemoteStreamWithoutMetadata(new MockMediaStream(STREAM_ID)); expect(call.getRemoteFeeds().length).toBe(1); expect(FEEDS_CHANGED_CALLBACK).toHaveBeenCalledTimes(1); @@ -858,18 +861,8 @@ describe("Call", function () { }); describe("receiving sdp_stream_metadata_changed events", () => { - const setupCall = (audio: boolean, video: boolean): SDPStreamMetadata => { - const metadata = { - stream: { - user_id: "user", - device_id: "device", - purpose: SDPStreamMetadataPurpose.Usermedia, - audio_muted: audio, - video_muted: video, - tracks: {}, - }, - }; - (call as any).pushRemoteFeed( + const setupCall = (audio: boolean, video: boolean): void => { + (call as any).pushRemoteStream( new MockMediaStream("stream", [ new MockMediaStreamTrack("track1", "audio"), new MockMediaStreamTrack("track1", "video"), @@ -877,22 +870,30 @@ describe("Call", function () { ); call.onSDPStreamMetadataChangedReceived({ getContent: () => ({ - [SDPStreamMetadataKey]: metadata, + [SDPStreamMetadataKey]: { + stream: { + user_id: "user", + device_id: "device", + purpose: SDPStreamMetadataPurpose.Usermedia, + audio_muted: audio, + video_muted: video, + tracks: {}, + }, + }, }), } as MatrixEvent); - return metadata; }; it("should handle incoming sdp_stream_metadata_changed with audio muted", async () => { - const metadata = setupCall(true, false); - expect((call as any).remoteSDPStreamMetadata).toStrictEqual(metadata); + setupCall(true, false); + expect(call.opponentSupportsSDPStreamMetadata()).toBe(true); expect(call.getRemoteFeeds()[0].isAudioMuted()).toBe(true); expect(call.getRemoteFeeds()[0].isVideoMuted()).toBe(false); }); it("should handle incoming sdp_stream_metadata_changed with video muted", async () => { - const metadata = setupCall(false, true); - expect((call as any).remoteSDPStreamMetadata).toStrictEqual(metadata); + setupCall(false, true); + expect(call.opponentSupportsSDPStreamMetadata()).toBe(true); expect(call.getRemoteFeeds()[0].isAudioMuted()).toBe(false); expect(call.getRemoteFeeds()[0].isVideoMuted()).toBe(true); }); @@ -1394,8 +1395,8 @@ describe("Call", function () { describe("onTrack", () => { it("ignores streamless track", async () => { - // @ts-ignore Mock pushRemoteFeed() is private - jest.spyOn(call, "pushRemoteFeed"); + // @ts-ignore Mock pushRemoteStream() is private + jest.spyOn(call, "pushRemoteStream"); await call.placeVoiceCall(); @@ -1404,13 +1405,13 @@ describe("Call", function () { track: new MockMediaStreamTrack("track_ev", "audio"), } as unknown as RTCTrackEvent); - // @ts-ignore Mock pushRemoteFeed() is private - expect(call.pushRemoteFeed).not.toHaveBeenCalled(); + // @ts-ignore Mock pushRemoteStream() is private + expect(call.pushRemoteStream).not.toHaveBeenCalled(); }); it("correctly pushes", async () => { - // @ts-ignore Mock pushRemoteFeed() is private - jest.spyOn(call, "pushRemoteFeed"); + // @ts-ignore Mock pushRemoteStream() is private + jest.spyOn(call, "pushRemoteStream"); await call.placeVoiceCall(); await call.onAnswerReceived( @@ -1430,9 +1431,9 @@ describe("Call", function () { track: stream.getAudioTracks()[0], } as unknown as RTCTrackEvent); - // @ts-ignore Mock pushRemoteFeed() is private - expect(call.pushRemoteFeed).toHaveBeenCalledWith(stream); - // @ts-ignore Mock pushRemoteFeed() is private + // @ts-ignore Mock pushRemoteStream() is private + expect(call.pushRemoteStream).toHaveBeenCalledWith(stream); + // @ts-ignore Mock pushRemoteStream() is private expect(call.removeTrackListeners.has(stream)).toBe(true); }); }); diff --git a/spec/unit/webrtc/callFeed.spec.ts b/spec/unit/webrtc/callFeed.spec.ts index e14a1a0c56b..c648f7aa066 100644 --- a/spec/unit/webrtc/callFeed.spec.ts +++ b/spec/unit/webrtc/callFeed.spec.ts @@ -102,6 +102,8 @@ describe("CallFeed", () => { [CallState.Connected, true], [CallState.Connecting, false], ])("should react to call state, when !isLocal()", (state: CallState, expected: Boolean) => { + feed.stream?.addTrack(new MockMediaStreamTrack("track1", "video").typed()); + call.state = state; call.emit(CallEvent.State, state); expect(feed.connected).toBe(expected); diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index 2cd1f4cdabe..8f271a24109 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -807,7 +807,7 @@ describe("Group Call", function () { await groupCall.setMicrophoneMuted(true); - groupCall.localCallFeed!.stream.getAudioTracks().forEach((track) => expect(track.enabled).toBe(false)); + groupCall.localCallFeed!.stream!.getAudioTracks().forEach((track) => expect(track.enabled).toBe(false)); expect(groupCall.localCallFeed!.setAudioVideoMuted).toHaveBeenCalledWith(true, null); setAVMutedArray.forEach((f) => expect(f).toHaveBeenCalledWith(true, null)); tracksArray.forEach((track) => expect(track.enabled).toBe(false)); @@ -835,9 +835,8 @@ describe("Group Call", function () { await groupCall.setLocalVideoMuted(true); - groupCall.localCallFeed!.stream.getVideoTracks().forEach((track) => expect(track.enabled).toBe(false)); - expect(mockClient.getMediaHandler().getUserMediaStream).toHaveBeenCalledWith(true, false); - expect(groupCall.updateLocalUsermediaStream).toHaveBeenCalled(); + groupCall.localCallFeed!.stream!.getVideoTracks().forEach((track) => expect(track.enabled).toBe(false)); + expect(groupCall.localCallFeed!.setAudioVideoMuted).toHaveBeenCalledWith(null, true); setAVMutedArray.forEach((f) => expect(f).toHaveBeenCalledWith(null, true)); tracksArray.forEach((track) => expect(track.enabled).toBe(false)); sendMetadataUpdateArray.forEach((f) => expect(f).toHaveBeenCalled()); @@ -872,7 +871,7 @@ describe("Group Call", function () { call.getOpponentMember = () => ({ userId: call.invitee } as RoomMember); call.onSDPStreamMetadataChangedReceived(metadataEvent); // @ts-ignore Mock - call.pushRemoteFeed( + call.pushRemoteStream( // @ts-ignore Mock new MockMediaStream("stream", [ new MockMediaStreamTrack("audio_track", "audio"), @@ -899,7 +898,7 @@ describe("Group Call", function () { call.getOpponentMember = () => ({ userId: call.invitee } as RoomMember); call.onSDPStreamMetadataChangedReceived(metadataEvent); // @ts-ignore Mock - call.pushRemoteFeed( + call.pushRemoteStream( // @ts-ignore Mock new MockMediaStream("stream", [ new MockMediaStreamTrack("audio_track", "audio"), @@ -1157,7 +1156,7 @@ describe("Group Call", function () { }), } as MatrixEvent); // @ts-ignore Mock - call.pushRemoteFeed( + call.pushRemoteStream( // @ts-ignore Mock new MockMediaStream("screensharing_stream", [new MockMediaStreamTrack("video_track", "video")]), ); @@ -1211,6 +1210,7 @@ describe("Group Call", function () { roomId: FAKE_ROOM_ID, userId: FAKE_USER_ID_2, deviceId: FAKE_DEVICE_ID_1, + feedId: "foo", stream: new MockMediaStream("foo", []).typed(), purpose: SDPStreamMetadataPurpose.Usermedia, audioMuted: false, @@ -1223,6 +1223,7 @@ describe("Group Call", function () { roomId: FAKE_ROOM_ID, userId: FAKE_USER_ID_3, deviceId: FAKE_DEVICE_ID_1, + feedId: "foo", stream: new MockMediaStream("foo", []).typed(), purpose: SDPStreamMetadataPurpose.Usermedia, audioMuted: false, diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index edcbebe3866..6da3b36d8c9 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -53,7 +53,7 @@ import { SDPStreamMetadataKeyStable, FocusEventBaseContent, } from "./callEventTypes"; -import { CallFeed } from "./callFeed"; +import { CallFeed, CallFeedEvent } from "./callFeed"; import { MatrixClient } from "../client"; import { EventEmitterEvents, TypedEventEmitter } from "../models/typed-event-emitter"; import { DeviceInfo } from "../crypto/deviceinfo"; @@ -93,6 +93,12 @@ interface AssertedIdentity { displayName: string; } +export enum SimulcastResolution { + Full = "f", + Half = "h", + Quarter = "q", +} + enum MediaType { AUDIO = "audio", VIDEO = "video", @@ -267,6 +273,31 @@ const CALL_TIMEOUT_MS = 60 * 1000; // ms const CALL_LENGTH_INTERVAL = 1000; // ms /** The time after which we end the call, if ICE got disconnected */ const ICE_DISCONNECTED_TIMEOUT = 30 * 1000; // ms +/** + * The time we wait for call feed size and visibility changing before we send a + * new m.call.track_subscription + */ +const SUBSCRIBE_TO_FOCUS_TIMEOUT = 2 * 1000; + +const SIMULCAST_ENCODINGS = [ + // Order is important here: some browsers (e.g. + // Chrome) will only send some of the encodings, if + // the track has a resolution to low for it to send + // all, in that case the encoding higher in the list + // has priority and therefore we put full as first + // as we always want to send the full resolution + { + rid: SimulcastResolution.Full, + }, + { + rid: SimulcastResolution.Half, + scaleResolutionDownBy: 2.0, + }, + { + rid: SimulcastResolution.Quarter, + scaleResolutionDownBy: 4.0, + }, +]; export class CallError extends Error { public readonly code: string; @@ -296,6 +327,10 @@ function getCodecParamMods(isPtt: boolean): CodecParamsMod[] { return mods; } +function isFirefox(): boolean { + return navigator.userAgent.indexOf("Firefox") !== -1; +} + export type CallEventHandlerMap = { [CallEvent.DataChannel]: (channel: RTCDataChannel) => void; [CallEvent.FeedsChanged]: (feeds: CallFeed[]) => void; @@ -352,7 +387,6 @@ export class MatrixCall extends TypedEventEmitter(); - private subscribedTracks: FocusTrackDescription[] = []; private inviteOrAnswerSent = false; private waitForLocalAVStream = false; @@ -388,7 +422,6 @@ export class MatrixCall extends TypedEventEmitter(); private remoteAssertedIdentity?: AssertedIdentity; - private remoteSDPStreamMetadata?: SDPStreamMetadata; private callLengthInterval?: ReturnType; private callStartTime?: number; @@ -400,6 +433,10 @@ export class MatrixCall extends TypedEventEmitter; + + private _opponentSupportsSDPStreamMetadata = false; + /** * Construct a new Matrix Call. * @param opts - Config options. @@ -549,8 +586,8 @@ export class MatrixCall extends TypedEventEmitter feed.stream.id === streamId); + private getFeedById(streamId: string): CallFeed | undefined { + return this.getFeeds().find((feed) => feed.feedId === streamId); } /** @@ -605,19 +642,16 @@ export class MatrixCall extends TypedEventEmitter { + if (!transceiver.sender.track) return tracks; if ( ![ getTransceiverKey(localFeed.purpose, "audio"), @@ -633,7 +667,11 @@ export class MatrixCall extends TypedEventEmitter !feed.isLocal()); } - private pushRemoteFeed(stream: MediaStream): void { + private pushRemoteStream(stream: MediaStream): void { // Fallback to old behavior if the other side doesn't support SDPStreamMetadata - const metadata = this.remoteSDPStreamMetadata?.[stream.id]; - if (!this.opponentSupportsSDPStreamMetadata() || !metadata) { - this.pushRemoteFeedWithoutMetadata(stream); + if (!this.opponentSupportsSDPStreamMetadata()) { + this.pushRemoteStreamWithoutMetadata(stream); return; } - // If we're calling with a focus we trust its metadata, otherwise we - // only trust ourselves to avoid impersonation - const userId = this.isFocus ? metadata.user_id : this.getOpponentMember()!.userId; - const deviceId = this.isFocus ? metadata.device_id : this.getOpponentDeviceId()!; - const purpose = metadata.purpose; - const audioMuted = metadata.audio_muted; - const videoMuted = metadata.video_muted; - - if (!purpose) { - logger.warn( - `Call ${this.callId} Ignoring stream with id ${stream.id} because we didn't get any metadata about it`, - ); + const feed = this.getFeedById(stream.id); + if (!feed) { + logger.warn(`Ignoring stream with id ${stream.id} because we don't have a feed for it`); return; } - if (this.getFeedByStreamId(stream.id)) { - logger.warn(`Ignoring stream with id ${stream.id} because we already have a feed for it`); - return; - } - - this.feeds.push( - new CallFeed({ - client: this.client, - call: this, - roomId: this.roomId, - userId, - deviceId, - stream, - purpose, - audioMuted, - videoMuted, - }), - ); - - this.emit(CallEvent.FeedsChanged, this.feeds); + feed.setNewStream(stream); logger.info( `Call ${this.callId} Pushed remote stream (` + - `id="${stream.id}" ` + + `id="${feed.feedId}" ` + `active="${stream.active}" ` + - `purpose=${purpose} ` + - `userId=${userId}` + - `deviceId=${deviceId}` + + `purpose=${feed.purpose} ` + + `userId=${feed.userId}` + + `deviceId=${feed.deviceId}` + `)`, ); } @@ -721,7 +730,7 @@ export class MatrixCall extends TypedEventEmitter callFeed.stream.id === feed.stream.id)) { - logger.info(`Ignoring duplicate local stream ${callFeed.stream.id} in call ${this.callId}`); + if (!callFeed.stream) { + logger.warn(`Ignoring stream-less local feed ${callFeed.feedId} in call ${this.callId}`); + return; + } + + if (this.feeds.some((feed) => callFeed.feedId === feed.feedId)) { + logger.info(`Ignoring duplicate local stream ${callFeed.feedId} in call ${this.callId}`); return; } this.feeds.push(callFeed); if (addToPeerConnection) { - for (const track of callFeed.stream.getTracks()) { + for (const track of callFeed.stream!.getTracks()) { logger.info( `Call ${this.callId} ` + `Adding track (` + `id="${track.id}", ` + `kind="${track.kind}", ` + - `streamId="${callFeed.stream.id}", ` + + `streamId="${callFeed.feedId}", ` + `streamPurpose="${callFeed.purpose}", ` + `enabled=${track.enabled}` + `) to peer connection`, ); + const encodings = track.kind === "video" ? SIMULCAST_ENCODINGS : undefined; + const tKey = getTransceiverKey(callFeed.purpose, track.kind); if (this.transceivers.has(tKey)) { // we already have a sender, so we re-use it. We try to re-use transceivers as much @@ -826,28 +844,39 @@ export class MatrixCall extends TypedEventEmitter t.sender === newSender); - if (newTransciever) { - this.transceivers.set(tKey, newTransciever); - } else { - logger.warn("Didn't find a matching transceiver after adding track!"); + // create a new one + const transceiver = this.peerConn!.addTransceiver(track, { + streams: [callFeed.stream!], + sendEncodings: this.isFocus && isFirefox() ? undefined : encodings, + }); + + if (this.isFocus && isFirefox()) { + const parameters = transceiver.sender.getParameters(); + transceiver.sender.setParameters({ + ...transceiver.sender.getParameters(), + encodings: encodings ?? parameters.encodings, + }); } + + this.transceivers.set(tKey, transceiver); } } } @@ -855,8 +884,8 @@ export class MatrixCall extends TypedEventEmitter t.sender === newSender); - if (newTransciever) { - this.transceivers.set(tKey, newTransciever); - } else { - logger.warn("Couldn't find matching transceiver for newly added track!"); + const newTransceiver = this.peerConn!.addTransceiver(track, { + streams: [this.localUsermediaStream!], + sendEncodings: this.isFocus && isFirefox() ? undefined : encodings, + }); + + if (this.isFocus && isFirefox()) { + const parameters = newTransceiver.sender.getParameters(); + newTransceiver.sender.setParameters({ + ...newTransceiver.sender.getParameters(), + encodings: encodings ?? parameters.encodings, + }); } + + this.transceivers.set(tKey, newTransceiver); } } } @@ -1602,7 +1672,7 @@ export class MatrixCall extends TypedEventEmitter Boolean(info.tracks)) // Skip trackless feeds - .reduce( - (a: FocusTrackDescription[], [s, i]) => [ - ...a, - ...Object.keys(i.tracks).map((t) => ({ stream_id: s, track_id: t })), - ], - [], - ) // Get array of tracks from feeds - .filter((track) => !this.subscribedTracks.find((subscribed) => utils.deepCompare(track, subscribed))); // Filter out already subscribed tracks + clearTimeout(this.subscribeToFocusTimeout); + this.subscribeToFocusTimeout = setTimeout(() => { + this.sendSubscriptionFocusEvent(); + }, SUBSCRIBE_TO_FOCUS_TIMEOUT); + } - if (tracks.length === 0) { - logger.info("Failed to find any new streams to subscribe to"); - return; - } else { - this.subscribedTracks.push(...tracks); + /** + * This method should only ever be called by MatrixCall::subscribeToFocus()! + */ + private sendSubscriptionFocusEvent(): void { + const subscribe: FocusTrackDescription[] = []; + const unsubscribe: FocusTrackDescription[] = []; + for (const { feedId, tracksMetadata, isVisible, width, height } of this.getRemoteFeeds()) { + for (const [trackId, trackMetadata] of Object.entries(tracksMetadata)) { + const trackDescription: FocusTrackDescription = { + track_id: trackId, + stream_id: feedId, + }; + + if (trackMetadata.kind === "audio") { + // We want audio from everyone + subscribe.push(trackDescription); + } else if (isVisible && width !== 0 && height !== 0) { + // Subscribe to visible videos + trackDescription.width = width; + trackDescription.height = height; + + subscribe.push(trackDescription); + } else { + // Unsubscribe from invisible videos + unsubscribe.push(trackDescription); + } + } } + // Return, if there is nothing to do + if (subscribe.length === 0 && unsubscribe.length === 0) return; + + // TODO: Is it ok to keep re-requesting tracks this.sendFocusEvent(EventType.CallTrackSubscription, { - subscribe: tracks, - unsubscribe: [], + subscribe, + unsubscribe, } as FocusTrackSubscriptionEvent); } + private onCallFeedSizeChanged = async (): Promise => { + this.subscribeToFocus(); + }; + public updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { - if (!metadata) return; - this.remoteSDPStreamMetadata = utils.recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); + this._opponentSupportsSDPStreamMetadata = true; + + let feedsChanged = false; + + // Add new feeds and update existing ones + for (const [streamId, streamMetadata] of Object.entries(metadata)) { + let feed = this.getRemoteFeeds().find((f) => f.feedId === streamId); + if (feed) { + feed.purpose = streamMetadata.purpose; + feed.tracksMetadata = streamMetadata.tracks; + } else { + feed = new CallFeed({ + client: this.client, + call: this, + roomId: this.roomId, + userId: this.isFocus ? streamMetadata.user_id : this.getOpponentMember()!.userId, + deviceId: this.isFocus ? streamMetadata.device_id : this.getOpponentDeviceId()!, + feedId: streamId, + purpose: streamMetadata.purpose, + audioMuted: streamMetadata.audio_muted, + videoMuted: streamMetadata.video_muted, + tracksMetadata: streamMetadata.tracks, + }); + this.addRemoteFeed(feed, false); + feedsChanged = true; + } + feed.setAudioVideoMuted(streamMetadata.audio_muted, streamMetadata.video_muted); + } + + // Remove old feeds for (const feed of this.getRemoteFeeds()) { - const streamId = feed.stream.id; - const metadata = this.remoteSDPStreamMetadata![streamId]; + if (!Object.keys(metadata).includes(feed.feedId)) { + this.deleteFeed(feed, false); + feedsChanged = true; + } + } - feed.setAudioVideoMuted(metadata?.audio_muted, metadata?.video_muted); - feed.purpose = this.remoteSDPStreamMetadata![streamId]?.purpose; - feed.userId = this.remoteSDPStreamMetadata![streamId]?.user_id; + if (feedsChanged) { + this.emit(CallEvent.FeedsChanged, this.feeds); } if (this.isFocus) { this.subscribeToFocus(); @@ -2082,9 +2219,10 @@ export class MatrixCall extends TypedEventEmitter(); - const metadata = content[SDPStreamMetadataKey]; - this.updateRemoteSDPStreamMetadata(metadata); + const metadata = event.getContent()?.[SDPStreamMetadataKey]; + if (metadata) { + this.updateRemoteSDPStreamMetadata(metadata); + } } public async onAssertedIdentityReceived(event: MatrixEvent): Promise { @@ -2197,7 +2335,7 @@ export class MatrixCall extends TypedEventEmitter { if (stream.getTracks().length === 0) { - // FIXME: We should really be doing this per-track. Šimon - // leaves this for when we switch to mids for signalling - const getIndex = (): number => this.subscribedTracks.findIndex((t) => t.stream_id === stream.id); - let indexOfTrackToRemove = getIndex(); - while (indexOfTrackToRemove !== -1) { - this.subscribedTracks.splice(indexOfTrackToRemove, 1); - indexOfTrackToRemove = getIndex(); - } - logger.info(`Call ${this.callId} removing track streamId: ${stream.id}`); - this.deleteFeedByStream(stream); stream.removeEventListener("removetrack", onRemoveTrack); this.removeTrackListeners.delete(stream); } @@ -2681,6 +2809,10 @@ export class MatrixCall extends TypedEventEmitter +Copyright 2021 - 2022 Šimon Brandner +Copyright 2021 - 2023 The Matrix.org Foundation C.I.C. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { SDPStreamMetadataPurpose } from "./callEventTypes"; +import { SDPStreamMetadataPurpose, SDPStreamMetadataTracks } from "./callEventTypes"; import { acquireContext, releaseContext } from "./audioContext"; import { MatrixClient } from "../client"; import { RoomMember } from "../models/room-member"; @@ -31,8 +32,14 @@ export interface ICallFeedOpts { roomId?: string; userId: string; deviceId: string | undefined; - stream: MediaStream; + /** + * Now, this should be the same as streamId but in the future we might want + * to use something different + */ + feedId: string; + stream?: MediaStream; purpose: SDPStreamMetadataPurpose; + tracksMetadata?: SDPStreamMetadataTracks; /** * Whether or not the remote SDPStreamMetadata says audio is muted */ @@ -53,28 +60,31 @@ export enum CallFeedEvent { LocalVolumeChanged = "local_volume_changed", VolumeChanged = "volume_changed", ConnectedChanged = "connected_changed", + SizeChanged = "size_changed", Speaking = "speaking", Disposed = "disposed", } type EventHandlerMap = { - [CallFeedEvent.NewStream]: (stream: MediaStream) => void; + [CallFeedEvent.NewStream]: (stream?: MediaStream) => void; [CallFeedEvent.MuteStateChanged]: (audioMuted: boolean, videoMuted: boolean) => void; [CallFeedEvent.LocalVolumeChanged]: (localVolume: number) => void; [CallFeedEvent.VolumeChanged]: (volume: number) => void; [CallFeedEvent.ConnectedChanged]: (connected: boolean) => void; + [CallFeedEvent.SizeChanged]: () => void; [CallFeedEvent.Speaking]: (speaking: boolean) => void; [CallFeedEvent.Disposed]: () => void; }; export class CallFeed extends TypedEventEmitter { - public stream: MediaStream; - public sdpMetadataStreamId: string; - public userId: string; + public feedId: string; + public readonly userId: string; public readonly deviceId: string | undefined; public purpose: SDPStreamMetadataPurpose; public speakingVolumeSamples: number[]; + public tracksMetadata: SDPStreamMetadataTracks = {}; + private _stream?: MediaStream; private client: MatrixClient; private call?: MatrixCall; private roomId?: string; @@ -91,6 +101,11 @@ export class CallFeed extends TypedEventEmitter private _disposed = false; private _connected = false; + private _width = 0; + private _height = 0; + + private _isVisible = false; + public constructor(opts: ICallFeedOpts) { super(); @@ -103,10 +118,9 @@ export class CallFeed extends TypedEventEmitter this.audioMuted = opts.audioMuted; this.videoMuted = opts.videoMuted; this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); - this.sdpMetadataStreamId = opts.stream.id; + this.feedId = opts.feedId; - this.updateStream(null, opts.stream); - this.stream = opts.stream; // updateStream does this, but this makes TS happier + this.updateStream(undefined, opts.stream); if (this.hasAudioTrack) { this.initVolumeMeasuring(); @@ -114,8 +128,12 @@ export class CallFeed extends TypedEventEmitter if (opts.call) { opts.call.addListener(CallEvent.State, this.onCallState); - this.onCallState(opts.call.state); } + this.updateConnected(); + } + + public get stream(): MediaStream | undefined { + return this._stream; } public get connected(): boolean { @@ -128,31 +146,44 @@ export class CallFeed extends TypedEventEmitter this.emit(CallFeedEvent.ConnectedChanged, this.connected); } + public get isVisible(): boolean { + return this._isVisible; + } + + public get width(): number | undefined { + return this._width; + } + + public get height(): number | undefined { + return this._height; + } + private get hasAudioTrack(): boolean { - return this.stream.getAudioTracks().length > 0; + return this.stream ? this.stream.getAudioTracks().length > 0 : false; } - private updateStream(oldStream: MediaStream | null, newStream: MediaStream): void { + private updateStream(oldStream?: MediaStream, newStream?: MediaStream): void { if (newStream === oldStream) return; if (oldStream) { oldStream.removeEventListener("addtrack", this.onAddTrack); - this.measureVolumeActivity(false); + oldStream.removeEventListener("removetrack", this.onRemoveTrack); + clearTimeout(this.volumeLooperTimeout); } - this.stream = newStream; - newStream.addEventListener("addtrack", this.onAddTrack); + this._stream = newStream; + newStream?.addEventListener("addtrack", this.onAddTrack); + newStream?.addEventListener("removetrack", this.onRemoveTrack); - if (this.hasAudioTrack) { - this.initVolumeMeasuring(); - } else { - this.measureVolumeActivity(false); - } + this.updateConnected(); + this.initVolumeMeasuring(); + this.volumeLooper(); this.emit(CallFeedEvent.NewStream, this.stream); } private initVolumeMeasuring(): void { + if (!this.stream) return; if (!this.hasAudioTrack) return; if (!this.audioContext) this.audioContext = acquireContext(); @@ -167,16 +198,30 @@ export class CallFeed extends TypedEventEmitter } private onAddTrack = (): void => { + this.updateConnected(); this.emit(CallFeedEvent.NewStream, this.stream); }; - private onCallState = (state: CallState): void => { - if (state === CallState.Connected) { - this.connected = true; - } else if (state === CallState.Connecting) { + private onRemoveTrack = (): void => { + this.updateConnected(); + this.emit(CallFeedEvent.NewStream, this.stream); + }; + + private onCallState = (): void => { + this.updateConnected(); + }; + + private updateConnected(): void { + if (this.call?.state === CallState.Connecting) { + this.connected = false; + } else if (!this.stream) { this.connected = false; + } else if (this.stream.getTracks().length === 0) { + this.connected = false; + } else if (this.call?.state === CallState.Connected) { + this.connected = true; } - }; + } /** * Returns callRoom member @@ -204,7 +249,7 @@ export class CallFeed extends TypedEventEmitter * @returns is audio muted? */ public isAudioMuted(): boolean { - return this.stream.getAudioTracks().length === 0 || this.audioMuted; + return !this.stream || this.stream.getAudioTracks().length === 0 || this.audioMuted; } /** @@ -214,7 +259,7 @@ export class CallFeed extends TypedEventEmitter */ public isVideoMuted(): boolean { // We assume only one video track - return this.stream.getVideoTracks().length === 0 || this.videoMuted; + return !this.stream || this.stream.getVideoTracks().length === 0 || this.videoMuted; } public isSpeaking(): boolean { @@ -255,8 +300,7 @@ export class CallFeed extends TypedEventEmitter */ public measureVolumeActivity(enabled: boolean): void { if (enabled) { - if (!this.analyser || !this.frequencyBinCount || !this.hasAudioTrack) return; - + clearTimeout(this.volumeLooperTimeout); this.measuringVolumeActivity = true; this.volumeLooper(); } else { @@ -272,7 +316,8 @@ export class CallFeed extends TypedEventEmitter private volumeLooper = (): void => { if (!this.analyser) return; - + if (!this.hasAudioTrack) return; + if (!this.frequencyBinCount) return; if (!this.measuringVolumeActivity) return; this.analyser.getFloatFrequencyData(this.frequencyBinCount!); @@ -308,13 +353,17 @@ export class CallFeed extends TypedEventEmitter public clone(): CallFeed { const mediaHandler = this.client.getMediaHandler(); - const stream = this.stream.clone(); - logger.log(`callFeed cloning stream ${this.stream.id} newStream ${stream.id}`); - if (this.purpose === SDPStreamMetadataPurpose.Usermedia) { - mediaHandler.userMediaStreams.push(stream); - } else { - mediaHandler.screensharingStreams.push(stream); + let stream: MediaStream | undefined; + if (this.stream) { + stream = this.stream.clone(); + logger.log(`callFeed cloning stream ${this.stream.id} newStream ${stream.id}`); + + if (this.purpose === SDPStreamMetadataPurpose.Usermedia) { + mediaHandler.userMediaStreams.push(stream); + } else { + mediaHandler.screensharingStreams.push(stream); + } } return new CallFeed({ @@ -322,6 +371,7 @@ export class CallFeed extends TypedEventEmitter roomId: this.roomId, userId: this.userId, deviceId: this.deviceId, + feedId: this.feedId, stream, purpose: this.purpose, audioMuted: this.audioMuted, @@ -332,6 +382,7 @@ export class CallFeed extends TypedEventEmitter public dispose(): void { clearTimeout(this.volumeLooperTimeout); this.stream?.removeEventListener("addtrack", this.onAddTrack); + this.stream?.removeEventListener("removetrack", this.onRemoveTrack); this.call?.removeListener(CallEvent.State, this.onCallState); if (this.audioContext) { this.audioContext = undefined; @@ -358,4 +409,17 @@ export class CallFeed extends TypedEventEmitter this.localVolume = localVolume; this.emit(CallFeedEvent.LocalVolumeChanged, localVolume); } + + public setResolution(width: number, height: number): void { + this._width = Math.round(width); + this._height = Math.round(height); + + this.emit(CallFeedEvent.SizeChanged); + } + + public setIsVisible(isVisible: boolean): void { + this._isVisible = isVisible; + + this.emit(CallFeedEvent.SizeChanged); + } } diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 541d090945f..bc752979f6b 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -396,6 +396,7 @@ export class GroupCall extends TypedEventEmitter< roomId: this.room.roomId, userId: this.client.getUserId()!, deviceId: this.client.getDeviceId()!, + feedId: stream.id, stream, purpose: SDPStreamMetadataPurpose.Usermedia, audioMuted: this.initWithAudioMuted || stream.getAudioTracks().length === 0 || this.isPtt, @@ -419,12 +420,15 @@ export class GroupCall extends TypedEventEmitter< this.localCallFeed.setNewStream(stream); const micShouldBeMuted = this.localCallFeed.isAudioMuted(); const vidShouldBeMuted = this.localCallFeed.isVideoMuted(); - logger.log( - `groupCall ${this.groupCallId} updateLocalUsermediaStream oldStream ${oldStream.id} newStream ${stream.id} micShouldBeMuted ${micShouldBeMuted} vidShouldBeMuted ${vidShouldBeMuted}`, - ); setTracksEnabled(stream.getAudioTracks(), !micShouldBeMuted); setTracksEnabled(stream.getVideoTracks(), !vidShouldBeMuted); - this.client.getMediaHandler().stopUserMediaStream(oldStream); + + if (oldStream) { + this.client.getMediaHandler().stopUserMediaStream(oldStream); + logger.log( + `groupCall ${this.groupCallId} updateLocalUsermediaStream oldStream ${oldStream.id} newStream ${stream.id} micShouldBeMuted ${micShouldBeMuted} vidShouldBeMuted ${vidShouldBeMuted}`, + ); + } } } @@ -523,7 +527,9 @@ export class GroupCall extends TypedEventEmitter< } if (this.localScreenshareFeed) { - this.client.getMediaHandler().stopScreensharingStream(this.localScreenshareFeed.stream); + if (this.localScreenshareFeed.stream) { + this.client.getMediaHandler().stopScreensharingStream(this.localScreenshareFeed.stream); + } this.removeScreenshareFeed(this.localScreenshareFeed); this.localScreenshareFeed = undefined; this.localDesktopCapturerSourceId = undefined; @@ -652,20 +658,26 @@ export class GroupCall extends TypedEventEmitter< if (this.localCallFeed) { logger.log( - `groupCall ${this.groupCallId} setMicrophoneMuted stream ${this.localCallFeed.stream.id} muted ${muted}`, + `groupCall ${this.groupCallId} setMicrophoneMuted stream ${this.localCallFeed.feedId} muted ${muted}`, ); this.localCallFeed.setAudioVideoMuted(muted, null); // I don't believe its actually necessary to enable these tracks: they // are the one on the groupcall's own CallFeed and are cloned before being // given to any of the actual calls, so these tracks don't actually go // anywhere. Let's do it anyway to avoid confusion. - setTracksEnabled(this.localCallFeed.stream.getAudioTracks(), !muted); + if (this.localCallFeed.stream) { + setTracksEnabled(this.localCallFeed.stream.getAudioTracks(), !muted); + } } else { logger.log(`groupCall ${this.groupCallId} setMicrophoneMuted no stream muted ${muted}`); this.initWithAudioMuted = muted; } - this.forEachCall((call) => setTracksEnabled(call.localUsermediaFeed!.stream.getAudioTracks(), !muted)); + this.forEachCall((call) => { + if (call.localUsermediaStream) { + setTracksEnabled(call.localUsermediaStream.getAudioTracks(), !muted); + } + }); this.emit(GroupCallEvent.LocalMuteStateChanged, muted, this.isLocalVideoMuted()); if (!sendUpdatesBefore) await sendUpdates(); @@ -688,13 +700,15 @@ export class GroupCall extends TypedEventEmitter< if (this.localCallFeed) { logger.log( - `groupCall ${this.groupCallId} setLocalVideoMuted stream ${this.localCallFeed.stream.id} muted ${muted}`, + `groupCall ${this.groupCallId} setLocalVideoMuted stream ${this.localCallFeed.feedId} muted ${muted}`, ); const stream = await this.client.getMediaHandler().getUserMediaStream(true, !muted); await this.updateLocalUsermediaStream(stream); this.localCallFeed.setAudioVideoMuted(null, muted); - setTracksEnabled(this.localCallFeed.stream.getVideoTracks(), !muted); + if (this.localCallFeed.stream) { + setTracksEnabled(this.localCallFeed.stream.getVideoTracks(), !muted); + } } else { logger.log(`groupCall ${this.groupCallId} setLocalVideoMuted no stream muted ${muted}`); this.initWithVideoMuted = muted; @@ -736,6 +750,7 @@ export class GroupCall extends TypedEventEmitter< roomId: this.room.roomId, userId: this.client.getUserId()!, deviceId: this.client.getDeviceId()!, + feedId: stream.id, stream, purpose: SDPStreamMetadataPurpose.Screenshare, audioMuted: false, @@ -771,7 +786,9 @@ export class GroupCall extends TypedEventEmitter< this.forEachCall((call) => { if (call.localScreensharingFeed) call.removeLocalFeed(call.localScreensharingFeed); }); - this.client.getMediaHandler().stopScreensharingStream(this.localScreenshareFeed!.stream); + if (this.localScreenshareFeed?.stream) { + this.client.getMediaHandler().stopScreensharingStream(this.localScreenshareFeed.stream); + } // We have to remove the feed manually as MatrixCall has its clone, // so it won't be removed automatically this.removeScreenshareFeed(this.localScreenshareFeed!); @@ -1103,7 +1120,7 @@ export class GroupCall extends TypedEventEmitter< if (state === CallState.Connected) { if (call.isFocus) { - call.subscribeToFocus(); + call.subscribeToFocus(true); } const opponentUserId = call.getOpponentMember()?.userId; diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index 338701d7189..c41e609fd3e 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -402,8 +402,8 @@ export class MediaHandler extends TypedEventEmitter< instead XXX: Is this still true? */ - width: isWebkit ? { exact: 640 } : { ideal: 640 }, - height: isWebkit ? { exact: 360 } : { ideal: 360 }, + width: isWebkit ? { exact: 1280 } : { ideal: 1280 }, + height: isWebkit ? { exact: 720 } : { ideal: 720 }, } : false, }; From e7c45ffe39f644beba72e80c8511d930348841a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 25 Jan 2023 17:14:36 +0100 Subject: [PATCH 116/137] Fix `call.spec.ts` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- spec/unit/webrtc/call.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index b8205423b7c..1c56dc0b256 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -541,7 +541,7 @@ describe("Call", function () { // since this is testing for the presence of a local sender, we need to add a transciever // rather than just a source track const mockTrack = new MockMediaStreamTrack("track_id", "video"); - const mockTransceiver = new MockRTCRtpTransceiver(call.peerConn as unknown as MockRTCPeerConnection); + const mockTransceiver = new MockRTCRtpTransceiver(call.peerConn as unknown as MockRTCPeerConnection, "0"); mockTransceiver.sender = new MockRTCRtpSender(mockTrack) as unknown as RTCRtpSender; (call as any).transceivers.set("m.usermedia:video", mockTransceiver); From 86d135815e7c8d8fb37cb6d8df58b6c0240f4a12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 25 Jan 2023 17:32:48 +0100 Subject: [PATCH 117/137] Don't `setParameters()` in `updateLocalUsermediaStream()` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is not necessary since we already have the encodings from when we added the transceive in the first place Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 117b56051e8..73d3470fdde 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1438,14 +1438,6 @@ export class MatrixCall extends TypedEventEmitter Date: Thu, 26 Jan 2023 16:50:36 +0100 Subject: [PATCH 118/137] Clean-up simulcast code (#3106) --- src/webrtc/call.ts | 265 +++++++++++++++++++++------------------------ 1 file changed, 121 insertions(+), 144 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 73d3470fdde..efd619790dd 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -600,8 +600,8 @@ export class MatrixCall extends TypedEventEmitter feed.feedId === streamId); + private getFeedById(feedId: string): CallFeed | undefined { + return this.getFeeds().find((feed) => feed.feedId === feedId); } /** @@ -818,92 +818,121 @@ export class MatrixCall extends TypedEventEmitter callFeed.feedId === feed.feedId)) { - logger.info(`Ignoring duplicate local stream ${callFeed.feedId} in call ${this.callId}`); - return; - } - - this.feeds.push(callFeed); + this.feeds.push(feed); + this.emit(CallEvent.FeedsChanged, this.feeds); + logger.info(`Call ${this.callId} pushLocalFeed() succeeded (id=${feed.feedId} purpose=${feed.purpose})`); if (addToPeerConnection) { - for (const track of callFeed.stream!.getTracks()) { - logger.info( - `Call ${this.callId} ` + - `Adding track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${callFeed.feedId}", ` + - `streamPurpose="${callFeed.purpose}", ` + - `enabled=${track.enabled}` + - `) to peer connection`, - ); + this.addTracksOfFeedToPeerConnection(feed); + } + } - const encodings = track.kind === "video" ? SIMULCAST_ENCODINGS : undefined; + /** + * This method takes the feeds tracks and adds them to the peer connection. + * It tries to re-use transceivers/senders by using replaceTrack(), if + * possible. + * @param callFeed - feed whose tracks we add to the peer connection + */ + private async addTracksOfFeedToPeerConnection(callFeed: CallFeed): Promise { + const purpose = callFeed.purpose; + const feedId = callFeed.feedId; + const stream = callFeed.stream; + if (!stream) { + logger.warn( + `Call ${this.callId} addTracksOfFeedToPeerConnection() failed: no stream (feedId=${feedId} purpose=${purpose})`, + ); + return; + } - const tKey = getTransceiverKey(callFeed.purpose, track.kind); - if (this.transceivers.has(tKey)) { - // we already have a sender, so we re-use it. We try to re-use transceivers as much - // as possible because they can't be removed once added, so otherwise they just - // accumulate which makes the SDP very large very quickly: in fact it only takes - // about 6 video tracks to exceed the maximum size of an Olm-encrypted - // Matrix event. - const transceiver = this.transceivers.get(tKey)!; + for (const track of stream.getTracks()) { + logger.info( + `Call ${this.callId} addTracksOfFeedToPeerConnection() running (feedId=${feedId} streamPurpose=${purpose} kind=${track.kind} enabled=${track.enabled})`, + ); - // RTCRtpSender::setStreams() is currently not supported by - // Firefox but we try to use it at least in other browsers - if (transceiver.sender.setStreams) transceiver.sender.setStreams(callFeed.stream!); + const encodings = track.kind === "video" ? SIMULCAST_ENCODINGS : undefined; + const transceiverKey = getTransceiverKey(purpose, track.kind); + const transceiver = this.transceivers.get(transceiverKey); + const sender = transceiver?.sender; - transceiver.sender.replaceTrack(track); + let added = false; + if (sender) { + try { + // We already have a sender, so we re-use it. We try to + // re-use transceivers as much as possible because they + // can't be removed once added, so otherwise they just + // accumulate which makes the SDP very large very quickly: + // in fact it only takes about 6 video tracks to exceed the + // maximum size of an Olm-encrypted Matrix event - Dave + + // setStreams() is currently not supported by Firefox but we + // try to use it at least in other browsers (once we switch + // to using mids and throw away streamIds we will be able to + // throw this away) + if (sender.setStreams) sender.setStreams(stream); + + sender.replaceTrack(track); + + // We don't need to set simulcast encodings in here since we + // have already done that the first time we added the + // transceiver + + // Set the direction of the transceiver to indicate we're + // going to be sending. This may trigger re-negotiation, if + // we weren't sending until now + transceiver.direction = transceiver.direction === "inactive" ? "sendonly" : "sendrecv"; - if (this.isFocus) { - const parameters = transceiver.sender.getParameters(); - transceiver.sender.setParameters({ - ...transceiver.sender.getParameters(), - encodings: encodings ?? parameters.encodings, - }); - } + added = true; + } catch (error) { + logger.info( + `Call ${this.callId} addTracksOfFeedToPeerConnection() failed to replaceTrack()`, + error, + ); + } + } - // set the direction to indicate we're going to start sending again - // (this will trigger the re-negotiation) - transceiver.direction = transceiver.direction === "inactive" ? "sendonly" : "sendrecv"; - } else { - // create a new one - const transceiver = this.peerConn!.addTransceiver(track, { - streams: [callFeed.stream!], - sendEncodings: this.isFocus && isFirefox() ? undefined : encodings, + if (!added) { + try { + // We either don't have a sender or we failed to do + // replaceTrack(), so we use addTransceiver() to add the + // track + const newTransceiver = this.peerConn!.addTransceiver(track, { + streams: [stream], + // Chrome does not allow us to change the encodings + // later, so we have to use addTransceiver() to set them + sendEncodings: this.isFocus ? undefined : encodings, }); if (this.isFocus && isFirefox()) { - const parameters = transceiver.sender.getParameters(); - transceiver.sender.setParameters({ - ...transceiver.sender.getParameters(), + const parameters = newTransceiver.sender.getParameters(); + newTransceiver.sender.setParameters({ + ...parameters, + // Firefox does not support the sendEncodings + // parameter on addTransceiver(), so we use + // setParameters() to set them encodings: encodings ?? parameters.encodings, }); } - this.transceivers.set(tKey, transceiver); + this.transceivers.set(transceiverKey, newTransceiver); + + added = true; + } catch (error) { + logger.error( + `Call ${this.callId} addTracksOfFeedToPeerConnection() failed to addTransceiver()`, + error, + ); } } } - - logger.info( - `Call ${this.callId} ` + - `Pushed local stream ` + - `(id="${callFeed.feedId}", ` + - `active="${callFeed.stream!.active}", ` + - `purpose="${callFeed.purpose}")`, - ); - - this.emit(CallEvent.FeedsChanged, this.feeds); } /** @@ -1383,98 +1412,46 @@ export class MatrixCall extends TypedEventEmitter { - const callFeed = this.localUsermediaFeed!; - const audioEnabled = forceAudio || (!callFeed.isAudioMuted() && !this.remoteOnHold); - const videoEnabled = forceVideo || (!callFeed.isVideoMuted() && !this.remoteOnHold); + const feed = this.localUsermediaFeed; + if (!feed) { + logger.log(`Call ${this.callId} updateLocalUsermediaStream() failed: we do not have localUsermediaFeed`); + return; + } + const oldStream = this.localUsermediaStream; + if (!oldStream) { + logger.log(`Call ${this.callId} updateLocalUsermediaStream() failed: we do not have localUsermediaStream`); + return; + } + + const audioEnabled = forceAudio || (!feed.isAudioMuted() && !this.remoteOnHold); + const videoEnabled = forceVideo || (!feed.isVideoMuted() && !this.remoteOnHold); logger.log( - `call ${this.callId} updateLocalUsermediaStream stream ${stream.id} audioEnabled ${audioEnabled} videoEnabled ${videoEnabled}`, + `call ${this.callId} updateLocalUsermediaStream stream ${newStream.id} audioEnabled ${audioEnabled} videoEnabled ${videoEnabled}`, ); - setTracksEnabled(stream.getAudioTracks(), audioEnabled); - setTracksEnabled(stream.getVideoTracks(), videoEnabled); - // We want to keep the same stream id, so we replace the tracks rather - // than the whole stream. + // Firstly, make sure we keep the mute state + setTracksEnabled(newStream.getAudioTracks(), audioEnabled); + setTracksEnabled(newStream.getVideoTracks(), videoEnabled); - // Firstly, we replace the tracks in our localUsermediaStream. - for (const track of this.localUsermediaStream!.getTracks()) { - this.localUsermediaStream!.removeTrack(track); + // Secondly, we replace the tracks in our oldStream with the tracks from + // the newStream + for (const track of oldStream.getTracks()) { + oldStream.removeTrack(track); track.stop(); } - for (const track of stream.getTracks()) { - this.localUsermediaStream!.addTrack(track); + for (const track of newStream.getTracks()) { + oldStream.addTrack(track); } - // Then replace the old tracks, if possible. - for (const track of stream.getTracks()) { - const tKey = getTransceiverKey(SDPStreamMetadataPurpose.Usermedia, track.kind); - - const transceiver = this.transceivers.get(tKey); - const oldSender = transceiver?.sender; - const encodings = track.kind === "video" ? SIMULCAST_ENCODINGS : undefined; - - let added = false; - if (oldSender) { - try { - logger.info( - `Call ${this.callId} ` + - `Replacing track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - - // RTCRtpSender::setStreams() is currently not supported by - // Firefox but we try to use it at least in other browsers - if (transceiver.sender.setStreams) transceiver.sender.setStreams(this.localUsermediaStream!); - - await oldSender.replaceTrack(track); - - // Set the direction to indicate we're going to be sending. - // This is only necessary in the cases where we're upgrading - // the call to video after downgrading it. - transceiver.direction = transceiver.direction === "inactive" ? "sendonly" : "sendrecv"; - added = true; - } catch (error) { - logger.warn(`replaceTrack failed: adding new transceiver instead`, error); - } - } - - if (!added) { - logger.info( - `Call ${this.callId} ` + - `Adding track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - - const newTransceiver = this.peerConn!.addTransceiver(track, { - streams: [this.localUsermediaStream!], - sendEncodings: this.isFocus && isFirefox() ? undefined : encodings, - }); - - if (this.isFocus && isFirefox()) { - const parameters = newTransceiver.sender.getParameters(); - newTransceiver.sender.setParameters({ - ...newTransceiver.sender.getParameters(), - encodings: encodings ?? parameters.encodings, - }); - } - - this.transceivers.set(tKey, newTransceiver); - } - } + // Then, we add the feed's tracks to the peer connection + this.addTracksOfFeedToPeerConnection(feed); } /** From 523302f189df907a9f3bc633a93797dfab5c2b80 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 27 Jan 2023 13:17:57 +0000 Subject: [PATCH 119/137] Fix simulcast on chrome (#3109) --- src/webrtc/call.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index efd619790dd..81c3d83841c 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -908,7 +908,9 @@ export class MatrixCall extends TypedEventEmitter Date: Fri, 27 Jan 2023 16:02:41 +0100 Subject: [PATCH 120/137] Retry SFU calls and refactor retry code (#3110) --- src/webrtc/groupCall.ts | 222 ++++++++++++++++++++++------------------ 1 file changed, 123 insertions(+), 99 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 0f14b237c95..ceef5edd458 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -175,6 +175,7 @@ interface ICallHandlers { } const DEVICE_TIMEOUT = 1000 * 60 * 60; // 1 hour +const FOCUS_SESSION_ID = "sfu"; function getCallUserId(call: MatrixCall): string | null { return call.getOpponentMember()?.userId || call.invitee || null; @@ -468,43 +469,6 @@ export class GroupCall extends TypedEventEmitter< this.activeSpeaker = undefined; this.onActiveSpeakerLoop(); this.activeSpeakerLoopInterval = setInterval(this.onActiveSpeakerLoop, this.activeSpeakerInterval); - - if (this.foci[0]) { - const onError = (e?: unknown): void => { - logger.warn(`Failed to place call to ${this.foci[0].user_id}!`, e); - this.emit( - GroupCallEvent.Error, - new GroupCallError( - GroupCallErrorCode.PlaceCallFailed, - `Failed to place call to ${this.foci[0].user_id}.`, - ), - ); - }; - - const focusCall = createNewMatrixCall(this.client, this.room.roomId, { - invitee: this.foci[0].user_id, - opponentDeviceId: this.foci[0].device_id, - opponentSessionId: "sfu", - groupCallId: this.groupCallId, - isFocus: true, - }); - if (!focusCall) { - onError(); - return; - } - - try { - focusCall.isPtt = this.isPtt; - await focusCall.placeCallWithCallFeeds(this.getLocalFeeds().map((feed) => feed.clone())); - focusCall.createDataChannel("datachannel", this.dataChannelOptions); - } catch (e) { - onError(e); - return; - } - - this.calls.set(this.foci[0].user_id, new Map([[this.foci[0].device_id, focusCall]])); - this.initCall(focusCall); - } } private chooseFocus(): void { @@ -523,7 +487,10 @@ export class GroupCall extends TypedEventEmitter< } } - this.foci = [focusOfAnotherMember ?? this.client.getFoci()[0]]; + const focus = focusOfAnotherMember ?? this.client.getFoci()[0]; + if (focus && !this.foci.some((f) => f.user_id === focus.user_id && f.device_id === focus.device_id)) { + this.foci.push(focus); + } } private dispose(): void { @@ -882,80 +849,122 @@ export class GroupCall extends TypedEventEmitter< * Places calls to all participants that we're responsible for calling. */ private placeOutgoingCalls(): void { - if (this.foci[0]) return; - let callsChanged = false; - for (const [{ userId }, participantMap] of this.participants) { - const callMap = this.calls.get(userId) ?? new Map(); + const onError = ( + error: Error, + userId: string, + deviceId: string, + newCall: MatrixCall | null, + callMap: Map, + ): void => { + logger.warn(`Failed to place call to ${userId} ${deviceId}`, error); + + if (error instanceof CallError && error.code === GroupCallErrorCode.UnknownDevice) { + this.emit(GroupCallEvent.Error, error); + } else { + this.emit(GroupCallEvent.Error, new GroupCallError(GroupCallErrorCode.PlaceCallFailed, error.message)); + } - for (const [deviceId, participant] of participantMap) { - const prevCall = callMap.get(deviceId); + if (newCall !== null) { + this.disposeCall(newCall, CallErrorCode.SignallingFailed); + if (callMap.get(deviceId) === newCall) callMap.delete(deviceId); + } + }; - if ( - prevCall?.getOpponentSessionId() !== participant.sessionId && - this.wantsOutgoingCall(userId, deviceId) - ) { - callsChanged = true; + const replaceSession = ( + userId: string, + deviceId: string, + prevCall: MatrixCall | undefined, + opponentSessionId: string, + opponentIsScreensharing: boolean, + callMap: Map, + ): void => { + callsChanged = true; + + if (prevCall) { + logger.debug(`Replacing call ${prevCall.callId} to ${userId} ${deviceId}`); + this.disposeCall(prevCall, CallErrorCode.NewSession); + } - if (prevCall !== undefined) { - logger.debug(`Replacing call ${prevCall.callId} to ${userId} ${deviceId}`); - this.disposeCall(prevCall, CallErrorCode.NewSession); - } + const newCall = createNewMatrixCall(this.client, this.room.roomId, { + invitee: userId, + opponentDeviceId: deviceId, + opponentSessionId: opponentSessionId, + groupCallId: this.groupCallId, + isFocus: opponentSessionId === FOCUS_SESSION_ID, + }); - const newCall = createNewMatrixCall(this.client, this.room.roomId, { - invitee: userId, - opponentDeviceId: deviceId, - opponentSessionId: participant.sessionId, - groupCallId: this.groupCallId, - }); + if (newCall === null) { + onError(new Error("Failed to create new call"), userId, deviceId, newCall, callMap); + return; + } + + this.initCall(newCall); + callMap.set(deviceId, newCall); + + logger.debug(`Placing call to ${userId} ${deviceId} (session ${opponentSessionId})`); - if (newCall === null) { - logger.error(`Failed to create call with ${userId} ${deviceId}`); - callMap.delete(deviceId); - } else { - this.initCall(newCall); - callMap.set(deviceId, newCall); - - logger.debug(`Placing call to ${userId} ${deviceId} (session ${participant.sessionId})`); - - newCall - .placeCallWithCallFeeds( - this.getLocalFeeds().map((feed) => feed.clone()), - participant.screensharing, - ) - .then(() => { - if (this.dataChannelsEnabled) { - newCall.createDataChannel("datachannel", this.dataChannelOptions); - } - }) - .catch((e) => { - logger.warn(`Failed to place call to ${userId}`, e); - - if (e instanceof CallError && e.code === GroupCallErrorCode.UnknownDevice) { - this.emit(GroupCallEvent.Error, e); - } else { - this.emit( - GroupCallEvent.Error, - new GroupCallError( - GroupCallErrorCode.PlaceCallFailed, - `Failed to place call to ${userId}`, - ), - ); - } - - this.disposeCall(newCall, CallErrorCode.SignallingFailed); - if (callMap.get(deviceId) === newCall) callMap.delete(deviceId); - }); + newCall + .placeCallWithCallFeeds( + this.getLocalFeeds().map((feed) => feed.clone()), + opponentIsScreensharing, + ) + .then(() => { + if (this.dataChannelsEnabled || opponentSessionId === FOCUS_SESSION_ID) { + newCall.createDataChannel("datachannel", this.dataChannelOptions); } - } - } + }) + .catch((e) => { + onError(e, userId, deviceId, newCall, callMap); + }); if (callMap.size > 0) { this.calls.set(userId, callMap); } else { this.calls.delete(userId); } + }; + + if (this.foci.length > 0) { + // We have a focus to call, so we call it + for (const { user_id: userId, device_id: deviceId } of this.foci) { + const callMap = this.calls.get(userId) ?? new Map(); + const prevCall = callMap.get(deviceId); + + if (prevCall && !prevCall.callHasEnded()) { + continue; + } + + callsChanged = true; + replaceSession(userId, deviceId, prevCall, FOCUS_SESSION_ID, false, callMap); + } + } else { + // There is no focus to call, so we connect full-mesh + for (const [{ userId }, participantMap] of this.participants) { + const callMap = this.calls.get(userId) ?? new Map(); + + for (const [deviceId, participant] of participantMap) { + const prevCall = callMap.get(deviceId); + + if ( + prevCall?.getOpponentSessionId() === participant.sessionId || + !this.wantsOutgoingCall(userId, deviceId) + ) { + continue; + } + + callsChanged = true; + replaceSession( + userId, + deviceId, + prevCall, + participant.sessionId, + participant.screensharing, + callMap, + ); + } + } } if (callsChanged) this.emit(GroupCallEvent.CallsChanged, this.calls); @@ -999,6 +1008,21 @@ export class GroupCall extends TypedEventEmitter< } } + for (const { user_id: userId, device_id: deviceId } of this.foci) { + const call = this.calls.get(userId)?.get(deviceId); + let retriesMap = this.retryCallCounts.get(userId); + const retries = retriesMap?.get(deviceId) ?? 0; + + if ((!call || call.callHasEnded()) && retries < 3) { + if (retriesMap === undefined) { + retriesMap = new Map(); + this.retryCallCounts.set(userId, retriesMap); + } + retriesMap.set(deviceId, retries + 1); + needsRetry = true; + } + } + if (needsRetry) this.placeOutgoingCalls(); }; @@ -1129,7 +1153,7 @@ export class GroupCall extends TypedEventEmitter< call.subscribeToFocus(true); } - const opponentUserId = call.getOpponentMember()?.userId; + const opponentUserId = call.getOpponentMember()?.userId || call.invitee; if (opponentUserId) { const retriesMap = this.retryCallCounts.get(opponentUserId); retriesMap?.delete(call.getOpponentDeviceId()!); From c8fed8f8ccf724bac3e9848c7851888976f26db1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 27 Jan 2023 16:09:10 +0100 Subject: [PATCH 121/137] Put `maxBitrate` back MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 81c3d83841c..50264dca29c 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -287,13 +287,16 @@ const SIMULCAST_ENCODINGS = [ // has priority and therefore we put full as first // as we always want to send the full resolution { + maxBitrate: 4_500_000, rid: SimulcastResolution.Full, }, { + maxBitrate: 1_500_000, rid: SimulcastResolution.Half, scaleResolutionDownBy: 2.0, }, { + maxBitrate: 300_000, rid: SimulcastResolution.Quarter, scaleResolutionDownBy: 4.0, }, From a429e59d75079bc4a45e78eeb55b690bbb7bb7d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 27 Jan 2023 17:16:50 +0100 Subject: [PATCH 122/137] Improve `maxBitrate` values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 26 +++++++++++++++++++++----- src/webrtc/mediaHandler.ts | 1 + 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index a2691e63810..6c1fdf06ff6 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -279,7 +279,7 @@ const ICE_DISCONNECTED_TIMEOUT = 30 * 1000; // ms */ const SUBSCRIBE_TO_FOCUS_TIMEOUT = 2 * 1000; -const SIMULCAST_ENCODINGS = [ +const SIMULCAST_USERMEDIA_ENCODINGS: RTCRtpEncodingParameters[] = [ // Order is important here: some browsers (e.g. // Chrome) will only send some of the encodings, if // the track has a resolution to low for it to send @@ -287,16 +287,23 @@ const SIMULCAST_ENCODINGS = [ // has priority and therefore we put full as first // as we always want to send the full resolution { - maxBitrate: 4_500_000, + // 720p (base) + maxFramerate: 30, + maxBitrate: 1_700_000, rid: SimulcastResolution.Full, }, { - maxBitrate: 1_500_000, + // 360p + maxFramerate: 20, + maxBitrate: 300_000, rid: SimulcastResolution.Half, scaleResolutionDownBy: 2.0, }, { - maxBitrate: 300_000, + // 180p + maxFramerate: 15, + maxBitrate: 120_000, + rid: SimulcastResolution.Quarter, scaleResolutionDownBy: 4.0, }, @@ -313,6 +320,15 @@ export class CallError extends Error { } } +export const getSimulcastEncodings = (purpose: SDPStreamMetadataPurpose): RTCRtpEncodingParameters[] => { + if (purpose === SDPStreamMetadataPurpose.Usermedia) { + return SIMULCAST_USERMEDIA_ENCODINGS; + } + + // Fallback to usermedia encodings + return SIMULCAST_USERMEDIA_ENCODINGS; +}; + export function genCallID(): string { return Date.now().toString() + randomString(16); } @@ -861,7 +877,7 @@ export class MatrixCall extends TypedEventEmitter Date: Fri, 27 Jan 2023 20:32:34 +0100 Subject: [PATCH 123/137] Add simulcast screensharing encodings (#3111) --- spec/unit/webrtc/mediaHandler.spec.ts | 4 +- src/@types/global.d.ts | 35 +++++++----------- src/webrtc/call.ts | 38 ++++++++++++++++--- src/webrtc/mediaHandler.ts | 53 +++++++++++++++------------ 4 files changed, 77 insertions(+), 53 deletions(-) diff --git a/spec/unit/webrtc/mediaHandler.spec.ts b/spec/unit/webrtc/mediaHandler.spec.ts index 1dc84e58550..8e6a1753627 100644 --- a/spec/unit/webrtc/mediaHandler.spec.ts +++ b/spec/unit/webrtc/mediaHandler.spec.ts @@ -350,12 +350,12 @@ describe("Media Handler", function () { expect(mockMediaDevices.getUserMedia).toHaveBeenCalledWith( expect.objectContaining({ - video: { + video: expect.objectContaining({ mandatory: expect.objectContaining({ chromeMediaSource: "desktop", chromeMediaSourceId: FAKE_DESKTOP_SOURCE_ID, }), - }, + }), }), ); }); diff --git a/src/@types/global.d.ts b/src/@types/global.d.ts index 749eb7f417b..ec6a1722be6 100644 --- a/src/@types/global.d.ts +++ b/src/@types/global.d.ts @@ -44,29 +44,22 @@ declare global { } interface MediaDevices { - // This is experimental and types don't know about it yet - // https://github.com/microsoft/TypeScript/issues/33232 - getDisplayMedia(constraints: MediaStreamConstraints | DesktopCapturerConstraints): Promise; - getUserMedia(constraints: MediaStreamConstraints | DesktopCapturerConstraints): Promise; + getDisplayMedia(constraints: ExtendedMediaStreamConstraints): Promise; + getUserMedia(constraints: ExtendedMediaStreamConstraints): Promise; } - interface DesktopCapturerConstraints { - audio: - | boolean - | { - mandatory: { - chromeMediaSource: string; - chromeMediaSourceId: string; - }; - }; - video: - | boolean - | { - mandatory: { - chromeMediaSource: string; - chromeMediaSourceId: string; - }; - }; + interface ChromeMediaSourceConstraints { + chromeMediaSource: string; + chromeMediaSourceId: string; + } + + interface ExtendedMediaTrackConstraints extends MediaTrackConstraints { + mandatory?: ChromeMediaSourceConstraints; + } + + interface ExtendedMediaStreamConstraints { + audio: boolean | ExtendedMediaTrackConstraints; + video: boolean | ExtendedMediaTrackConstraints; } interface DummyInterfaceWeShouldntBeUsingThis {} diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 6c1fdf06ff6..f3ffd89aa2f 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -279,13 +279,13 @@ const ICE_DISCONNECTED_TIMEOUT = 30 * 1000; // ms */ const SUBSCRIBE_TO_FOCUS_TIMEOUT = 2 * 1000; +// Order is important here: some browsers (e.g. +// Chrome) will only send some of the encodings, if +// the track has a resolution to low for it to send +// all, in that case the encoding higher in the list +// has priority and therefore we put full as first +// as we always want to send the full resolution const SIMULCAST_USERMEDIA_ENCODINGS: RTCRtpEncodingParameters[] = [ - // Order is important here: some browsers (e.g. - // Chrome) will only send some of the encodings, if - // the track has a resolution to low for it to send - // all, in that case the encoding higher in the list - // has priority and therefore we put full as first - // as we always want to send the full resolution { // 720p (base) maxFramerate: 30, @@ -309,6 +309,29 @@ const SIMULCAST_USERMEDIA_ENCODINGS: RTCRtpEncodingParameters[] = [ }, ]; +const SIMULCAST_SCREENSHARING_ENCODINGS: RTCRtpEncodingParameters[] = [ + { + // 1080p (base) + maxFramerate: 30, + maxBitrate: 3_000_000, + rid: SimulcastResolution.Full, + }, + { + // 720p + maxFramerate: 15, + maxBitrate: 1_000_000, + rid: SimulcastResolution.Half, + scaleResolutionDownBy: 1.5, + }, + { + // 360p + maxFramerate: 3, + maxBitrate: 200_000, + rid: SimulcastResolution.Quarter, + scaleResolutionDownBy: 3, + }, +]; + export class CallError extends Error { public readonly code: string; @@ -324,6 +347,9 @@ export const getSimulcastEncodings = (purpose: SDPStreamMetadataPurpose): RTCRtp if (purpose === SDPStreamMetadataPurpose.Usermedia) { return SIMULCAST_USERMEDIA_ENCODINGS; } + if (purpose === SDPStreamMetadataPurpose.Screenshare) { + return SIMULCAST_SCREENSHARING_ENCODINGS; + } // Fallback to usermedia encodings return SIMULCAST_USERMEDIA_ENCODINGS; diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index 682d7a60e60..9a84d396657 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -22,6 +22,20 @@ import { GroupCallType, GroupCallState } from "../webrtc/groupCall"; import { logger } from "../logger"; import { MatrixClient } from "../client"; +const isWebkit = (): boolean => Boolean(navigator.webkitGetUserMedia); +const getIdealOrExact = (value: number): ConstrainULong => (isWebkit() ? { exact: value } : { ideal: value }); +const getDimensionConstraints = ( + width: number, + height: number, + framerate: number, +): Pick => ({ + // Chrome will give it only if we ask exactly, FF refuses entirely if we ask + // exactly, so have to ask for ideal instead XXX: Is this still true? + width: getIdealOrExact(width), + height: getIdealOrExact(height), + frameRate: getIdealOrExact(framerate), +}); + export enum MediaHandlerEvent { LocalStreamsChanged = "local_streams_changed", } @@ -399,8 +413,6 @@ export class MediaHandler extends TypedEventEmitter< } private getUserMediaContraints(audio: boolean, video: boolean): MediaStreamConstraints { - const isWebkit = !!navigator.webkitGetUserMedia; - return { audio: audio ? { @@ -412,39 +424,32 @@ export class MediaHandler extends TypedEventEmitter< : false, video: video ? { + ...getDimensionConstraints(1280, 720, 30), deviceId: this.videoInput ? { ideal: this.videoInput } : undefined, - /* We want 640x360. Chrome will give it only if we ask exactly, - FF refuses entirely if we ask exactly, so have to ask for ideal - instead - XXX: Is this still true? - */ - width: isWebkit ? { exact: 1280 } : { ideal: 1280 }, - height: isWebkit ? { exact: 720 } : { ideal: 720 }, - frameRate: isWebkit ? { exact: 30 } : { ideal: 30 }, } : false, }; } - private getScreenshareContraints(opts: IScreensharingOpts): DesktopCapturerConstraints { + private getScreenshareContraints(opts: IScreensharingOpts): ExtendedMediaStreamConstraints { const { desktopCapturerSourceId, audio } = opts; if (desktopCapturerSourceId) { logger.debug("Using desktop capturer source", desktopCapturerSourceId); - return { - audio: audio ?? false, - video: { - mandatory: { - chromeMediaSource: "desktop", - chromeMediaSourceId: desktopCapturerSourceId, - }, - }, - }; } else { logger.debug("Not using desktop capturer source"); - return { - audio: audio ?? false, - video: true, - }; } + + return { + audio: audio ?? false, + video: { + ...getDimensionConstraints(1920, 1080, 30), + mandatory: desktopCapturerSourceId + ? { + chromeMediaSource: "desktop", + chromeMediaSourceId: desktopCapturerSourceId, + } + : undefined, + }, + }; } } From a735aae6ae290a85edca5f83a88738d3d7e93cdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 31 Jan 2023 15:02:00 +0100 Subject: [PATCH 124/137] Don't use ideal/exact in case of screen-sharing (#3112) --- src/webrtc/mediaHandler.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index 9a84d396657..5b58d16269e 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -23,17 +23,19 @@ import { logger } from "../logger"; import { MatrixClient } from "../client"; const isWebkit = (): boolean => Boolean(navigator.webkitGetUserMedia); +// Chrome will give it only if we ask exactly, FF refuses entirely if we ask +// exactly, so have to ask for ideal instead XXX: Is this still true? const getIdealOrExact = (value: number): ConstrainULong => (isWebkit() ? { exact: value } : { ideal: value }); +const getIdealAndMax = (value: number): ConstrainULong => ({ ideal: value, max: value }); const getDimensionConstraints = ( + func: (value: number) => ConstrainULong, width: number, height: number, framerate: number, ): Pick => ({ - // Chrome will give it only if we ask exactly, FF refuses entirely if we ask - // exactly, so have to ask for ideal instead XXX: Is this still true? - width: getIdealOrExact(width), - height: getIdealOrExact(height), - frameRate: getIdealOrExact(framerate), + width: func(width), + height: func(height), + frameRate: func(framerate), }); export enum MediaHandlerEvent { @@ -424,7 +426,7 @@ export class MediaHandler extends TypedEventEmitter< : false, video: video ? { - ...getDimensionConstraints(1280, 720, 30), + ...getDimensionConstraints(getIdealOrExact, 1280, 720, 30), deviceId: this.videoInput ? { ideal: this.videoInput } : undefined, } : false, @@ -442,7 +444,7 @@ export class MediaHandler extends TypedEventEmitter< return { audio: audio ?? false, video: { - ...getDimensionConstraints(1920, 1080, 30), + ...getDimensionConstraints(getIdealAndMax, 1920, 1080, 30), mandatory: desktopCapturerSourceId ? { chromeMediaSource: "desktop", From 61c8fb3b21b0b934afb7a5d0bb6340362d55a1eb Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 31 Jan 2023 16:45:43 +0000 Subject: [PATCH 125/137] Don't require e2e keys for foci Since we don't speak e2e to them anyway --- src/webrtc/call.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index f3ffd89aa2f..bb36b81dcd7 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -676,6 +676,9 @@ export class MatrixCall extends TypedEventEmitter { if (!this.opponentDeviceId) return; if (!this.client.getUseE2eForGroupCall()) return; + // We (currently) don't speak e2e with foci. It's debateable whether there would be + // any benefit in doing so. + if (this.isFocus) return; // It's possible to want E2EE and yet not have the means to manage E2EE // ourselves (for example if the client is a RoomWidgetClient) if (!this.client.isCryptoEnabled()) { From 6ee8715945d652ac2703367459733b6c82a7f2a0 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 2 Feb 2023 11:46:32 +0000 Subject: [PATCH 126/137] Work around pion bug to fix screen sharing https://github.com/matrix-org/waterfall/issues/98 --- src/webrtc/call.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index bb36b81dcd7..23ac5c73cbe 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -912,7 +912,9 @@ export class MatrixCall extends TypedEventEmitter Date: Thu, 2 Feb 2023 11:53:22 +0000 Subject: [PATCH 127/137] More commenting --- src/webrtc/call.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 23ac5c73cbe..42316a99d8d 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -914,6 +914,11 @@ export class MatrixCall extends TypedEventEmitter Date: Thu, 9 Feb 2023 20:37:01 +0100 Subject: [PATCH 128/137] Fix tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index c0ac54c7445..92c56cd3b17 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -993,12 +993,6 @@ export class MatrixCall extends TypedEventEmitter Date: Fri, 10 Feb 2023 13:06:52 +0100 Subject: [PATCH 129/137] Re-use transceivers for user media As per comment. --- src/webrtc/call.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 92c56cd3b17..6d044d6defa 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -914,14 +914,18 @@ export class MatrixCall extends TypedEventEmitter Date: Fri, 17 Feb 2023 14:21:46 +0100 Subject: [PATCH 130/137] Refactor `CallFeed`s (#3153) --- spec/test-utils/webrtc.ts | 59 ++- spec/unit/webrtc/call.spec.ts | 234 +++++---- spec/unit/webrtc/callFeed.spec.ts | 23 +- spec/unit/webrtc/groupCall.spec.ts | 86 +-- src/webrtc/call.ts | 809 ++++++++++++----------------- src/webrtc/callEventTypes.ts | 10 +- src/webrtc/callFeed.ts | 192 ++----- src/webrtc/callTrack.ts | 47 ++ src/webrtc/groupCall.ts | 33 +- src/webrtc/localCallFeed.ts | 164 ++++++ src/webrtc/localCallTrack.ts | 311 +++++++++++ src/webrtc/remoteCallFeed.ts | 229 ++++++++ src/webrtc/remoteCallTrack.ts | 93 ++++ 13 files changed, 1463 insertions(+), 827 deletions(-) create mode 100644 src/webrtc/callTrack.ts create mode 100644 src/webrtc/localCallFeed.ts create mode 100644 src/webrtc/localCallTrack.ts create mode 100644 src/webrtc/remoteCallFeed.ts create mode 100644 src/webrtc/remoteCallTrack.ts diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index 6edfd9a0c36..48ceef555ec 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -33,6 +33,7 @@ import { RoomStateEventHandlerMap, } from "../../src"; import { TypedEventEmitter } from "../../src/models/typed-event-emitter"; +import { randomString } from "../../src/randomstring"; import { ReEmitter } from "../../src/ReEmitter"; import { SyncState } from "../../src/sync"; import { CallEvent, CallEventHandlerMap, CallState, MatrixCall } from "../../src/webrtc/call"; @@ -96,6 +97,30 @@ export const FAKE_DEVICE_ID_2 = "@BBBBBB"; export const FAKE_SESSION_ID_2 = "bob1"; export const FAKE_USER_ID_3 = "@charlie:test.dummy"; +export const runOnTrackForStream = (call: MatrixCall, stream: MediaStream) => { + let sdp = "v=0\n" + "o=- 7135465365607179083 2 IN IP4 127.0.0.1\n" + "s=-\n" + "t=0 0\n" + "a=group:BUNDLE 0 1 2\n"; + + stream.getTracks().forEach((track, index) => { + sdp += `m=${track.kind}\n`; + sdp += `a=mid:${index}\n`; + sdp += `a=msid:${stream.id} ${track.id}\n`; + }); + + // @ts-ignore + call.peerConn.remoteDescription = { + sdp: sdp, + }; + + stream.getTracks().forEach((track, index) => { + // @ts-ignore + call.onTrack({ + track, + streams: [stream], + transceiver: { mid: `${index}`, receiver: { track } }, + } as TrackEvent); + }); +}; + class MockMediaStreamAudioSourceNode { public connect() {} } @@ -227,7 +252,7 @@ export class MockRTCPeerConnection { let newSDP = this.localDescription.sdp; init?.streams?.forEach((stream) => { - newSDP += `m=${track.kind}\r\n`; + newSDP += `m=${track.kind === "audio" ? "audio 50609 UDP 126" : "video 9 UDP 114"} \r\n`; newSDP += `a=sendrecv\r\n`; newSDP += `a=mid:${this.transceivers.length}\r\n`; newSDP += `a=msid:${stream.id} ${track.id}\r\n`; @@ -240,9 +265,12 @@ export class MockRTCPeerConnection { return this.addTransceiver(track, { streams }).sender as unknown as MockRTCRtpSender; } - public removeTrack() { - this.needsNegotiation = true; - if (this.onReadyToNegotiate) this.onReadyToNegotiate(); + public removeTrack(sender: MockRTCRtpSender) { + const transceiver = this.transceivers.find((transceiver) => transceiver.sender === sender.typed()); + transceiver?.sender?.replaceTrack(null); + if (transceiver) { + transceiver.direction = transceiver?.direction === "sendrecv" ? "recvonly" : "inactive"; + } } public getTransceivers(): MockRTCRtpTransceiver[] { @@ -269,6 +297,10 @@ export class MockRTCRtpSender { public getParameters() {} public setParameters() {} + + public typed(): RTCRtpSender { + return this as unknown as RTCRtpSender; + } } export class MockRTCRtpReceiver { @@ -281,7 +313,15 @@ export class MockRTCRtpTransceiver { public sender?: RTCRtpSender; public receiver?: RTCRtpReceiver; - public set direction(_: string) { + private _direction = "sendrecv"; + + public get direction() { + return this._direction; + } + + public set direction(newDirection: string) { + if (this._direction === newDirection) return; + this._direction = newDirection; this.peerConn.needsNegotiation = true; } @@ -326,7 +366,7 @@ export class MockMediaStreamTrack { // XXX: Using EventTarget in jest doesn't seem to work, so we write our own // implementation export class MockMediaStream { - constructor(public id: string, private tracks: MockMediaStreamTrack[] = []) {} + constructor(public id: string = randomString(32), private tracks: MockMediaStreamTrack[] = []) {} public listeners: [string, (...args: any[]) => any][] = []; public isStopped = false; @@ -547,6 +587,7 @@ export class MockMatrixCall extends TypedEventEmitter(); public answerWithCallFeeds = jest.fn(); public hangup = jest.fn(); + public opponentSupportsSDPStreamMetadata = jest.fn(); public sendMetadataUpdate = jest.fn(); @@ -567,7 +608,7 @@ export class MockMatrixCall extends TypedEventEmitter !feed.isLocal()); + return this.feeds.filter((feed) => !feed.isLocal); } public typed(): MatrixCall { @@ -580,6 +621,7 @@ export class MockCallFeed { public userId: string, public deviceId: string | undefined, public stream: MockMediaStream, + public isLocal?: boolean, public purpose?: SDPStreamMetadataPurpose, ) {} @@ -589,7 +631,6 @@ export class MockCallFeed { public dispose() { this.disposed = true; } - public isLocal: () => boolean = jest.fn(); public typed(): CallFeed { return this as unknown as CallFeed; @@ -603,6 +644,8 @@ export function installWebRTCMocks() { } as unknown as Navigator; global.window = { + // @ts-ignore Mock + MediaStream: MockMediaStream, // @ts-ignore Mock RTCPeerConnection: MockRTCPeerConnection, // @ts-ignore Mock diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 1c56dc0b256..3f2861aafbf 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -39,12 +39,12 @@ import { MockMediaStreamTrack, installWebRTCMocks, MockRTCPeerConnection, - MockRTCRtpTransceiver, SCREENSHARE_STREAM_ID, - MockRTCRtpSender, + runOnTrackForStream, } from "../../test-utils/webrtc"; -import { CallFeed } from "../../../src/webrtc/callFeed"; import { EventType, IContent, ISendEventResponse, MatrixEvent, Room } from "../../../src"; +import { RemoteCallFeed } from "../../../src/webrtc/remoteCallFeed"; +import { LocalCallFeed } from "../../../src/webrtc/localCallFeed"; const FAKE_ROOM_ID = "!foo:bar"; const CALL_LIFETIME = 60000; @@ -80,17 +80,14 @@ const fakeIncomingCall = async (client: TestClient, call: MatrixCall, version: s getLocalAge: () => 1, } as unknown as MatrixEvent); call.getFeeds().push( - new CallFeed({ + new RemoteCallFeed({ client: client.client, - userId: "remote_user_id", - deviceId: undefined, - feedId: "remote_stream_id", + call: call, + streamId: "remote_stream_id", stream: new MockMediaStream("remote_stream_id", [ new MockMediaStreamTrack("remote_tack_id", "audio"), ]) as unknown as MediaStream, - purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: false, - videoMuted: false, + metadata: { purpose: SDPStreamMetadataPurpose.Usermedia }, }), ); await callPromise; @@ -178,6 +175,7 @@ describe("Call", function () { }), ); + // @ts-ignore const mockAddIceCandidate = (call.peerConn!.addIceCandidate = jest.fn()); call.onRemoteIceCandidatesReceived( makeMockEvent("@test:foo", { @@ -212,6 +210,7 @@ describe("Call", function () { it("should add candidates received before answer if party ID is correct", async function () { await startVoiceCall(client, call); + // @ts-ignore const mockAddIceCandidate = (call.peerConn!.addIceCandidate = jest.fn()); call.onRemoteIceCandidatesReceived( @@ -318,16 +317,12 @@ describe("Call", function () { }), ); - (call as any).pushRemoteStream( - new MockMediaStream("remote_stream", [ - new MockMediaStreamTrack("remote_audio_track", "audio"), - new MockMediaStreamTrack("remote_video_track", "video"), - ]), - ); - const feed = call.getFeeds().find((feed) => feed.stream?.id === "remote_stream"); + const feed = call.getFeeds().find((feed) => feed.streamId === "remote_stream"); expect(feed?.purpose).toBe(SDPStreamMetadataPurpose.Usermedia); - expect(feed?.isAudioMuted()).toBeTruthy(); - expect(feed?.isVideoMuted()).not.toBeTruthy(); + // @ts-ignore + expect(feed?.audioMuted).toBeTruthy(); + // @ts-ignore + expect(feed?.videoMuted).not.toBeTruthy(); }); it("should fallback to replaceTrack() if the other side doesn't support SPDStreamMetadata", async () => { @@ -385,22 +380,24 @@ describe("Call", function () { }), ); + const audioTrack = new MockMediaStreamTrack("new_audio_track", "audio"); + const videoTrack = new MockMediaStreamTrack("video_track", "video"); await call.updateLocalUsermediaStream( - new MockMediaStream("replacement_stream", [ - new MockMediaStreamTrack("new_audio_track", "audio"), - new MockMediaStreamTrack("video_track", "video"), - ]).typed(), + new MockMediaStream("replacement_stream", [audioTrack, videoTrack]).typed(), ); - // XXX: Lots of inspecting the prvate state of the call object here - const transceivers: Map = (call as any).transceivers; - - expect(call.localUsermediaStream!.id).toBe("stream"); expect(call.localUsermediaStream!.getAudioTracks()[0].id).toBe("new_audio_track"); expect(call.localUsermediaStream!.getVideoTracks()[0].id).toBe("video_track"); - // call has a function for generating these but we hardcode here to avoid exporting it - expect(transceivers.get("m.usermedia:audio")!.sender.track!.id).toBe("new_audio_track"); - expect(transceivers.get("m.usermedia:video")!.sender.track!.id).toBe("video_track"); + expect( + // @ts-ignore + call.peerConn?.getTransceivers().find((transceiver) => transceiver.sender.track?.kind === "audio")?.sender + .track, + ).toBe(audioTrack); + expect( + // @ts-ignore + call.peerConn?.getTransceivers().find((transceiver) => transceiver.sender.track?.kind === "video")?.sender + .track, + ).toBe(videoTrack); }); it("should handle upgrade to video call", async () => { @@ -422,27 +419,32 @@ describe("Call", function () { // setLocalVideoMuted probably? await (call as any).upgradeCall(false, true); - // XXX: More inspecting private state of the call object - const transceivers: Map = (call as any).transceivers; - expect(call.localUsermediaStream!.getAudioTracks()[0].id).toBe("usermedia_audio_track"); expect(call.localUsermediaStream!.getVideoTracks()[0].id).toBe("usermedia_video_track"); - expect(transceivers.get("m.usermedia:audio")!.sender.track!.id).toBe("usermedia_audio_track"); - expect(transceivers.get("m.usermedia:video")!.sender.track!.id).toBe("usermedia_video_track"); + expect( + // @ts-ignore + call.peerConn?.getTransceivers().find((transceiver) => transceiver.sender.track?.kind === "audio")?.sender + .track?.id, + ).toBe("usermedia_audio_track"); + expect( + // @ts-ignore + call.peerConn?.getTransceivers().find((transceiver) => transceiver.sender.track?.kind === "video")?.sender + .track?.id, + ).toBe("usermedia_video_track"); }); it("should handle SDPStreamMetadata changes", async () => { await startVoiceCall(client, call); - (call as any).updateRemoteSDPStreamMetadata({ + (call as any).onMetadata({ remote_stream: { purpose: SDPStreamMetadataPurpose.Usermedia, audio_muted: false, video_muted: false, }, }); - (call as any).pushRemoteStream(new MockMediaStream("remote_stream", [])); - const feed = call.getFeeds().find((feed) => feed.stream?.id === "remote_stream"); + //(call as any).pushRemoteStream(new MockMediaStream("remote_stream", [])); + const feed = call.getFeeds().find((feed) => feed.streamId === "remote_stream"); call.onSDPStreamMetadataChangedReceived( makeMockEvent("@test:foo", { @@ -522,14 +524,14 @@ describe("Call", function () { it("if no video", async () => { call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); - (call as any).pushRemoteStream(new MockMediaStream("remote_stream1", [])); + (call as any).addRemoteFeedWithoutMetadata(new MockMediaStream("remote_stream1", [])); expect(call.type).toBe(CallType.Voice); }); it("if remote video", async () => { call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); - (call as any).pushRemoteStream( + (call as any).addRemoteFeedWithoutMetadata( new MockMediaStream("remote_stream1", [new MockMediaStreamTrack("track_id", "video")]), ); expect(call.type).toBe(CallType.Video); @@ -541,40 +543,33 @@ describe("Call", function () { // since this is testing for the presence of a local sender, we need to add a transciever // rather than just a source track const mockTrack = new MockMediaStreamTrack("track_id", "video"); - const mockTransceiver = new MockRTCRtpTransceiver(call.peerConn as unknown as MockRTCPeerConnection, "0"); - mockTransceiver.sender = new MockRTCRtpSender(mockTrack) as unknown as RTCRtpSender; - (call as any).transceivers.set("m.usermedia:video", mockTransceiver); - (call as any).pushNewLocalFeed( + (call as any).addLocalFeedFromStream( new MockMediaStream("remote_stream1", [mockTrack]), SDPStreamMetadataPurpose.Usermedia, false, ); + call.getLocalFeeds()[0].publish(call); expect(call.type).toBe(CallType.Video); }); }); it("should correctly generate local SDPStreamMetadata", async () => { const callPromise = call.placeCallWithCallFeeds([ - new CallFeed({ + new LocalCallFeed({ client: client.client, stream: new MockMediaStream("local_stream1", [ new MockMediaStreamTrack("track_id", "audio"), ]) as unknown as MediaStream, roomId: call.roomId, - userId: client.getUserId(), - deviceId: undefined, - feedId: "local_stream1", purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: false, - videoMuted: false, }), ]); await client.httpBackend!.flush(""); await callPromise; call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); - (call as any).pushNewLocalFeed( + (call as any).addLocalFeedFromStream( new MockMediaStream("local_stream2", [ new MockMediaStreamTrack("track_id", "video"), ]) as unknown as MediaStream, @@ -582,7 +577,7 @@ describe("Call", function () { ); await call.setMicrophoneMuted(true); - expect((call as any).getLocalSDPStreamMetadata()).toStrictEqual({ + expect((call as any).metadata).toStrictEqual({ local_stream1: expect.objectContaining({ purpose: SDPStreamMetadataPurpose.Usermedia, audio_muted: true, @@ -603,32 +598,22 @@ describe("Call", function () { const remoteScreensharingStream = new MockMediaStream("remote_screensharing_stream_id", []); const callPromise = call.placeCallWithCallFeeds([ - new CallFeed({ + new LocalCallFeed({ client: client.client, - userId: client.getUserId(), - deviceId: undefined, - feedId: localUsermediaStream.id, stream: localUsermediaStream as unknown as MediaStream, purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: false, - videoMuted: false, }), - new CallFeed({ + new LocalCallFeed({ client: client.client, - userId: client.getUserId(), - deviceId: undefined, - feedId: localScreensharingStream.id, stream: localScreensharingStream as unknown as MediaStream, purpose: SDPStreamMetadataPurpose.Screenshare, - audioMuted: false, - videoMuted: false, }), ]); await client.httpBackend!.flush(""); await callPromise; call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); - (call as any).updateRemoteSDPStreamMetadata({ + (call as any).onMetadata({ remote_usermedia_stream_id: { purpose: SDPStreamMetadataPurpose.Usermedia, audio_muted: false, @@ -641,17 +626,18 @@ describe("Call", function () { video_muted: false, }, }); - (call as any).pushRemoteStream(remoteUsermediaStream); - (call as any).pushRemoteStream(remoteScreensharingStream); + + runOnTrackForStream(call, remoteUsermediaStream.typed()); + runOnTrackForStream(call, remoteScreensharingStream.typed()); expect(call.localUsermediaFeed!.stream).toBe(localUsermediaStream); expect(call.localUsermediaStream).toBe(localUsermediaStream); expect(call.localScreensharingFeed!.stream).toBe(localScreensharingStream); expect(call.localScreensharingStream).toBe(localScreensharingStream); - expect(call.remoteUsermediaFeed!.stream).toBe(remoteUsermediaStream); - expect(call.remoteUsermediaStream).toBe(remoteUsermediaStream); - expect(call.remoteScreensharingFeed!.stream).toBe(remoteScreensharingStream); - expect(call.remoteScreensharingStream).toBe(remoteScreensharingStream); + expect(call.remoteUsermediaFeed?.stream?.getTracks()).toStrictEqual(remoteUsermediaStream.getTracks()); + expect(call.remoteUsermediaStream?.getTracks()).toStrictEqual(remoteUsermediaStream.getTracks()); + expect(call.remoteScreensharingFeed?.stream?.getTracks()).toStrictEqual(remoteScreensharingStream.getTracks()); + expect(call.remoteScreensharingStream?.getTracks()).toStrictEqual(remoteScreensharingStream.getTracks()); expect(call.hasRemoteUserMediaAudioTrack).toBe(false); }); @@ -774,41 +760,21 @@ describe("Call", function () { call.off(CallEvent.FeedsChanged, FEEDS_CHANGED_CALLBACK); }); - it("should ignore stream passed to pushRemoteStream()", async () => { - await call.onAnswerReceived( - makeMockEvent("@test:foo", { - version: 1, - call_id: call.callId, - party_id: "party_id", - answer: { - sdp: DUMMY_SDP, - }, - [SDPStreamMetadataKey]: { - [STREAM_ID]: { - purpose: SDPStreamMetadataPurpose.Usermedia, - }, - }, - }), - ); - - (call as any).pushRemoteStream(new MockMediaStream(STREAM_ID)); - (call as any).pushRemoteStream(new MockMediaStream(STREAM_ID)); - - expect(call.getRemoteFeeds().length).toBe(1); - expect(FEEDS_CHANGED_CALLBACK).toHaveBeenCalledTimes(1); - }); - - it("should ignore stream passed to pushRemoteStreamWithoutMetadata()", async () => { - (call as any).pushRemoteStreamWithoutMetadata(new MockMediaStream(STREAM_ID)); - (call as any).pushRemoteStreamWithoutMetadata(new MockMediaStream(STREAM_ID)); + it("should ignore stream passed to addRemoteFeedWithoutMetadata()", async () => { + const stream = new MockMediaStream(STREAM_ID); + try { + (call as any).addRemoteFeedWithoutMetadata(stream); + (call as any).addRemoteFeedWithoutMetadata(stream); + } catch (e) {} expect(call.getRemoteFeeds().length).toBe(1); expect(FEEDS_CHANGED_CALLBACK).toHaveBeenCalledTimes(1); }); - it("should ignore stream passed to pushNewLocalFeed()", async () => { - (call as any).pushNewLocalFeed(new MockMediaStream(STREAM_ID), SDPStreamMetadataPurpose.Screenshare); - (call as any).pushNewLocalFeed(new MockMediaStream(STREAM_ID), SDPStreamMetadataPurpose.Screenshare); + it("should ignore stream passed to addLocalFeedFromStream()", async () => { + const stream = new MockMediaStream(STREAM_ID); + (call as any).addLocalFeedFromStream(stream, SDPStreamMetadataPurpose.Screenshare); + (call as any).addLocalFeedFromStream(stream, SDPStreamMetadataPurpose.Screenshare); // We already have one local feed from placeVoiceCall() expect(call.getLocalFeeds().length).toBe(2); @@ -920,12 +886,6 @@ describe("Call", function () { describe("receiving sdp_stream_metadata_changed events", () => { const setupCall = (audio: boolean, video: boolean): void => { - (call as any).pushRemoteStream( - new MockMediaStream("stream", [ - new MockMediaStreamTrack("track1", "audio"), - new MockMediaStreamTrack("track1", "video"), - ]), - ); call.onSDPStreamMetadataChangedReceived({ getContent: () => ({ [SDPStreamMetadataKey]: { @@ -935,11 +895,40 @@ describe("Call", function () { purpose: SDPStreamMetadataPurpose.Usermedia, audio_muted: audio, video_muted: video, - tracks: {}, }, }, }), } as MatrixEvent); + const stream = new MockMediaStream("stream", [ + new MockMediaStreamTrack("track1", "audio"), + new MockMediaStreamTrack("track2", "video"), + ]); + let sdp = + "v=0\n" + + "o=- 7135465365607179083 2 IN IP4 127.0.0.1\n" + + "s=-\n" + + "t=0 0\n" + + "a=group:BUNDLE 0 1 2\n"; + + stream.getTracks().forEach((track, index) => { + sdp += `m=${track.kind}\n`; + sdp += `a=mid:${index}\n`; + sdp += `a=msid:${stream.id} ${track.id}\n`; + }); + + // @ts-ignore + call.peerConn.remoteDescription = { + sdp: sdp, + }; + + stream.getTracks().forEach((track, index) => { + // @ts-ignore + call.onTrack({ + track, + streams: [stream], + transceiver: { mid: `${index}`, receiver: { track } }, + } as TrackEvent); + }); }; it("should handle incoming sdp_stream_metadata_changed with audio muted", async () => { @@ -1085,6 +1074,7 @@ describe("Call", function () { beforeEach(async () => { await call.answer(); await untilEventSent(FAKE_ROOM_ID, EventType.CallAnswer, expect.objectContaining({})); + // @ts-ignore mockPeerConn = call.peerConn as unknown as MockRTCPeerConnection; }); @@ -1267,6 +1257,7 @@ describe("Call", function () { await sendNegotiatePromise; + // @ts-ignore const mockPeerConn = call.peerConn as unknown as MockRTCPeerConnection; expect( mockPeerConn.transceivers[mockPeerConn.transceivers.length - 1].setCodecPreferences, @@ -1274,6 +1265,7 @@ describe("Call", function () { }); it("re-uses transceiver when screen sharing is re-enabled", async () => { + // @ts-ignore const mockPeerConn = call.peerConn as unknown as MockRTCPeerConnection; // sanity check: we should start with one transciever (user media audio) @@ -1324,7 +1316,8 @@ describe("Call", function () { MockRTCPeerConnection.triggerAllNegotiations(); - const mockVideoSender = call.peerConn!.getSenders().find((s) => s.track!.kind === "video"); + // @ts-ignore + const mockVideoSender = call.peerConn.getSenders().find((s) => s.track!.kind === "video"); const mockReplaceTrack = (mockVideoSender!.replaceTrack = jest.fn()); await call.setScreensharingEnabled(true); @@ -1453,23 +1446,24 @@ describe("Call", function () { describe("onTrack", () => { it("ignores streamless track", async () => { - // @ts-ignore Mock pushRemoteStream() is private - jest.spyOn(call, "pushRemoteStream"); + // @ts-ignore Mock addRemoteFeedWithoutMetadata() is private + jest.spyOn(call, "addRemoteFeedWithoutMetadata"); await call.placeVoiceCall(); + // @ts-ignore (call.peerConn as unknown as MockRTCPeerConnection).onTrackListener!({ streams: [], track: new MockMediaStreamTrack("track_ev", "audio"), } as unknown as RTCTrackEvent); - // @ts-ignore Mock pushRemoteStream() is private - expect(call.pushRemoteStream).not.toHaveBeenCalled(); + // @ts-ignore Mock addRemoteFeedWithoutMetadata() is private + expect(call.addRemoteFeedWithoutMetadata).not.toHaveBeenCalled(); }); it("correctly pushes", async () => { - // @ts-ignore Mock pushRemoteStream() is private - jest.spyOn(call, "pushRemoteStream"); + // @ts-ignore Mock addRemoteFeedWithoutMetadata() is private + jest.spyOn(call, "addRemoteFeedWithoutMetadata"); await call.placeVoiceCall(); await call.onAnswerReceived( @@ -1484,14 +1478,16 @@ describe("Call", function () { ); const stream = new MockMediaStream("stream_ev", [new MockMediaStreamTrack("track_ev", "audio")]); + // @ts-ignore (call.peerConn as unknown as MockRTCPeerConnection).onTrackListener!({ streams: [stream], track: stream.getAudioTracks()[0], + transceiver: { mid: "0" }, } as unknown as RTCTrackEvent); - // @ts-ignore Mock pushRemoteStream() is private - expect(call.pushRemoteStream).toHaveBeenCalledWith(stream); - // @ts-ignore Mock pushRemoteStream() is private + // @ts-ignore Mock addRemoteFeedWithoutMetadata() is private + expect(call.addRemoteFeedWithoutMetadata).toHaveBeenCalledWith(stream); + // @ts-ignore Mock removeTrackListeners() is private expect(call.removeTrackListeners.has(stream)).toBe(true); }); }); @@ -1575,6 +1571,7 @@ describe("Call", function () { jest.useFakeTimers(); call.addListener(CallEvent.LengthChanged, lengthChangedListener); await fakeIncomingCall(client, call, "1"); + // @ts-ignore (call.peerConn as unknown as MockRTCPeerConnection).iceConnectionStateChangeListener!(); let hasAdvancedBy = 0; @@ -1596,6 +1593,7 @@ describe("Call", function () { await fakeIncomingCall(client, call, "1"); + // @ts-ignore mockPeerConn = call.peerConn as unknown as MockRTCPeerConnection; mockPeerConn.iceConnectionState = "disconnected"; mockPeerConn.iceConnectionStateChangeListener!(); diff --git a/spec/unit/webrtc/callFeed.spec.ts b/spec/unit/webrtc/callFeed.spec.ts index c648f7aa066..d1b6cea6154 100644 --- a/spec/unit/webrtc/callFeed.spec.ts +++ b/spec/unit/webrtc/callFeed.spec.ts @@ -17,8 +17,10 @@ limitations under the License. import { SDPStreamMetadataPurpose } from "../../../src/webrtc/callEventTypes"; import { CallFeed } from "../../../src/webrtc/callFeed"; import { TestClient } from "../../TestClient"; -import { MockMatrixCall, MockMediaStream, MockMediaStreamTrack } from "../../test-utils/webrtc"; +import { installWebRTCMocks, MockMatrixCall, MockMediaStream, MockMediaStreamTrack } from "../../test-utils/webrtc"; import { CallEvent, CallState } from "../../../src/webrtc/call"; +import { LocalCallFeed } from "../../../src/webrtc/localCallFeed"; +import { RemoteCallFeed } from "../../../src/webrtc/remoteCallFeed"; describe("CallFeed", () => { const roomId = "room1"; @@ -27,10 +29,12 @@ describe("CallFeed", () => { let feed: CallFeed; beforeEach(() => { + installWebRTCMocks(); + client = new TestClient("@alice:foo", "somedevice", "token", undefined, {}); call = new MockMatrixCall(roomId); - feed = new CallFeed({ + feed = new LocalCallFeed({ client: client.client, call: call.typed(), roomId, @@ -90,10 +94,9 @@ describe("CallFeed", () => { }); describe("connected", () => { - it.each([true, false])("should always be connected, if isLocal()", (val: boolean) => { + it.each([true, false])("should always be connected, if isLocal", (val: boolean) => { // @ts-ignore feed._connected = val; - jest.spyOn(feed, "isLocal").mockReturnValue(true); expect(feed.connected).toBeTruthy(); }); @@ -102,11 +105,19 @@ describe("CallFeed", () => { [CallState.Connected, true], [CallState.Connecting, false], ])("should react to call state, when !isLocal()", (state: CallState, expected: Boolean) => { - feed.stream?.addTrack(new MockMediaStreamTrack("track1", "video").typed()); + jest.spyOn(call, "opponentSupportsSDPStreamMetadata").mockReturnValue(false); + + const remoteFeed = new RemoteCallFeed({ + client: client.client, + call: call.typed(), + streamId: "id", + }); + + remoteFeed.stream?.addTrack(new MockMediaStreamTrack("track1", "video").typed()); call.state = state; call.emit(CallEvent.State, state); - expect(feed.connected).toBe(expected); + expect(remoteFeed.connected).toBe(expected); }); }); }); diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index d54d18c0469..64a1822899a 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -34,7 +34,7 @@ import { FAKE_USER_ID_2, FAKE_DEVICE_ID_1, FAKE_SESSION_ID_1, - FAKE_USER_ID_3, + runOnTrackForStream, } from "../../test-utils/webrtc"; import { SDPStreamMetadataKey, SDPStreamMetadataPurpose } from "../../../src/webrtc/callEventTypes"; import { sleep } from "../../../src/utils"; @@ -42,6 +42,7 @@ import { CallEventHandlerEvent } from "../../../src/webrtc/callEventHandler"; import { CallFeed } from "../../../src/webrtc/callFeed"; import { CallEvent, CallState } from "../../../src/webrtc/call"; import { flushPromises } from "../../test-utils/flushPromises"; +import { RemoteCallFeed } from "../../../src/webrtc/remoteCallFeed"; const FAKE_STATE_EVENTS = [ { @@ -168,13 +169,13 @@ describe("Group Call", function () { groupCall.leave(); }); - it("does not start initializing local call feed twice", () => { + it("does not start initializing local call feed twice", async () => { const promise1 = groupCall.initLocalCallFeed(); // @ts-ignore Mock groupCall.state = GroupCallState.LocalCallFeedUninitialized; const promise2 = groupCall.initLocalCallFeed(); - expect(promise1).toEqual(promise2); + expect(await promise1).toEqual(await promise2); }); it("sets state to local call feed uninitialized when getUserMedia() fails", async () => { @@ -376,8 +377,6 @@ describe("Group Call", function () { beforeEach(async () => { jest.spyOn(currentFeed, "dispose"); jest.spyOn(newFeed, "measureVolumeActivity"); - jest.spyOn(currentFeed, "isLocal").mockReturnValue(false); - jest.spyOn(newFeed, "isLocal").mockReturnValue(false); jest.spyOn(groupCall, "emit"); @@ -753,11 +752,13 @@ describe("Group Call", function () { while ( // @ts-ignore (newCall = groupCall1.calls.get(client2.userId)?.get(client2.deviceId)) === undefined || + // @ts-ignore newCall.peerConn === undefined || newCall.callId == oldCall.callId ) { await flushPromises(); } + // @ts-ignore const mockPc = newCall.peerConn as unknown as MockRTCPeerConnection; // ...then wait for it to be ready to negotiate @@ -922,13 +923,13 @@ describe("Group Call", function () { const call = groupCall.calls.get(FAKE_USER_ID_2)!.get(FAKE_DEVICE_ID_2)!; call.getOpponentMember = () => ({ userId: call.invitee } as RoomMember); call.onSDPStreamMetadataChangedReceived(metadataEvent); - // @ts-ignore Mock - call.pushRemoteStream( - // @ts-ignore Mock + + runOnTrackForStream( + call, new MockMediaStream("stream", [ new MockMediaStreamTrack("audio_track", "audio"), new MockMediaStreamTrack("video_track", "video"), - ]), + ]).typed(), ); const feed = groupCall.getUserMediaFeed(call.invitee!, call.getOpponentDeviceId()!); @@ -939,6 +940,10 @@ describe("Group Call", function () { }); it("should mute remote feed's video after receiving metadata with video muted", async () => { + const stream = new MockMediaStream("stream", [ + new MockMediaStreamTrack("track1", "audio"), + new MockMediaStreamTrack("track2", "video"), + ]); const metadataEvent = getMetadataEvent(false, true); const groupCall = await createAndEnterGroupCall(mockClient, room); @@ -949,14 +954,33 @@ describe("Group Call", function () { const call = groupCall.calls.get(FAKE_USER_ID_2).get(FAKE_DEVICE_ID_2)!; call.getOpponentMember = () => ({ userId: call.invitee } as RoomMember); call.onSDPStreamMetadataChangedReceived(metadataEvent); - // @ts-ignore Mock - call.pushRemoteStream( - // @ts-ignore Mock - new MockMediaStream("stream", [ - new MockMediaStreamTrack("audio_track", "audio"), - new MockMediaStreamTrack("video_track", "video"), - ]), - ); + + let sdp = + "v=0\n" + + "o=- 7135465365607179083 2 IN IP4 127.0.0.1\n" + + "s=-\n" + + "t=0 0\n" + + "a=group:BUNDLE 0 1 2\n"; + + stream.getTracks().forEach((track, index) => { + sdp += `m=${track.kind}\n`; + sdp += `a=mid:${index}\n`; + sdp += `a=msid:${stream.id} ${track.id}\n`; + }); + + // @ts-ignore + call.peerConn.remoteDescription = { + sdp: sdp, + }; + + stream.getTracks().forEach((track, index) => { + // @ts-ignore + call.onTrack({ + track, + streams: [stream], + transceiver: { mid: `${index}`, receiver: { track } }, + } as TrackEvent); + }); const feed = groupCall.getUserMediaFeed(call.invitee!, call.getOpponentDeviceId()!); expect(feed!.isAudioMuted()).toBe(false); @@ -1207,11 +1231,6 @@ describe("Group Call", function () { }, }), } as MatrixEvent); - // @ts-ignore Mock - call.pushRemoteStream( - // @ts-ignore Mock - new MockMediaStream("screensharing_stream", [new MockMediaStreamTrack("video_track", "video")]), - ); expect(groupCall.screenshareFeeds).toHaveLength(1); expect(groupCall.getScreenshareFeed(call.invitee!, call.getOpponentDeviceId()!)).toBeDefined(); @@ -1250,6 +1269,7 @@ describe("Group Call", function () { jest.useFakeTimers(); const mockClient = new MockCallMatrixClient(FAKE_USER_ID_1, FAKE_DEVICE_ID_1, FAKE_SESSION_ID_1); + const mockCall = new MockMatrixCall(FAKE_ROOM_ID).typed(); room = new Room(FAKE_ROOM_ID, mockClient.typed(), FAKE_USER_ID_1); room.currentState.members[FAKE_USER_ID_1] = { @@ -1257,29 +1277,19 @@ describe("Group Call", function () { } as unknown as RoomMember; groupCall = await createAndEnterGroupCall(mockClient.typed(), room); - mediaFeed1 = new CallFeed({ + mediaFeed1 = new RemoteCallFeed({ client: mockClient.typed(), roomId: FAKE_ROOM_ID, - userId: FAKE_USER_ID_2, - deviceId: FAKE_DEVICE_ID_1, - feedId: "foo", - stream: new MockMediaStream("foo", []).typed(), - purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: false, - videoMuted: true, + streamId: "stream1", + call: mockCall, }); groupCall.userMediaFeeds.push(mediaFeed1); - mediaFeed2 = new CallFeed({ + mediaFeed2 = new RemoteCallFeed({ client: mockClient.typed(), roomId: FAKE_ROOM_ID, - userId: FAKE_USER_ID_3, - deviceId: FAKE_DEVICE_ID_1, - feedId: "foo", - stream: new MockMediaStream("foo", []).typed(), - purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: false, - videoMuted: true, + streamId: "stream2", + call: mockCall, }); groupCall.userMediaFeeds.push(mediaFeed2); diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 6d044d6defa..c0bd8785498 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -22,7 +22,7 @@ limitations under the License. */ import { v4 as uuidv4 } from "uuid"; -import { parse as parseSdp, write as writeSdp } from "sdp-transform"; +import { parse as parseSdp, SessionDescription, write as writeSdp } from "sdp-transform"; import { logger } from "../logger"; import * as utils from "../utils"; @@ -48,7 +48,6 @@ import { FocusTrackSubscriptionEvent, FocusNegotiateEvent, FocusSDPStreamMetadataChangedEvent, - SDPStreamMetadataTracks, FocusEvent, SDPStreamMetadataKeyStable, FocusEventBaseContent, @@ -60,6 +59,9 @@ import { DeviceInfo } from "../crypto/deviceinfo"; import { IScreensharingOpts } from "./mediaHandler"; import { GroupCallUnknownDeviceError } from "./groupCall"; import { MatrixError } from "../http-api"; +import { RemoteCallFeed } from "./remoteCallFeed"; +import { LocalCallFeed } from "./localCallFeed"; +import { LocalCallTrack } from "./localCallTrack"; interface CallOpts { // The room ID for this call. @@ -93,12 +95,6 @@ interface AssertedIdentity { displayName: string; } -export enum SimulcastResolution { - Full = "f", - Half = "h", - Quarter = "q", -} - enum MediaType { AUDIO = "audio", VIDEO = "video", @@ -279,59 +275,6 @@ const ICE_DISCONNECTED_TIMEOUT = 30 * 1000; // ms */ const SUBSCRIBE_TO_FOCUS_TIMEOUT = 2 * 1000; -// Order is important here: some browsers (e.g. -// Chrome) will only send some of the encodings, if -// the track has a resolution to low for it to send -// all, in that case the encoding higher in the list -// has priority and therefore we put full as first -// as we always want to send the full resolution -const SIMULCAST_USERMEDIA_ENCODINGS: RTCRtpEncodingParameters[] = [ - { - // 720p (base) - maxFramerate: 30, - maxBitrate: 1_700_000, - rid: SimulcastResolution.Full, - }, - { - // 360p - maxFramerate: 20, - maxBitrate: 300_000, - rid: SimulcastResolution.Half, - scaleResolutionDownBy: 2.0, - }, - { - // 180p - maxFramerate: 15, - maxBitrate: 120_000, - - rid: SimulcastResolution.Quarter, - scaleResolutionDownBy: 4.0, - }, -]; - -const SIMULCAST_SCREENSHARING_ENCODINGS: RTCRtpEncodingParameters[] = [ - { - // 1080p (base) - maxFramerate: 30, - maxBitrate: 3_000_000, - rid: SimulcastResolution.Full, - }, - { - // 720p - maxFramerate: 15, - maxBitrate: 1_000_000, - rid: SimulcastResolution.Half, - scaleResolutionDownBy: 1.5, - }, - { - // 360p - maxFramerate: 3, - maxBitrate: 200_000, - rid: SimulcastResolution.Quarter, - scaleResolutionDownBy: 3, - }, -]; - export class CallError extends Error { public readonly code: string; @@ -343,18 +286,6 @@ export class CallError extends Error { } } -export const getSimulcastEncodings = (purpose: SDPStreamMetadataPurpose): RTCRtpEncodingParameters[] => { - if (purpose === SDPStreamMetadataPurpose.Usermedia) { - return SIMULCAST_USERMEDIA_ENCODINGS; - } - if (purpose === SDPStreamMetadataPurpose.Screenshare) { - return SIMULCAST_SCREENSHARING_ENCODINGS; - } - - // Fallback to usermedia encodings - return SIMULCAST_USERMEDIA_ENCODINGS; -}; - export function genCallID(): string { return Date.now().toString() + randomString(16); } @@ -392,16 +323,6 @@ export type CallEventHandlerMap = { [CallEvent.SendVoipEvent]: (event: Record) => void; }; -// The key of the transceiver map (purpose + media type, separated by ':') -type TransceiverKey = string; - -// generates keys for the map of transceivers -// kind is unfortunately a string rather than MediaType as this is the type of -// track.kind -function getTransceiverKey(purpose: SDPStreamMetadataPurpose, kind: TransceiverKey): string { - return purpose + ":" + kind; -} - export class MatrixCall extends TypedEventEmitter { public roomId?: string; public callId: string; @@ -410,7 +331,6 @@ export class MatrixCall extends TypedEventEmitter; @@ -430,9 +351,6 @@ export class MatrixCall extends TypedEventEmitter = []; - // our transceivers for each purpose and type of media - private transceivers = new Map(); - private inviteOrAnswerSent = false; private waitForLocalAVStream = false; private successor?: MatrixCall; @@ -569,6 +487,22 @@ export class MatrixCall extends TypedEventEmitter t.kind === "audio")?.sender); } private get hasUserMediaVideoSender(): boolean { - return Boolean(this.transceivers.get(getTransceiverKey(SDPStreamMetadataPurpose.Usermedia, "video"))?.sender); + return Boolean(this.localUsermediaFeed?.tracks?.find((t) => t.kind === "video")?.sender); } - public get localUsermediaFeed(): CallFeed | undefined { + public get localUsermediaFeed(): LocalCallFeed | undefined { return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); } - public get localScreensharingFeed(): CallFeed | undefined { + public get localScreensharingFeed(): LocalCallFeed | undefined { return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); } @@ -629,11 +563,11 @@ export class MatrixCall extends TypedEventEmitter feed.purpose === SDPStreamMetadataPurpose.Usermedia); } - public get remoteScreensharingFeed(): CallFeed | undefined { + public get remoteScreensharingFeed(): RemoteCallFeed | undefined { return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); } @@ -645,8 +579,68 @@ export class MatrixCall extends TypedEventEmitter feed.feedId === feedId); + public getFreeTransceiverByKind(kind: string): RTCRtpTransceiver | undefined { + return this.peerConn?.getTransceivers().find((transceiver) => { + if (transceiver.sender.track) return false; + if (!transceiver.mid) return false; + if (this.getLocalMediaTypeByMid(transceiver.mid) !== kind) return false; + + return true; + }); + } + + public getLocalMediaLineByMid(mid: string): SessionDescription["media"][number] | undefined { + return this.localSDP?.media?.find((m) => m.mid == mid); + } + + public getLocalMediaTypeByMid(mid: string): string | undefined { + return this.getLocalMediaLineByMid(mid)?.type; + } + + public getLocalMSIDByMid(mid: string): string[] | undefined { + return this.getLocalMediaLineByMid(mid)?.msid?.split(" "); + } + + public getLocalStreamIdByMid(mid: string): string | undefined { + return this.getLocalMSIDByMid(mid)?.[0]; + } + + public getLocalTrackIdByMid(mid: string): string | undefined { + return this.getLocalMSIDByMid(mid)?.[1]; + } + + public getRemoteMediaLineByMid(mid: string): SessionDescription["media"][number] | undefined { + return this.remoteSDP?.media?.find((m) => m.mid == mid); + } + + public getRemoteMediaTypeByMid(mid: string): string | undefined { + return this.getRemoteMediaLineByMid(mid)?.type; + } + + public getRemoteMSIDByMid(mid: string): string[] | undefined { + return this.getRemoteMediaLineByMid(mid)?.msid?.split(" "); + } + + public getRemoteStreamIdByMid(mid: string): string | undefined { + return this.getRemoteMSIDByMid(mid)?.[0]; + } + + public getRemoteTrackIdByMid(mid: string): string | undefined { + return this.getRemoteMSIDByMid(mid)?.[1]; + } + + public getRemoteTrackInfoByMid(mid: string): string { + return `streamId=${this.getRemoteStreamIdByMid(mid)}, trackId=${this.getRemoteTrackIdByMid( + mid, + )}, mid=${mid}, kind=${this.getRemoteMediaTypeByMid(mid)}`; + } + + private getLocalFeedById(feedId: string): LocalCallFeed | undefined { + return this.getLocalFeeds().find((feed) => feed.id === feedId); + } + + private getLocalFeedByStream(stream: MediaStream): LocalCallFeed | undefined { + return this.getLocalFeeds().find((feed) => feed.stream === stream); } /** @@ -661,16 +655,16 @@ export class MatrixCall extends TypedEventEmitter { - return this.feeds.filter((feed) => feed.isLocal()); + public getLocalFeeds(): Array { + return this.feeds.filter((feed) => feed instanceof LocalCallFeed) as LocalCallFeed[]; } /** * Returns an array of all remote CallFeeds * @returns remote CallFeeds */ - public getRemoteFeeds(): Array { - return this.feeds.filter((feed) => !feed.isLocal()); + public getRemoteFeeds(): Array { + return this.feeds.filter((feed) => feed instanceof RemoteCallFeed) as RemoteCallFeed[]; } private async initOpponentCrypto(): Promise { @@ -704,54 +698,13 @@ export class MatrixCall extends TypedEventEmitter { - if (!transceiver.sender.track) return tracks; - if ( - ![ - getTransceiverKey(localFeed.purpose, "audio"), - getTransceiverKey(localFeed.purpose, "video"), - ].includes(transceiverKey) - ) { - return tracks; - } - - // XXX: We only use double equals because MediaDescription::mid is in fact a number - const msid = sdp?.media?.find((m) => m.mid == transceiver.mid)?.msid?.split(" "); - if (msid?.[0]) { - streamId = msid?.[0]; - } - if (msid?.[1]) { - tracks[msid[1]] = { - kind: transceiver.sender.track?.kind, - width: transceiver.sender.track.getSettings().width, - height: transceiver.sender.track.getSettings().height, - }; - } - return tracks; - }, {} as SDPStreamMetadataTracks); - if (!Object.keys(tracks).length) continue; + private get metadata(): SDPStreamMetadata { + return this.getLocalFeeds().reduce((metadata: SDPStreamMetadata, feed: LocalCallFeed) => { + if (!feed.streamId) return metadata; - metadata[streamId] = { - // FIXME: This allows for impersonation - the focus should be - // handling these - user_id: this.client.getUserId()!, - device_id: this.client.getDeviceId()!, - purpose: localFeed.purpose, - // FIXME: This is very ineffective as state is slow, we should really be sending this over DC - audio_muted: localFeed.isAudioMuted(), - video_muted: localFeed.isVideoMuted(), - tracks: tracks, - }; - } - return metadata; + metadata[feed.streamId] = feed.metadata; + return metadata; + }, {}); } /** @@ -760,105 +713,31 @@ export class MatrixCall extends TypedEventEmitter !feed.isLocal()); - } - - private pushRemoteStream(stream: MediaStream): void { - // Fallback to old behavior if the other side doesn't support SDPStreamMetadata - if (!this.opponentSupportsSDPStreamMetadata()) { - this.pushRemoteStreamWithoutMetadata(stream); - return; - } - - const feed = this.getFeedById(stream.id); - if (!feed) { - logger.warn( - `Call ${this.callId} Ignoring stream because we don't have a feed for it (streamId=${stream.id})`, - ); - return; - } - - feed.setNewStream(stream); - - logger.info( - `Call ${this.callId} Pushed remote stream (feedId=${feed.feedId}, active=${stream.active}, purpose=${feed.purpose}, userId=${feed.userId}, deviceId=${feed.deviceId})`, - ); + return !this.feeds.some((feed) => !feed.isLocal); } - /** - * This method is used ONLY if the other client doesn't support sending SDPStreamMetadata - */ - private pushRemoteStreamWithoutMetadata(stream: MediaStream): void { - const userId = this.getOpponentMember()!.userId; - // We can guess the purpose here since the other client can only send one stream - const purpose = SDPStreamMetadataPurpose.Usermedia; - const oldRemoteStream = this.feeds.find((feed) => !feed.isLocal())?.stream; - - // Note that we check by ID and always set the remote stream: Chrome appears - // to make new stream objects when transceiver directionality is changed and the 'active' - // status of streams change - Dave - // If we already have a stream, check this stream has the same id - if (oldRemoteStream && stream.id !== oldRemoteStream.id) { - logger.warn( - `Call ${this.callId} pushRemoteFeedWithoutMetadata() ignoring new stream because we already have stream (streamId=${stream.id})`, - ); - return; - } - - if (this.getFeedById(stream.id)) { + private addLocalFeedFromStream( + stream: MediaStream, + purpose: SDPStreamMetadataPurpose, + addToPeerConnection = true, + ): void { + if (this.getLocalFeedByStream(stream)) { logger.warn( - `Call ${this.callId} pushRemoteFeedWithoutMetadata() ignoring stream because we already have a feed for it (streamId=${stream.id})`, + `Call ${this.callId} addLocalFeedFromStream() ignoring stream for which we already have a feed (streamId=${stream.id})`, ); return; } - this.feeds.push( - new CallFeed({ - client: this.client, - call: this, - roomId: this.roomId, - audioMuted: false, - videoMuted: false, - userId, - deviceId: this.getOpponentDeviceId(), - feedId: stream.id, - stream, - purpose, - }), - ); - - this.emit(CallEvent.FeedsChanged, this.feeds); - - logger.info( - `Call ${this.callId} pushRemoteFeedWithoutMetadata() pushed stream (streamId=${stream.id}, active=${stream.active})`, - ); - } - - private pushNewLocalFeed(stream: MediaStream, purpose: SDPStreamMetadataPurpose, addToPeerConnection = true): void { - const userId = this.client.getUserId()!; - // Tracks don't always start off enabled, eg. chrome will give a disabled // audio track if you ask for user media audio and already had one that // you'd set to disabled (presumably because it clones them internally). setTracksEnabled(stream.getAudioTracks(), true); setTracksEnabled(stream.getVideoTracks(), true); - if (this.getFeedById(stream.id)) { - logger.warn( - `Call ${this.callId} pushNewLocalFeed() ignoring stream because we already have a feed for it (streamId=${stream.id})`, - ); - return; - } - this.pushLocalFeed( - new CallFeed({ + new LocalCallFeed({ client: this.client, roomId: this.roomId, - audioMuted: false, - videoMuted: false, - userId, - deviceId: this.getOpponentDeviceId(), - feedId: stream.id, stream, purpose, }), @@ -871,165 +750,91 @@ export class MatrixCall extends TypedEventEmitter { - const purpose = callFeed.purpose; - const feedId = callFeed.feedId; - const stream = callFeed.stream; - if (!stream) { - logger.warn( - `Call ${this.callId} addTracksOfFeedToPeerConnection() failed: no stream (feedId=${feedId} purpose=${purpose})`, - ); - return; + public publishTrack(track: LocalCallTrack): RTCRtpTransceiver { + if (!this.peerConn) { + throw new Error("MatrixCall publish() called on call without a peer connection"); } + logger.info(`Call ${this.callId} publishTrack() running (id=${track.id}, kind=${track.kind})`); - for (const track of stream.getTracks()) { - logger.info( - `Call ${this.callId} addTracksOfFeedToPeerConnection() running (feedId=${feedId} streamPurpose=${purpose} kind=${track.kind} enabled=${track.enabled})`, - ); - - const encodings = track.kind === "video" ? getSimulcastEncodings(callFeed.purpose) : undefined; - const transceiverKey = getTransceiverKey(purpose, track.kind); - const transceiver = this.transceivers.get(transceiverKey); - const sender = transceiver?.sender; - - let added = false; - // XXX: We don't re-use transceivers for screen shares with the SFU: this is to work around - // https://github.com/matrix-org/waterfall/issues/98 - see the bug for more. - // Since we use WebRTC data channels to renegotiate with the SFU, we're not - // limited to the size of a Matrix event, so it's 'ok' if the SDP grows - // indefinitely (although presumably this would break if we tried to do - // an ICE restart over to-device messages after you'd turned screen sharing - // on & off too many times...) - // The reason we're OK re-using transceivers for user media is that we establish - // usermedia transceivers in sendrecv mode anyway and let the SFU use the other - // direction to send us some other media (for better or worse) so the same bug - // doesn't occur. - if (sender && (!this.isFocus || callFeed.purpose == SDPStreamMetadataPurpose.Usermedia)) { - try { - // We already have a sender, so we re-use it. We try to - // re-use transceivers as much as possible because they - // can't be removed once added, so otherwise they just - // accumulate which makes the SDP very large very quickly: - // in fact it only takes about 6 video tracks to exceed the - // maximum size of an Olm-encrypted Matrix event - Dave - - // setStreams() is currently not supported by Firefox but we - // try to use it at least in other browsers (once we switch - // to using mids and throw away streamIds we will be able to - // throw this away) - if (sender.setStreams) sender.setStreams(stream); - - sender.replaceTrack(track); - - // We don't need to set simulcast encodings in here since we - // have already done that the first time we added the - // transceiver - - // Set the direction of the transceiver to indicate we're - // going to be sending. This may trigger re-negotiation, if - // we weren't sending until now - transceiver.direction = transceiver.direction === "inactive" ? "sendonly" : "sendrecv"; - - added = true; - } catch (error) { - logger.info( - `Call ${this.callId} addTracksOfFeedToPeerConnection() failed to replaceTrack()`, - error, - ); - } - } + const { stream, encodings } = track; - if (!added) { - try { - // We either don't have a sender or we failed to do - // replaceTrack(), so we use addTransceiver() to add the - // track - const newTransceiver = this.peerConn!.addTransceiver(track, { - streams: [stream], - // Chrome does not allow us to change the encodings - // later, so we have to use addTransceiver() to set them - // (It's fine to specify the parameter on Firefox too, - // it just won't work.) - sendEncodings: this.isFocus ? encodings : undefined, - direction: callFeed.purpose === SDPStreamMetadataPurpose.Usermedia ? "sendrecv" : "sendonly", - }); + const transceiver = this.peerConn.addTransceiver(track.track, { + streams: stream ? [stream] : undefined, + // Chrome does not allow us to change the encodings + // later, so we have to use addTransceiver() to set them + // (It's fine to specify the parameter on Firefox too, + // it just won't work.) + sendEncodings: this.isFocus ? track.encodings : undefined, + direction: track.purpose === SDPStreamMetadataPurpose.Usermedia ? "sendrecv" : "sendonly", + }); - if (this.isFocus && isFirefox()) { - const parameters = newTransceiver.sender.getParameters(); - newTransceiver.sender.setParameters({ - ...parameters, - // Firefox does not support the sendEncodings - // parameter on addTransceiver(), so we use - // setParameters() to set them - encodings: encodings ?? parameters.encodings, - }); - } + if (this.isFocus && isFirefox()) { + const parameters = transceiver.sender.getParameters(); + transceiver.sender.setParameters({ + ...parameters, + // Firefox does not support the sendEncodings + // parameter on addTransceiver(), so we use + // setParameters() to set them + encodings: encodings, + }); + } - this.transceivers.set(transceiverKey, newTransceiver); + return transceiver; + } - added = true; - } catch (error) { - logger.error( - `Call ${this.callId} addTracksOfFeedToPeerConnection() failed to addTransceiver()`, - error, - ); - } - } + public unpublishTrack(track: LocalCallTrack): void { + if (!this.peerConn) { + throw new Error("MatrixCall unpublish() called on call without a peer connection"); + } + if (!track.sender) { + throw new Error("MatrixCall unpublish() called with track without sender"); } + logger.info(`Call ${this.callId} unpublishTrack() running (id=${track.id}, kind=${track.kind})`); + + this.peerConn?.removeTrack(track.sender); } /** * Removes local call feed from the call and its tracks from the peer * connection - * @param callFeed - to remove + * @param feed - to remove */ - public removeLocalFeed(callFeed: CallFeed): void { - const audioTransceiverKey = getTransceiverKey(callFeed.purpose, "audio"); - const videoTransceiverKey = getTransceiverKey(callFeed.purpose, "video"); - - for (const transceiverKey of [audioTransceiverKey, videoTransceiverKey]) { - // this is slightly mixing the track and transceiver API but is basically just shorthand. - // There is no way to actually remove a transceiver, so this just sets it to inactive - // (or recvonly) and replaces the source with nothing. - if (this.transceivers.has(transceiverKey)) { - const transceiver = this.transceivers.get(transceiverKey)!; - if (transceiver.sender) { - this.peerConn!.removeTrack(transceiver.sender); - } + public removeLocalFeed(feed: LocalCallFeed): void { + feed.unpublish(); + + if (feed.stream) { + switch (feed.purpose) { + case SDPStreamMetadataPurpose.Usermedia: + this.client.getMediaHandler().stopUserMediaStream(feed.stream); + break; + + case SDPStreamMetadataPurpose.Screenshare: + this.client.getMediaHandler().stopScreensharingStream(feed.stream); + break; } } - if (callFeed.purpose === SDPStreamMetadataPurpose.Screenshare && callFeed.stream) { - this.client.getMediaHandler().stopScreensharingStream(callFeed.stream); - } - - this.deleteFeed(callFeed); + this.removeFeed(feed); } private deleteAllFeeds(): void { for (const feed of this.feeds) { - if (!feed.isLocal()) { + if (!feed.isLocal) { feed.removeListener(CallFeedEvent.SizeChanged, this.onCallFeedSizeChanged); if (!this.groupCallId) { feed.dispose(); @@ -1042,17 +847,23 @@ export class MatrixCall extends TypedEventEmitter 0) { + throw new Error( + "MatrixCall addRemoteFeed() cannot add multiple remote feeds if opponent does not support sdp_stream_metadata", + ); + } + this.feeds.push(feed); feed.addListener(CallFeedEvent.SizeChanged, this.onCallFeedSizeChanged); @@ -1061,7 +872,7 @@ export class MatrixCall extends TypedEventEmitter !feed.isLocal())?.stream; + const remoteStream = this.feeds.find((feed) => !feed.isLocal)?.stream; // According to previous comments in this file, firefox at some point did not // add streams until media started arriving on them. Testing latest firefox @@ -1229,16 +1047,11 @@ export class MatrixCall extends TypedEventEmitter track.kind === "video"); + if (!screensharingTrack) return false; + const usermediaTrack = this.localUsermediaFeed?.tracks?.find((track) => track.kind === "video"); + if (!usermediaTrack) return false; - const track = stream.getTracks().find((track) => track.kind === "video"); - - const sender = this.transceivers.get( - getTransceiverKey(SDPStreamMetadataPurpose.Usermedia, "video"), - )?.sender; - - sender?.replaceTrack(track ?? null); - - this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare, false); + usermediaTrack?.setNewTrack(screensharingTrack); + this.addLocalFeedFromStream(screensharingStream, SDPStreamMetadataPurpose.Screenshare, false); return true; } catch (err) { @@ -1482,12 +1280,13 @@ export class MatrixCall extends TypedEventEmitter track.kind === "video"); - const sender = this.transceivers.get( - getTransceiverKey(SDPStreamMetadataPurpose.Usermedia, "video"), - )?.sender; - sender?.replaceTrack(track ?? null); + const usermediaTrack = this.localUsermediaStream?.getTracks().find((track) => track.kind === "video"); + if (!usermediaTrack) return true; + // We get the screensharing track here from the USERMEDIA feed + const screensharingTrack = this.localUsermediaFeed?.tracks?.find((track) => track.kind === "video"); + if (!screensharingTrack) return true; + screensharingTrack.setNewTrack(usermediaTrack); this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream!); this.deleteFeedByStream(this.localScreensharingStream!); @@ -1521,6 +1320,7 @@ export class MatrixCall extends TypedEventEmitter { - logger.log(`Call ${this.callId} setLocalVideoMuted() running ${muted}`); + logger.log(`Call ${this.callId} setLocalVideoMuted() running (muted=${muted})`); // if we were still thinking about stopping and removing the video // track: don't, because we want it back. @@ -1706,9 +1500,9 @@ export class MatrixCall extends TypedEventEmitter this.gotCallFeedsForAnswer(callFeeds)); @@ -1871,7 +1665,7 @@ export class MatrixCall extends TypedEventEmitter { + private async gotCallFeedsForAnswer(callFeeds: LocalCallFeed[]): Promise { if (this.callHasEnded()) return; this.waitForLocalAVStream = false; @@ -2020,13 +1814,20 @@ export class MatrixCall extends TypedEventEmitter f.feedId === streamId); + const feed = this.getRemoteFeeds().find((f) => f.streamId === streamId); if (feed) { - feed.purpose = streamMetadata.purpose; - feed.tracksMetadata = streamMetadata.tracks; - } else { - feed = new CallFeed({ + feed.metadata = streamMetadata; + continue; + } + + // We don't emit here and only emit at the end of onMetadata to + // avoid spam + this.addRemoteFeed( + new RemoteCallFeed({ client: this.client, call: this, roomId: this.roomId, - userId: this.isFocus ? streamMetadata.user_id : this.getOpponentMember()!.userId, - deviceId: this.isFocus ? streamMetadata.device_id : this.getOpponentDeviceId()!, - feedId: streamId, - purpose: streamMetadata.purpose, - audioMuted: streamMetadata.audio_muted, - videoMuted: streamMetadata.video_muted, - tracksMetadata: streamMetadata.tracks, - }); - this.addRemoteFeed(feed, false); - feedsChanged = true; - } - feed.setAudioVideoMuted(streamMetadata.audio_muted, streamMetadata.video_muted); + streamId: streamId, + metadata: streamMetadata, + }), + false, + ); + feedsChanged = true; } // Remove old feeds for (const feed of this.getRemoteFeeds()) { - if (!Object.keys(metadata).includes(feed.feedId)) { - this.deleteFeed(feed, false); + if (!Object.keys(metadata).includes(feed.streamId ?? "")) { + this.removeFeed(feed, false); feedsChanged = true; } } @@ -2319,7 +2147,7 @@ export class MatrixCall extends TypedEventEmitter()?.[SDPStreamMetadataKey]; if (metadata) { - this.updateRemoteSDPStreamMetadata(metadata); + this.onMetadata(metadata); } } @@ -2435,7 +2263,7 @@ export class MatrixCall extends TypedEventEmitter { - if (ev.streams.length === 0) { - logger.warn( - `Call ${this.callId} onTrack() called with streamless track streamless (kind=${ev.track.kind})`, - ); + const stream = ev.streams[0]; + const transceiver = ev.transceiver; + + if (!stream) { + logger.warn(`Call ${this.callId} onTrack() called with streamless track (kind=${ev.track.kind})`); + return; + } + if (!transceiver?.mid) { + logger.warn(`Call ${this.callId} onTrack() called with transceiver without an mid (kind=${ev.track.kind})`); + return; + } + if (!this.opponentSupportsSDPStreamMetadata()) { + this.addRemoteFeedWithoutMetadata(stream); + if (!this.removeTrackListeners.has(stream)) { + const onRemoveTrack = (): void => { + if (stream.getTracks().length === 0) { + logger.info(`Call ${this.callId} onTrack() removing track (streamId=${stream.id})`); + this.deleteFeedByStream(stream); + stream.removeEventListener("removetrack", onRemoveTrack); + this.removeTrackListeners.delete(stream); + } + }; + stream.addEventListener("removetrack", onRemoveTrack); + this.removeTrackListeners.set(stream, onRemoveTrack); + } return; } - const stream = ev.streams[0]; - this.pushRemoteStream(stream); - - if (!this.removeTrackListeners.has(stream)) { - const onRemoveTrack = (): void => { - if (stream.getTracks().length === 0) { - logger.info(`Call ${this.callId} onTrack() removing track (streamId=${stream.id})`); - this.deleteFeedByStream(stream); - stream.removeEventListener("removetrack", onRemoveTrack); - this.removeTrackListeners.delete(stream); - } - }; - stream.addEventListener("removetrack", onRemoveTrack); - this.removeTrackListeners.set(stream, onRemoveTrack); + logger.log( + `Call ${this.callId} onTrack() running (mid=${transceiver.mid}, kind=${transceiver.receiver.track.kind})`, + ); + + const feed = this.getRemoteFeeds().find((feed) => feed.canAddTransceiver(transceiver)); + if (!feed) { + logger.warn( + `Call ${ + this.callId + } onTrack() did not find feed for transceiver (streamId=${this.getRemoteStreamIdByMid( + transceiver.mid, + )}, trackId=${this.getRemoteTrackIdByMid(transceiver.mid)} kind=${transceiver.receiver.track.kind})`, + ); + return; } + feed.addTransceiver(transceiver); }; private onDataChannel = (ev: RTCDataChannelEvent): void => { @@ -2630,10 +2480,8 @@ export class MatrixCall extends TypedEventEmitter => { @@ -2772,12 +2620,12 @@ export class MatrixCall extends TypedEventEmitter { + public async placeCallWithCallFeeds(callFeeds: LocalCallFeed[], requestScreenshareFeed = false): Promise { this.checkForErrorListener(); this.direction = CallDirection.Outbound; diff --git a/src/webrtc/callEventTypes.ts b/src/webrtc/callEventTypes.ts index 62ba362d2e5..53a08ac1974 100644 --- a/src/webrtc/callEventTypes.ts +++ b/src/webrtc/callEventTypes.ts @@ -24,12 +24,12 @@ export interface SDPStreamMetadataTracks { } export interface SDPStreamMetadataObject { - user_id: string; - device_id: string; + user_id?: string; + device_id?: string; purpose: SDPStreamMetadataPurpose; - audio_muted: boolean; - video_muted: boolean; - tracks: SDPStreamMetadataTracks; + audio_muted?: boolean; + video_muted?: boolean; + tracks?: SDPStreamMetadataTracks; } export interface SDPStreamMetadata { diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 91a93a7d975..ab8e8cd0838 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -15,13 +15,15 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { SDPStreamMetadataPurpose, SDPStreamMetadataTracks } from "./callEventTypes"; +import { SDPStreamMetadataPurpose } from "./callEventTypes"; import { acquireContext, releaseContext } from "./audioContext"; import { MatrixClient } from "../client"; import { RoomMember } from "../models/room-member"; -import { logger } from "../logger"; import { TypedEventEmitter } from "../models/typed-event-emitter"; -import { CallEvent, CallState, MatrixCall } from "./call"; +import { MatrixCall } from "./call"; +import { CallTrack } from "./callTrack"; +import { randomString } from "../randomstring"; +import { logger } from "../logger"; const POLLING_INTERVAL = 200; // ms export const SPEAKING_THRESHOLD = -60; // dB @@ -30,28 +32,6 @@ const SPEAKING_SAMPLE_COUNT = 8; // samples export interface ICallFeedOpts { client: MatrixClient; roomId?: string; - userId: string; - deviceId: string | undefined; - /** - * Now, this should be the same as streamId but in the future we might want - * to use something different - */ - feedId: string; - stream?: MediaStream; - purpose: SDPStreamMetadataPurpose; - tracksMetadata?: SDPStreamMetadataTracks; - /** - * Whether or not the remote SDPStreamMetadata says audio is muted - */ - audioMuted: boolean; - /** - * Whether or not the remote SDPStreamMetadata says video is muted - */ - videoMuted: boolean; - /** - * The MatrixCall which is the source of this CallFeed - */ - call?: MatrixCall; } export enum CallFeedEvent { @@ -76,20 +56,28 @@ type EventHandlerMap = { [CallFeedEvent.Disposed]: () => void; }; -export class CallFeed extends TypedEventEmitter { - public feedId: string; - public readonly userId: string; - public readonly deviceId: string | undefined; - public purpose: SDPStreamMetadataPurpose; +export abstract class CallFeed extends TypedEventEmitter { + public abstract get id(): string; + public abstract get streamId(): string | undefined; + public abstract get purpose(): SDPStreamMetadataPurpose; + public abstract get connected(): boolean; + public abstract get userId(): string; + public abstract get deviceId(): string | undefined; + + public abstract isLocal: boolean; + public abstract isRemote: boolean; + public speakingVolumeSamples: number[]; - public tracksMetadata: SDPStreamMetadataTracks = {}; - - private _stream?: MediaStream; - private client: MatrixClient; - private call?: MatrixCall; - private roomId?: string; - private audioMuted: boolean; - private videoMuted: boolean; + + protected readonly _id: string; + protected _tracks: CallTrack[] = []; + protected _stream?: MediaStream; + protected call?: MatrixCall; + protected roomId?: string; + protected client: MatrixClient; + protected audioMuted = false; + protected videoMuted = false; + private localVolume = 1; private measuringVolumeActivity = false; private audioContext?: AudioContext; @@ -99,53 +87,27 @@ export class CallFeed extends TypedEventEmitter private speaking = false; private volumeLooperTimeout?: ReturnType; private _disposed = false; - private _connected = false; - private _width = 0; private _height = 0; - private _isVisible = false; public constructor(opts: ICallFeedOpts) { super(); + this._id = randomString(32); this.client = opts.client; - this.call = opts.call; this.roomId = opts.roomId; - this.userId = opts.userId; - this.deviceId = opts.deviceId; - this.purpose = opts.purpose; - this.audioMuted = opts.audioMuted; - this.videoMuted = opts.videoMuted; this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); - this.feedId = opts.feedId; - - this.updateStream(undefined, opts.stream); if (this.hasAudioTrack) { this.initVolumeMeasuring(); } - - if (opts.call) { - opts.call.addListener(CallEvent.State, this.onCallState); - } - this.updateConnected(); } public get stream(): MediaStream | undefined { return this._stream; } - public get connected(): boolean { - // Local feeds are always considered connected - return this.isLocal() || this._connected; - } - - private set connected(connected: boolean) { - this._connected = connected; - this.emit(CallFeedEvent.ConnectedChanged, this.connected); - } - public get isVisible(): boolean { return this._isVisible; } @@ -158,24 +120,31 @@ export class CallFeed extends TypedEventEmitter return this._height; } + public get audioTrack(): CallTrack | undefined { + return this._tracks.find((track) => track.isAudio); + } + + public get videoTrack(): CallTrack | undefined { + return this._tracks.find((track) => track.isVideo); + } + private get hasAudioTrack(): boolean { return this.stream ? this.stream.getAudioTracks().length > 0 : false; } - private updateStream(oldStream?: MediaStream, newStream?: MediaStream): void { + protected get tracks(): CallTrack[] { + return [...this._tracks]; + } + + protected updateStream(oldStream?: MediaStream, newStream?: MediaStream): void { if (newStream === oldStream) return; if (oldStream) { - oldStream.removeEventListener("addtrack", this.onAddTrack); - oldStream.removeEventListener("removetrack", this.onRemoveTrack); clearTimeout(this.volumeLooperTimeout); } this._stream = newStream; - newStream?.addEventListener("addtrack", this.onAddTrack); - newStream?.addEventListener("removetrack", this.onRemoveTrack); - this.updateConnected(); this.initVolumeMeasuring(); this.volumeLooper(); @@ -197,32 +166,6 @@ export class CallFeed extends TypedEventEmitter this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount); } - private onAddTrack = (): void => { - this.updateConnected(); - this.emit(CallFeedEvent.NewStream, this.stream); - }; - - private onRemoveTrack = (): void => { - this.updateConnected(); - this.emit(CallFeedEvent.NewStream, this.stream); - }; - - private onCallState = (): void => { - this.updateConnected(); - }; - - private updateConnected(): void { - if (this.call?.state === CallState.Connecting) { - this.connected = false; - } else if (!this.stream) { - this.connected = false; - } else if (this.stream.getTracks().length === 0) { - this.connected = false; - } else if (this.call?.state === CallState.Connected) { - this.connected = true; - } - } - /** * Returns callRoom member * @returns member of the callRoom @@ -232,17 +175,6 @@ export class CallFeed extends TypedEventEmitter return callRoom?.getMember(this.userId) ?? null; } - /** - * Returns true if CallFeed is local, otherwise returns false - * @returns is local? - */ - public isLocal(): boolean { - return ( - this.userId === this.client.getUserId() && - (this.deviceId === undefined || this.deviceId === this.client.getDeviceId()) - ); - } - /** * Returns true if audio is muted or if there are no audio * tracks, otherwise returns false @@ -266,17 +198,6 @@ export class CallFeed extends TypedEventEmitter return this.speaking; } - /** - * Replaces the current MediaStream with a new one. - * The stream will be different and new stream as remote parties are - * concerned, but this can be used for convenience locally to set up - * volume listeners automatically on the new stream etc. - * @param newStream - new stream with which to replace the current one - */ - public setNewStream(newStream: MediaStream): void { - this.updateStream(this.stream, newStream); - } - /** * Set one or both of feed's internal audio and video video mute state * Either value may be null to leave it as-is @@ -284,6 +205,8 @@ export class CallFeed extends TypedEventEmitter * @param videoMuted - is the feed's video muted? */ public setAudioVideoMuted(audioMuted: boolean | null, videoMuted: boolean | null): void { + logger.log(`CallFeed ${this.id} setAudioVideoMuted() running (audio=${audioMuted}, video=${videoMuted})`); + if (audioMuted !== null) { if (this.audioMuted !== audioMuted) { this.speakingVolumeSamples.fill(-Infinity); @@ -351,39 +274,8 @@ export class CallFeed extends TypedEventEmitter this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL); }; - public clone(): CallFeed { - const mediaHandler = this.client.getMediaHandler(); - - let stream: MediaStream | undefined; - if (this.stream) { - stream = this.stream.clone(); - logger.log(`CallFeed clone() cloning stream (originalStreamId=${this.stream.id}, newStreamId${stream.id})`); - - if (this.purpose === SDPStreamMetadataPurpose.Usermedia) { - mediaHandler.userMediaStreams.push(stream); - } else { - mediaHandler.screensharingStreams.push(stream); - } - } - - return new CallFeed({ - client: this.client, - roomId: this.roomId, - userId: this.userId, - deviceId: this.deviceId, - feedId: this.feedId, - stream, - purpose: this.purpose, - audioMuted: this.audioMuted, - videoMuted: this.videoMuted, - }); - } - public dispose(): void { clearTimeout(this.volumeLooperTimeout); - this.stream?.removeEventListener("addtrack", this.onAddTrack); - this.stream?.removeEventListener("removetrack", this.onRemoveTrack); - this.call?.removeListener(CallEvent.State, this.onCallState); if (this.audioContext) { this.audioContext = undefined; this.analyser = undefined; diff --git a/src/webrtc/callTrack.ts b/src/webrtc/callTrack.ts new file mode 100644 index 00000000000..37d7c2e8144 --- /dev/null +++ b/src/webrtc/callTrack.ts @@ -0,0 +1,47 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { randomString } from "../randomstring"; +import { SDPStreamMetadataTrack } from "./callEventTypes"; + +export interface CallTrackOpts {} + +export abstract class CallTrack { + public abstract get id(): string | undefined; + public abstract get track(): MediaStreamTrack | undefined; + public abstract get trackId(): string | undefined; + public abstract get metadata(): SDPStreamMetadataTrack | undefined; + public abstract get kind(): string | undefined; + + protected readonly _id: string; + protected _transceiver?: RTCRtpTransceiver; + + public constructor(opts: CallTrackOpts) { + this._id = randomString(32); + } + + public get transceiver(): RTCRtpTransceiver | undefined { + return this._transceiver; + } + + public get isAudio(): boolean { + return this.kind === "audio"; + } + + public get isVideo(): boolean { + return this.kind === "video"; + } +} diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 4065eb92a75..e653d383cd6 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -24,6 +24,7 @@ import { CallEventHandlerEvent } from "./callEventHandler"; import { GroupCallEventHandlerEvent } from "./groupCallEventHandler"; import { IScreensharingOpts } from "./mediaHandler"; import { mapsEqual } from "../utils"; +import { LocalCallFeed } from "./localCallFeed"; export enum GroupCallIntent { Ring = "m.ring", @@ -192,8 +193,8 @@ export class GroupCall extends TypedEventEmitter< public pttMaxTransmitTime = 1000 * 20; public activeSpeaker?: CallFeed; - public localCallFeed?: CallFeed; - public localScreenshareFeed?: CallFeed; + public localCallFeed?: LocalCallFeed; + public localScreenshareFeed?: LocalCallFeed; public localDesktopCapturerSourceId?: string; public readonly userMediaFeeds: CallFeed[] = []; public readonly screenshareFeeds: CallFeed[] = []; @@ -346,8 +347,8 @@ export class GroupCall extends TypedEventEmitter< return isUsingPreferredFocus ? [] : preferredFoci; } - public getLocalFeeds(): CallFeed[] { - const feeds: CallFeed[] = []; + public getLocalFeeds(): LocalCallFeed[] { + const feeds: LocalCallFeed[] = []; if (this.localCallFeed) feeds.push(this.localCallFeed); if (this.localScreenshareFeed) feeds.push(this.localScreenshareFeed); @@ -400,17 +401,16 @@ export class GroupCall extends TypedEventEmitter< throw new Error("Group call disposed while gathering media stream"); } - const callFeed = new CallFeed({ + const callFeed = new LocalCallFeed({ client: this.client, roomId: this.room.roomId, - userId: this.client.getUserId()!, - deviceId: this.client.getDeviceId()!, - feedId: stream.id, stream, purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: this.initWithAudioMuted || stream.getAudioTracks().length === 0 || this.isPtt, - videoMuted: this.initWithVideoMuted || stream.getVideoTracks().length === 0, }); + callFeed.setAudioVideoMuted( + this.initWithAudioMuted || stream.getAudioTracks().length === 0 || this.isPtt, + this.initWithVideoMuted || stream.getVideoTracks().length === 0, + ); setTracksEnabled(stream.getAudioTracks(), !callFeed.isAudioMuted()); setTracksEnabled(stream.getVideoTracks(), !callFeed.isVideoMuted()); @@ -636,7 +636,7 @@ export class GroupCall extends TypedEventEmitter< if (this.localCallFeed) { logger.log( - `GroupCall ${this.groupCallId} setMicrophoneMuted() (feedId=${this.localCallFeed.feedId}, muted=${muted})`, + `GroupCall ${this.groupCallId} setMicrophoneMuted() (feedId=${this.localCallFeed.id}, muted=${muted})`, ); this.localCallFeed.setAudioVideoMuted(muted, null); // I don't believe its actually necessary to enable these tracks: they @@ -678,7 +678,7 @@ export class GroupCall extends TypedEventEmitter< if (this.localCallFeed) { logger.log( - `GroupCall ${this.groupCallId} setLocalVideoMuted() (feedId=${this.localCallFeed.feedId}, muted=${muted})`, + `GroupCall ${this.groupCallId} setLocalVideoMuted() running (feedId=${this.localCallFeed.id}, muted=${muted})`, ); const stream = await this.client.getMediaHandler().getUserMediaStream(true, !muted); @@ -727,16 +727,11 @@ export class GroupCall extends TypedEventEmitter< ); this.localDesktopCapturerSourceId = opts.desktopCapturerSourceId; - this.localScreenshareFeed = new CallFeed({ + this.localScreenshareFeed = new LocalCallFeed({ client: this.client, roomId: this.room.roomId, - userId: this.client.getUserId()!, - deviceId: this.client.getDeviceId()!, - feedId: stream.id, stream, purpose: SDPStreamMetadataPurpose.Screenshare, - audioMuted: false, - videoMuted: false, }); this.addScreenshareFeed(this.localScreenshareFeed); @@ -1271,7 +1266,7 @@ export class GroupCall extends TypedEventEmitter< let nextActiveSpeaker: CallFeed | undefined = undefined; for (const callFeed of this.userMediaFeeds) { - if (callFeed.isLocal() && this.userMediaFeeds.length > 1) continue; + if (callFeed.isLocal && this.userMediaFeeds.length > 1) continue; const total = callFeed.speakingVolumeSamples.reduce( (acc, volume) => acc + Math.max(volume, SPEAKING_THRESHOLD), diff --git a/src/webrtc/localCallFeed.ts b/src/webrtc/localCallFeed.ts new file mode 100644 index 00000000000..eba08863e69 --- /dev/null +++ b/src/webrtc/localCallFeed.ts @@ -0,0 +1,164 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { logger } from "../logger"; +import { MatrixCall } from "./call"; +import { SDPStreamMetadataObject, SDPStreamMetadataPurpose, SDPStreamMetadataTracks } from "./callEventTypes"; +import { CallFeed, ICallFeedOpts } from "./callFeed"; +import { LocalCallTrack } from "./localCallTrack"; + +export interface LocalCallFeedOpts extends ICallFeedOpts { + purpose: SDPStreamMetadataPurpose; + stream: MediaStream; +} + +export class LocalCallFeed extends CallFeed { + protected _tracks: LocalCallTrack[] = []; + private _purpose: SDPStreamMetadataPurpose; + + protected _stream: MediaStream; + + public readonly connected = true; + public readonly isLocal = true; + public readonly isRemote = false; + + public constructor(opts: LocalCallFeedOpts) { + super(opts); + + this._purpose = opts.purpose; + + this.updateStream(undefined, opts.stream); + // updateStream() already did the job, but this shuts up typescript from + // complaining about it not being set in the constructor + this._stream = opts.stream; + } + + public get id(): string { + return this._id; + } + + public get tracks(): LocalCallTrack[] { + return super.tracks as LocalCallTrack[]; + } + + public get metadata(): SDPStreamMetadataObject { + return { + user_id: this.userId, + device_id: this.deviceId, + purpose: this.purpose, + audio_muted: this.isAudioMuted(), + video_muted: this.isVideoMuted(), + tracks: this._tracks.reduce((metadata: SDPStreamMetadataTracks, track: LocalCallTrack) => { + if (!track.trackId) return metadata; + + metadata[track.trackId] = track.metadata; + return metadata; + }, {}), + }; + } + + public get purpose(): SDPStreamMetadataPurpose { + return this._purpose; + } + + public get userId(): string { + return this.client.getUserId()!; + } + + public get deviceId(): string | undefined { + return this.client.getDeviceId() ?? undefined; + } + + public get streamId(): string | undefined { + return this._tracks[0]?.streamId; + } + + public clone(): LocalCallFeed { + const mediaHandler = this.client.getMediaHandler(); + const stream = this._stream.clone(); + logger.log( + `CallFeed ${this.id} clone() cloning stream (originalStreamId=${this._stream.id}, newStreamId=${stream.id})`, + ); + + if (this.purpose === SDPStreamMetadataPurpose.Usermedia) { + mediaHandler.userMediaStreams.push(stream); + } else if (this.purpose === SDPStreamMetadataPurpose.Screenshare) { + mediaHandler.screensharingStreams.push(stream); + } + + const feed = new LocalCallFeed({ + client: this.client, + roomId: this.roomId, + stream, + purpose: this.purpose, + }); + feed.setAudioVideoMuted(this.audioMuted, this.videoMuted); + return feed; + } + + public setNewStream(newStream: MediaStream): void { + this.updateStream(this.stream, newStream); + } + + protected updateStream(oldStream?: MediaStream, newStream?: MediaStream): void { + super.updateStream(oldStream, newStream); + + if (!newStream) return; + + // First, remove tracks which won't be used anymore + for (const track of this._tracks) { + if (!newStream.getTracks().some((streamTrack) => streamTrack.kind === track.kind)) { + this._tracks.splice(this._tracks.indexOf(track), 1); + if (track.published) { + track.unpublish(); + } + } + } + + // Then, replace old track where we can and add new tracks + for (const streamTrack of newStream.getTracks()) { + let track = this._tracks.find((track) => track.kind === streamTrack.kind); + if (track) { + track.setNewTrack(streamTrack); + continue; + } + + track = new LocalCallTrack({ + feed: this, + track: streamTrack, + }); + this._tracks.push(track); + + if (this.call) { + track.publish(this.call); + } + } + } + + public publish(call: MatrixCall): void { + this.call = call; + for (const track of this._tracks) { + track.publish(call); + } + } + + public unpublish(): void { + this.call = undefined; + for (const track of this._tracks) { + track.unpublish(); + } + } +} diff --git a/src/webrtc/localCallTrack.ts b/src/webrtc/localCallTrack.ts new file mode 100644 index 00000000000..c56c94a8391 --- /dev/null +++ b/src/webrtc/localCallTrack.ts @@ -0,0 +1,311 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { logger } from "../logger"; +import { MatrixCall } from "./call"; +import { SDPStreamMetadataPurpose, SDPStreamMetadataTrack } from "./callEventTypes"; +import { CallTrack, CallTrackOpts } from "./callTrack"; +import { LocalCallFeed } from "./localCallFeed"; + +export enum SimulcastResolution { + Full = "f", + Half = "h", + Quarter = "q", +} + +// Order is important here: some browsers (e.g. +// Chrome) will only send some of the encodings, if +// the track has a resolution to low for it to send +// all, in that case the encoding higher in the list +// has priority and therefore we put full as first +// as we always want to send the full resolution +const SIMULCAST_USERMEDIA_ENCODINGS: RTCRtpEncodingParameters[] = [ + { + // 720p (base) + maxFramerate: 30, + maxBitrate: 1_700_000, + rid: SimulcastResolution.Full, + }, + { + // 360p + maxFramerate: 20, + maxBitrate: 300_000, + rid: SimulcastResolution.Half, + scaleResolutionDownBy: 2.0, + }, + { + // 180p + maxFramerate: 15, + maxBitrate: 120_000, + + rid: SimulcastResolution.Quarter, + scaleResolutionDownBy: 4.0, + }, +]; + +const SIMULCAST_SCREENSHARING_ENCODINGS: RTCRtpEncodingParameters[] = [ + { + // 1080p (base) + maxFramerate: 30, + maxBitrate: 3_000_000, + rid: SimulcastResolution.Full, + }, + { + // 720p + maxFramerate: 15, + maxBitrate: 1_000_000, + rid: SimulcastResolution.Half, + scaleResolutionDownBy: 1.5, + }, + { + // 360p + maxFramerate: 3, + maxBitrate: 200_000, + rid: SimulcastResolution.Quarter, + scaleResolutionDownBy: 3, + }, +]; + +export const getSimulcastEncodings = (purpose: SDPStreamMetadataPurpose): RTCRtpEncodingParameters[] => { + if (purpose === SDPStreamMetadataPurpose.Usermedia) { + return SIMULCAST_USERMEDIA_ENCODINGS; + } + if (purpose === SDPStreamMetadataPurpose.Screenshare) { + return SIMULCAST_SCREENSHARING_ENCODINGS; + } + + // Fallback to usermedia encodings + return SIMULCAST_USERMEDIA_ENCODINGS; +}; + +export interface LocalCallTrackOpts extends CallTrackOpts { + feed: LocalCallFeed; + track: MediaStreamTrack; +} + +export class LocalCallTrack extends CallTrack { + private _track: MediaStreamTrack; + private feed: LocalCallFeed; + private call?: MatrixCall; + + public constructor(opts: LocalCallTrackOpts) { + super(opts); + + this._track = opts.track; + this.feed = opts.feed; + } + + private get logInfo(): string { + return `streamId=${this.streamId}, trackId=${this.trackId}, mid=${this.mid} kind=${this.kind}`; + } + + public get id(): string | undefined { + return this._id; + } + + public get metadata(): SDPStreamMetadataTrack { + const trackMetadata: SDPStreamMetadataTrack = { + kind: this.track.kind, + }; + + if (this.isVideo) { + trackMetadata.width = this.track.getSettings().width; + trackMetadata.height = this.track.getSettings().height; + } + + return trackMetadata; + } + + public get mid(): string | undefined { + return this._transceiver?.mid ?? undefined; + } + + public get trackId(): string | undefined { + const mid = this._transceiver?.mid; + return mid ? this.call?.getLocalTrackIdByMid(mid) : undefined; + } + + public get streamId(): string | undefined { + const mid = this._transceiver?.mid; + return mid ? this.call?.getLocalStreamIdByMid(mid) : undefined; + } + + public get track(): MediaStreamTrack { + return this._track; + } + + public get kind(): string { + return this.track.kind; + } + + public get purpose(): SDPStreamMetadataPurpose { + return this.feed.purpose; + } + + public get stream(): MediaStream | undefined { + return this.feed.stream; + } + + public get sender(): RTCRtpSender | undefined { + return this._transceiver?.sender; + } + + public get encodings(): RTCRtpEncodingParameters[] { + return getSimulcastEncodings(this.purpose); + } + + public get published(): boolean { + if (!this._transceiver?.sender) return false; + if (!this.call) return false; + + return true; + } + + public publish(call: MatrixCall): void { + if (this.published) { + throw new Error("Cannot publish already published track"); + } + + const oldCall = this.call; + + // We try to re-use transceivers here + const transceiver = call.getFreeTransceiverByKind(this.kind); + if (transceiver) { + try { + this._transceiver = transceiver; + this.call = call; + this.replaceTrackOnPeerConnection(); + } catch (error) { + this._transceiver = undefined; + this.call = oldCall; + } + } else { + try { + this.call = call; + this.addTrackToPeerConnection(); + } catch (error) { + this.call = undefined; + logger.warn( + `LocalCallTrack ${this.id} publish() failed to publish track to call (callId${call.callId})`, + ); + } + } + } + + public unpublish(): void { + const call = this.call; + if (!this.published || !call) { + throw new Error("Cannot unpublish track that is not published"); + } + + try { + call.unpublishTrack(this); + this.call = undefined; + this._transceiver = undefined; + } catch (error) { + logger.warn( + `LocalCallTrack ${this.id} unpublish() failed to publish track to call (callId${call.callId})`, + error, + ); + } + } + + public setNewTrack(track: MediaStreamTrack): void { + logger.log(`LocalCallTrack ${this.id} setNewTrack() running (${this.logInfo})`); + + const oldTrack = this._track; + this._track = track; + + // If the track is not published, we don't need to try to publish the + // new one either + if (!this.published || !this.call) return; + + try { + // XXX: We don't re-use transceivers with the SFU: this is to work around + // https://github.com/matrix-org/waterfall/issues/98 - see the bug for more. + // Since we use WebRTC data channels to renegotiate with the SFU, we're not + // limited to the size of a Matrix event, so it's 'ok' if the SDP grows + // indefinitely (although presumably this would break if we tried to do + // an ICE restart over to-device messages after you'd turned screen sharing + // on & off too many times...) + if (!this.sender || (this.call.isFocus && this.purpose === SDPStreamMetadataPurpose.Screenshare)) { + this.unpublish(); + this.addTrackToPeerConnection(); + } else { + this.replaceTrackOnPeerConnection(); + } + + logger.log(`LocalCallTrack ${this.id} setNewTrack() updated published track (${this.logInfo})`); + } catch (error) { + this._track = oldTrack; + logger.log( + `LocalCallTrack ${this.id} setNewTrack() failed to update published track (${this.logInfo})`, + error, + ); + } + } + + private addTrackToPeerConnection(): void { + if (!this.call) { + throw new Error("Called without a call"); + } + this._transceiver = this.call.publishTrack(this); + } + + private replaceTrackOnPeerConnection(): void { + const stream = this.stream; + const sender = this.sender; + const transceiver = this._transceiver; + + if (!stream || !transceiver || !sender || !this.track) { + throw new Error("Called without"); + } + + logger.log(`LocalCallTrack LocalCallTrack ${this.id} replaceTrack() running (${this.logInfo})`); + + try { + // We already have a sender, so we re-use it. We try to + // re-use transceivers as much as possible because they + // can't be removed once added, so otherwise they just + // accumulate which makes the SDP very large very quickly: + // in fact it only takes about 6 video tracks to exceed the + // maximum size of an Olm-encrypted Matrix event - Dave + + // setStreams() is currently not supported by Firefox but we + // try to use it at least in other browsers (once we switch + // to using mids and throw away streamIds we will be able to + // throw this away) + if (sender.setStreams && stream) sender.setStreams(stream); + + sender.replaceTrack(this.track); + + // We don't need to set simulcast encodings in here since we + // have already done that the first time we added the + // transceiver + + // Set the direction of the transceiver to indicate we're + // going to be sending. This may trigger re-negotiation, if + // we weren't sending until now + transceiver.direction = transceiver.direction === "inactive" ? "sendonly" : "sendrecv"; + } catch (error) { + logger.warn( + `LocalCallTrack ${this.id} setNewTrack() failed to replace track: falling back to adding a new one (${this.logInfo})`, + error, + ); + this.addTrackToPeerConnection(); + } + } +} diff --git a/src/webrtc/remoteCallFeed.ts b/src/webrtc/remoteCallFeed.ts new file mode 100644 index 00000000000..8a44575a15b --- /dev/null +++ b/src/webrtc/remoteCallFeed.ts @@ -0,0 +1,229 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { logger } from "../logger"; +import { CallEvent, CallState, MatrixCall } from "./call"; +import { SDPStreamMetadataObject, SDPStreamMetadataPurpose } from "./callEventTypes"; +import { CallFeed, CallFeedEvent, ICallFeedOpts } from "./callFeed"; +import { RemoteCallTrack } from "./remoteCallTrack"; + +export interface RemoteCallFeedOpts extends ICallFeedOpts { + streamId: string; + metadata?: SDPStreamMetadataObject; + call: MatrixCall; + + /** + * @deprecated addTransceiver() should be used instead + */ + stream?: MediaStream; +} + +export class RemoteCallFeed extends CallFeed { + private _connected = false; + private _metadata?: SDPStreamMetadataObject; + + protected _tracks: RemoteCallTrack[] = []; + protected call: MatrixCall; + protected _stream: MediaStream; + + public readonly streamId: string; + public readonly isLocal = false; + public readonly isRemote = true; + + public constructor(opts: RemoteCallFeedOpts) { + super(opts); + + if (!opts.metadata && opts.call.opponentSupportsSDPStreamMetadata()) { + throw new Error( + "Cannot create RemoteCallFeed without metadata if the opponents supports sending sdp_stream_metadata", + ); + } + + this.streamId = opts.streamId; + this.call = opts.call; + this.metadata = opts.metadata; + + this._stream = opts.stream || new window.MediaStream(); + + if (opts.call) { + opts.call.addListener(CallEvent.State, this.onCallState); + } + this.updateConnected(); + } + + public get id(): string { + return this.streamId; + } + + public get metadata(): SDPStreamMetadataObject | undefined { + return this._metadata; + } + + public set metadata(metadata: SDPStreamMetadataObject | undefined) { + if (!metadata) return; + + this.setAudioVideoMuted(metadata.audio_muted ?? false, metadata.video_muted ?? false); + this._metadata = metadata; + + if (!metadata.tracks) return; + for (const [metadataTrackId, metadataTrack] of Object.entries(metadata.tracks)) { + const track = this._tracks.find((track) => track.trackId === metadataTrackId); + if (track) { + track.metadata = metadataTrack; + continue; + } + + logger.info( + `RemoteCallFeed ${this.id} set metadata() adding track (streamId=${this.streamId} trackId=${metadataTrackId}, kind=${metadataTrack.kind})`, + ); + this._tracks.push( + new RemoteCallTrack({ + call: this.call, + trackId: metadataTrackId, + metadata: metadataTrack, + }), + ); + } + + for (const track of this._tracks) { + if (!track.trackId) continue; + if (!Object.keys(metadata.tracks).includes(track.trackId)) { + logger.info( + `RemoteCallFeed ${this.id} set metadata() removing track (streamId=${this.streamId} trackId=${track.trackId}, kind=${track.kind})`, + ); + this._tracks.splice(this._tracks.indexOf(track), 1); + if (track.track) { + this.stream?.removeTrack(track.track); + } + } + } + } + + public get purpose(): SDPStreamMetadataPurpose { + // If the opponent did not send a purpose, they probably don't support + // sdp_stream_metadata, so we can assume they're only sending usermedia + return this._metadata?.purpose ?? SDPStreamMetadataPurpose.Usermedia; + } + + public get userId(): string { + const metadataUserId = this._metadata?.user_id; + return this.call.isFocus && metadataUserId + ? metadataUserId + : (this.call.invitee ?? this.call.getOpponentMember()?.userId)!; + } + + public get deviceId(): string | undefined { + return this.call.isFocus ? this._metadata?.device_id : this.call.getOpponentDeviceId(); + } + + public get tracks(): RemoteCallTrack[] { + return [...this._tracks]; + } + + public get connected(): boolean { + return this._connected; + } + + private set connected(connected: boolean) { + if (this._connected === connected) return; + this._connected = connected; + this.emit(CallFeedEvent.ConnectedChanged, this.connected); + } + + private onCallState = (): void => { + this.updateConnected(); + }; + + private updateConnected(): void { + if (this.call?.state === CallState.Connecting) { + this.connected = false; + } else if (!this.stream) { + this.connected = false; + } else if (this.stream.getTracks().length === 0) { + this.connected = false; + } else if (this.call?.state === CallState.Connected) { + this.connected = true; + } + } + + private streamIdMatches(transceiver: RTCRtpTransceiver): boolean { + if (!transceiver.mid) return false; + if (this.streamId !== this.call.getRemoteStreamIdByMid(transceiver.mid)) return false; + + return true; + } + + public canAddTransceiver(transceiver: RTCRtpTransceiver): boolean { + if (!transceiver.mid) return false; + + // If the opponent does not support sdp_stream_metadata at all, we + // always allow adding transceivers + if (!this._metadata) return true; + // If the opponent does not support tracks on sdp_stream_metadata, we + // just check the streamId + if (!this._metadata.tracks && this.streamIdMatches(transceiver)) return true; + + if (!this._tracks.some((track) => track.canSetTransceiver(transceiver))) return false; + if (!this.streamIdMatches(transceiver)) return false; + + return true; + } + + public addTransceiver(transceiver: RTCRtpTransceiver): void { + if (!transceiver.mid) { + throw new Error("RemoteCallFeed addTransceiver() called with transceiver without an mid"); + } + if (!transceiver.receiver?.track) { + throw new Error("RemoteCallFeed addTransceiver() called with transceiver without a receiver or track"); + } + if (!this.canAddTransceiver(transceiver)) { + throw new Error("RemoteCallFeed addTransceiver() called with wrong trackId or streamId"); + } + + const track = this._tracks.find((t) => t.canSetTransceiver(transceiver)); + const trackId = this.call.getRemoteTrackIdByMid(transceiver.mid); + + const trackInfo = `streamId=${this.streamId}, trackId=${trackId}, kind=${transceiver.receiver.track.kind}`; + logger.log(`RemoteCallFeed ${this.id} addTransceiver() running (${trackInfo})`); + + if (!track && !this._metadata?.tracks) { + // If the opponent does not support tracks on sdp_stream_metadata or + // it does not support sdp_stream_metadata at all, we simply create + // new tracks + logger.info(`RemoteCallFeed ${this.id} addTransceiver() adding track (${trackInfo})`); + const track = new RemoteCallTrack({ + call: this.call, + trackId, + }); + track.setTransceiver(transceiver); + this._tracks.push(track); + } else if (!track) { + logger.warn(`RemoteCallFeed ${this.id} addTransceiver() did not find track for transceiver (${trackInfo})`); + return; + } else { + track.setTransceiver(transceiver); + } + + this.stream?.addTrack(transceiver.receiver.track); + this.updateConnected(); + this.emit(CallFeedEvent.NewStream, this.stream); + } + + public dispose(): void { + super.dispose(); + this.call?.removeListener(CallEvent.State, this.onCallState); + } +} diff --git a/src/webrtc/remoteCallTrack.ts b/src/webrtc/remoteCallTrack.ts new file mode 100644 index 00000000000..4fc3fa77273 --- /dev/null +++ b/src/webrtc/remoteCallTrack.ts @@ -0,0 +1,93 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { logger } from "../logger"; +import { MatrixCall } from "./call"; +import { SDPStreamMetadataTrack } from "./callEventTypes"; +import { CallTrack, CallTrackOpts } from "./callTrack"; + +export interface RemoteCallTrackOpts extends CallTrackOpts { + call: MatrixCall; + trackId?: string; + metadata?: SDPStreamMetadataTrack; +} + +export class RemoteCallTrack extends CallTrack { + private readonly _trackId?: string; + private _metadata?: SDPStreamMetadataTrack; + private call: MatrixCall; + + public constructor(opts: RemoteCallTrackOpts) { + super(opts); + + this.call = opts.call; + this._trackId = opts.trackId; + } + + public get id(): string | undefined { + return this._trackId; + } + + public get trackId(): string | undefined { + return this._trackId; + } + + public get metadata(): SDPStreamMetadataTrack | undefined { + return this._metadata; + } + + public set metadata(metadata: SDPStreamMetadataTrack | undefined) { + if (!metadata) return; + this._metadata = metadata; + } + + public get track(): MediaStreamTrack | undefined { + return this._transceiver?.receiver?.track; + } + + public get kind(): string | undefined { + return this.track?.kind ?? this._metadata?.kind; + } + + public canSetTransceiver(transceiver: RTCRtpTransceiver): boolean { + if (!this._trackId) return true; + + if (!transceiver.mid) return false; + if (this.call.getRemoteTrackIdByMid(transceiver.mid) !== this._trackId) return false; + + return true; + } + + public setTransceiver(transceiver: RTCRtpTransceiver): void { + if (!this.canSetTransceiver(transceiver)) { + throw new Error("Wrong track_id"); + } + if (!transceiver.receiver.track) { + throw new Error("No receiver or track"); + } + if (!transceiver.mid) { + throw new Error("No mid"); + } + + logger.log( + `RemoteCallTrack ${this.id} setTransceiver() running (${this.call.getRemoteTrackInfoByMid( + transceiver.mid, + )})`, + ); + + this._transceiver = transceiver; + } +} From ba51373c2c8303d4a2283031f8366f40d286c977 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 23 Feb 2023 14:27:27 +0100 Subject: [PATCH 131/137] Move mute state into `CallTrack`s (#3176) --- spec/unit/webrtc/call.spec.ts | 8 ++ spec/unit/webrtc/callFeed.spec.ts | 23 +++-- src/webrtc/call.ts | 2 +- src/webrtc/callFeed.ts | 159 +++++++++++++++--------------- src/webrtc/callTrack.ts | 1 + src/webrtc/groupCall.ts | 10 +- src/webrtc/localCallFeed.ts | 32 +++++- src/webrtc/localCallTrack.ts | 8 ++ src/webrtc/remoteCallFeed.ts | 21 +++- src/webrtc/remoteCallTrack.ts | 14 +++ 10 files changed, 181 insertions(+), 97 deletions(-) diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 3f2861aafbf..586d01e3d2b 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -317,6 +317,14 @@ describe("Call", function () { }), ); + runOnTrackForStream( + call, + new MockMediaStream("remote_stream", [ + new MockMediaStreamTrack("audio", "audio"), + new MockMediaStreamTrack("video", "video"), + ]).typed(), + ); + const feed = call.getFeeds().find((feed) => feed.streamId === "remote_stream"); expect(feed?.purpose).toBe(SDPStreamMetadataPurpose.Usermedia); // @ts-ignore diff --git a/spec/unit/webrtc/callFeed.spec.ts b/spec/unit/webrtc/callFeed.spec.ts index d1b6cea6154..6032fec6995 100644 --- a/spec/unit/webrtc/callFeed.spec.ts +++ b/spec/unit/webrtc/callFeed.spec.ts @@ -15,7 +15,6 @@ limitations under the License. */ import { SDPStreamMetadataPurpose } from "../../../src/webrtc/callEventTypes"; -import { CallFeed } from "../../../src/webrtc/callFeed"; import { TestClient } from "../../TestClient"; import { installWebRTCMocks, MockMatrixCall, MockMediaStream, MockMediaStreamTrack } from "../../test-utils/webrtc"; import { CallEvent, CallState } from "../../../src/webrtc/call"; @@ -26,7 +25,7 @@ describe("CallFeed", () => { const roomId = "room1"; let client: TestClient; let call: MockMatrixCall; - let feed: CallFeed; + let feed: LocalCallFeed; beforeEach(() => { installWebRTCMocks(); @@ -54,41 +53,41 @@ describe("CallFeed", () => { describe("muting", () => { describe("muting by default", () => { it("should mute audio by default", () => { - expect(feed.isAudioMuted()).toBeTruthy(); + expect(feed.audioMuted).toBeTruthy(); }); it("should mute video by default", () => { - expect(feed.isVideoMuted()).toBeTruthy(); + expect(feed.videoMuted).toBeTruthy(); }); }); describe("muting after adding a track", () => { it("should un-mute audio", () => { // @ts-ignore Mock - feed.stream.addTrack(new MockMediaStreamTrack("track", "audio", true)); - expect(feed.isAudioMuted()).toBeFalsy(); + feed.setNewStream(new MockMediaStream("stream", [new MockMediaStreamTrack("track", "audio", true)])); + expect(feed.audioMuted).toBeFalsy(); }); it("should un-mute video", () => { // @ts-ignore Mock - feed.stream.addTrack(new MockMediaStreamTrack("track", "video", true)); - expect(feed.isVideoMuted()).toBeFalsy(); + feed.setNewStream(new MockMediaStream("stream", [new MockMediaStreamTrack("track", "video", true)])); + expect(feed.videoMuted).toBeFalsy(); }); }); describe("muting after calling setAudioVideoMuted()", () => { it("should mute audio by default ", () => { // @ts-ignore Mock - feed.stream.addTrack(new MockMediaStreamTrack("track", "audio", true)); + feed.setNewStream(new MockMediaStream("stream", [new MockMediaStreamTrack("track", "audio", true)])); feed.setAudioVideoMuted(true, false); - expect(feed.isAudioMuted()).toBeTruthy(); + expect(feed.audioMuted).toBeTruthy(); }); it("should mute video by default", () => { // @ts-ignore Mock - feed.stream.addTrack(new MockMediaStreamTrack("track", "video", true)); + feed.setNewStream(new MockMediaStream("stream", [new MockMediaStreamTrack("track", "video", true)])); feed.setAudioVideoMuted(false, true); - expect(feed.isVideoMuted()).toBeTruthy(); + expect(feed.videoMuted).toBeTruthy(); }); }); }); diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index c0bd8785498..5262e6eaba9 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2386,7 +2386,7 @@ export class MatrixCall extends TypedEventEmitter (track.metadataMuted = true)); } } }; diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index ab8e8cd0838..7dfe1857355 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -23,7 +23,6 @@ import { TypedEventEmitter } from "../models/typed-event-emitter"; import { MatrixCall } from "./call"; import { CallTrack } from "./callTrack"; import { randomString } from "../randomstring"; -import { logger } from "../logger"; const POLLING_INTERVAL = 200; // ms export const SPEAKING_THRESHOLD = -60; // dB @@ -75,13 +74,12 @@ export abstract class CallFeed extends TypedEventEmitter track.isAudio); + } + + public get videoTracks(): CallTrack[] { + return this._tracks.filter((track) => track.isVideo); + } + public get audioTrack(): CallTrack | undefined { - return this._tracks.find((track) => track.isAudio); + return this.audioTracks[0]; } public get videoTrack(): CallTrack | undefined { - return this._tracks.find((track) => track.isVideo); + return this.videoTracks[0]; + } + + public get audioMuted(): boolean { + return !this.audioTracks.some((track) => !track.muted); + } + + public get videoMuted(): boolean { + return !this.videoTracks.some((track) => !track.muted); } private get hasAudioTrack(): boolean { @@ -145,25 +157,73 @@ export abstract class CallFeed extends TypedEventEmitter { + if (!this.analyser || !this.frequencyBinCount) { + clearTimeout(this.volumeLooperTimeout); + this.volumeLooperTimeout = undefined; + return; + } + + this.analyser.getFloatFrequencyData(this.frequencyBinCount); + + let maxVolume = -Infinity; + for (const volume of this.frequencyBinCount!) { + if (volume > maxVolume) { + maxVolume = volume; + } + } - this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount); + this.speakingVolumeSamples.shift(); + this.speakingVolumeSamples.push(maxVolume); + + this.emit(CallFeedEvent.VolumeChanged, maxVolume); + + let newSpeaking = false; + + for (const volume of this.speakingVolumeSamples) { + if (volume > this.speakingThreshold) { + newSpeaking = true; + break; + } + } + + if (this.speaking !== newSpeaking) { + this.speaking = newSpeaking; + this.emit(CallFeedEvent.Speaking, this.speaking); + } + + this.volumeLooperTimeout = setTimeout(loop, POLLING_INTERVAL); + }; + + loop(); } /** @@ -178,45 +238,27 @@ export abstract class CallFeed extends TypedEventEmitter { - if (!this.analyser) return; - if (!this.hasAudioTrack) return; - if (!this.frequencyBinCount) return; - if (!this.measuringVolumeActivity) return; - - this.analyser.getFloatFrequencyData(this.frequencyBinCount!); - - let maxVolume = -Infinity; - for (const volume of this.frequencyBinCount!) { - if (volume > maxVolume) { - maxVolume = volume; - } - } - - this.speakingVolumeSamples.shift(); - this.speakingVolumeSamples.push(maxVolume); - - this.emit(CallFeedEvent.VolumeChanged, maxVolume); - - let newSpeaking = false; - - for (const volume of this.speakingVolumeSamples) { - if (volume > this.speakingThreshold) { - newSpeaking = true; - break; - } - } - - if (this.speaking !== newSpeaking) { - this.speaking = newSpeaking; - this.emit(CallFeedEvent.Speaking, this.speaking); - } - - this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL); - }; - public dispose(): void { clearTimeout(this.volumeLooperTimeout); if (this.audioContext) { diff --git a/src/webrtc/callTrack.ts b/src/webrtc/callTrack.ts index 37d7c2e8144..37311102001 100644 --- a/src/webrtc/callTrack.ts +++ b/src/webrtc/callTrack.ts @@ -25,6 +25,7 @@ export abstract class CallTrack { public abstract get trackId(): string | undefined; public abstract get metadata(): SDPStreamMetadataTrack | undefined; public abstract get kind(): string | undefined; + public abstract get muted(): boolean; protected readonly _id: string; protected _transceiver?: RTCRtpTransceiver; diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index e653d383cd6..d1a8a43e21b 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -424,11 +424,11 @@ export class GroupCall extends TypedEventEmitter< public async updateLocalUsermediaStream(stream: MediaStream): Promise { if (this.localCallFeed) { const oldStream = this.localCallFeed.stream; - this.localCallFeed.setNewStream(stream); - const micShouldBeMuted = this.localCallFeed.isAudioMuted(); - const vidShouldBeMuted = this.localCallFeed.isVideoMuted(); + const micShouldBeMuted = this.localCallFeed.audioMuted; + const vidShouldBeMuted = this.localCallFeed.videoMuted; setTracksEnabled(stream.getAudioTracks(), !micShouldBeMuted); setTracksEnabled(stream.getVideoTracks(), !vidShouldBeMuted); + this.localCallFeed.setNewStream(stream); if (oldStream) { this.client.getMediaHandler().stopUserMediaStream(oldStream); @@ -576,7 +576,7 @@ export class GroupCall extends TypedEventEmitter< public isLocalVideoMuted(): boolean { if (this.localCallFeed) { - return this.localCallFeed.isVideoMuted(); + return this.localCallFeed.videoMuted; } return true; @@ -584,7 +584,7 @@ export class GroupCall extends TypedEventEmitter< public isMicrophoneMuted(): boolean { if (this.localCallFeed) { - return this.localCallFeed.isAudioMuted(); + return this.localCallFeed.audioMuted; } return true; diff --git a/src/webrtc/localCallFeed.ts b/src/webrtc/localCallFeed.ts index eba08863e69..c2b5ff1fa4c 100644 --- a/src/webrtc/localCallFeed.ts +++ b/src/webrtc/localCallFeed.ts @@ -17,7 +17,7 @@ limitations under the License. import { logger } from "../logger"; import { MatrixCall } from "./call"; import { SDPStreamMetadataObject, SDPStreamMetadataPurpose, SDPStreamMetadataTracks } from "./callEventTypes"; -import { CallFeed, ICallFeedOpts } from "./callFeed"; +import { CallFeed, CallFeedEvent, ICallFeedOpts } from "./callFeed"; import { LocalCallTrack } from "./localCallTrack"; export interface LocalCallFeedOpts extends ICallFeedOpts { @@ -54,6 +54,14 @@ export class LocalCallFeed extends CallFeed { return super.tracks as LocalCallTrack[]; } + public get audioTracks(): LocalCallTrack[] { + return super.audioTracks as LocalCallTrack[]; + } + + public get videoTracks(): LocalCallTrack[] { + return super.videoTracks as LocalCallTrack[]; + } + public get metadata(): SDPStreamMetadataObject { return { user_id: this.userId, @@ -86,6 +94,28 @@ export class LocalCallFeed extends CallFeed { return this._tracks[0]?.streamId; } + /** + * Set one or both of feed's internal audio and video video mute state + * Either value may be null to leave it as-is + * @param audioMuted - is the feed's audio muted? + * @param videoMuted - is the feed's video muted? + */ + public setAudioVideoMuted(audioMuted: boolean | null, videoMuted: boolean | null): void { + logger.log(`CallFeed ${this.id} setAudioVideoMuted() running (audio=${audioMuted}, video=${videoMuted})`); + + if (audioMuted !== null) { + if (this.audioMuted !== audioMuted) { + this.speakingVolumeSamples.fill(-Infinity); + } + this.audioTracks.forEach((track) => (track.muted = audioMuted)); + } + if (videoMuted !== null) { + this.videoTracks.forEach((track) => (track.muted = videoMuted)); + } + + this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); + } + public clone(): LocalCallFeed { const mediaHandler = this.client.getMediaHandler(); const stream = this._stream.clone(); diff --git a/src/webrtc/localCallTrack.ts b/src/webrtc/localCallTrack.ts index c56c94a8391..8d046dc5abe 100644 --- a/src/webrtc/localCallTrack.ts +++ b/src/webrtc/localCallTrack.ts @@ -151,6 +151,14 @@ export class LocalCallTrack extends CallTrack { return this.track.kind; } + public get muted(): boolean { + return !this.track.enabled; + } + + public set muted(muted: boolean) { + this.track.enabled = !muted; + } + public get purpose(): SDPStreamMetadataPurpose { return this.feed.purpose; } diff --git a/src/webrtc/remoteCallFeed.ts b/src/webrtc/remoteCallFeed.ts index 8a44575a15b..af4a57cd6af 100644 --- a/src/webrtc/remoteCallFeed.ts +++ b/src/webrtc/remoteCallFeed.ts @@ -75,9 +75,11 @@ export class RemoteCallFeed extends CallFeed { public set metadata(metadata: SDPStreamMetadataObject | undefined) { if (!metadata) return; - this.setAudioVideoMuted(metadata.audio_muted ?? false, metadata.video_muted ?? false); this._metadata = metadata; + this.audioTracks.forEach((track) => (track.metadataMuted = metadata.audio_muted ?? false)); + this.videoTracks.forEach((track) => (track.metadataMuted = metadata.video_muted ?? false)); + if (!metadata.tracks) return; for (const [metadataTrackId, metadataTrack] of Object.entries(metadata.tracks)) { const track = this._tracks.find((track) => track.trackId === metadataTrackId); @@ -93,6 +95,8 @@ export class RemoteCallFeed extends CallFeed { new RemoteCallTrack({ call: this.call, trackId: metadataTrackId, + metadataMuted: + (metadataTrack.kind === "audio" ? metadata.audio_muted : metadata.video_muted) ?? false, metadata: metadataTrack, }), ); @@ -110,6 +114,8 @@ export class RemoteCallFeed extends CallFeed { } } } + + this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); } public get purpose(): SDPStreamMetadataPurpose { @@ -133,6 +139,14 @@ export class RemoteCallFeed extends CallFeed { return [...this._tracks]; } + public get audioTracks(): RemoteCallTrack[] { + return super.audioTracks as RemoteCallTrack[]; + } + + public get videoTracks(): RemoteCallTrack[] { + return super.videoTracks as RemoteCallTrack[]; + } + public get connected(): boolean { return this._connected; } @@ -206,6 +220,10 @@ export class RemoteCallFeed extends CallFeed { logger.info(`RemoteCallFeed ${this.id} addTransceiver() adding track (${trackInfo})`); const track = new RemoteCallTrack({ call: this.call, + metadataMuted: + (transceiver.receiver.track.kind === "audio" + ? this._metadata?.audio_muted + : this._metadata?.video_muted) ?? false, trackId, }); track.setTransceiver(transceiver); @@ -218,6 +236,7 @@ export class RemoteCallFeed extends CallFeed { } this.stream?.addTrack(transceiver.receiver.track); + this.startMeasuringVolume(); this.updateConnected(); this.emit(CallFeedEvent.NewStream, this.stream); } diff --git a/src/webrtc/remoteCallTrack.ts b/src/webrtc/remoteCallTrack.ts index 4fc3fa77273..e56abc7de00 100644 --- a/src/webrtc/remoteCallTrack.ts +++ b/src/webrtc/remoteCallTrack.ts @@ -23,11 +23,13 @@ export interface RemoteCallTrackOpts extends CallTrackOpts { call: MatrixCall; trackId?: string; metadata?: SDPStreamMetadataTrack; + metadataMuted?: boolean; } export class RemoteCallTrack extends CallTrack { private readonly _trackId?: string; private _metadata?: SDPStreamMetadataTrack; + private _metadataMuted?: boolean; private call: MatrixCall; public constructor(opts: RemoteCallTrackOpts) { @@ -35,6 +37,8 @@ export class RemoteCallTrack extends CallTrack { this.call = opts.call; this._trackId = opts.trackId; + this.metadata = opts.metadata; + this.metadataMuted = opts.metadataMuted; } public get id(): string | undefined { @@ -62,6 +66,16 @@ export class RemoteCallTrack extends CallTrack { return this.track?.kind ?? this._metadata?.kind; } + public get muted(): boolean { + if (!this.track) return true; + + return this._metadataMuted ?? false; + } + + public set metadataMuted(metadataMuted: boolean | undefined) { + this._metadataMuted = metadataMuted; + } + public canSetTransceiver(transceiver: RTCRtpTransceiver): boolean { if (!this._trackId) return true; From 30ece895ba09a312b16dd683c879f77ec847f277 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 28 Feb 2023 15:08:58 +0100 Subject: [PATCH 132/137] Remove nonsensical test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- spec/unit/webrtc/groupCall.spec.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index d39fa6964a9..053b437cf9e 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -385,14 +385,6 @@ describe("Group Call", function () { await groupCall.create(); }); - it("ignores changes, if we can't get user id of opponent", async () => { - const call = new MockMatrixCall(room.roomId, groupCall.groupCallId); - jest.spyOn(call, "getOpponentMember").mockReturnValue({ userId: undefined }); - - // @ts-ignore Mock - expect(() => groupCall.onCallFeedsChanged(call)).toThrow(); - }); - describe("usermedia feeds", () => { beforeEach(() => { currentFeed.purpose = SDPStreamMetadataPurpose.Usermedia; From 97404ae55197ed39af8f427ebc78fe8341a973b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 28 Feb 2023 15:24:38 +0100 Subject: [PATCH 133/137] Use util MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- spec/unit/webrtc/groupCall.spec.ts | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index 053b437cf9e..51664709544 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -955,32 +955,7 @@ describe("Group Call", function () { call.getOpponentMember = () => ({ userId: call.invitee } as RoomMember); call.onSDPStreamMetadataChangedReceived(metadataEvent); - let sdp = - "v=0\n" + - "o=- 7135465365607179083 2 IN IP4 127.0.0.1\n" + - "s=-\n" + - "t=0 0\n" + - "a=group:BUNDLE 0 1 2\n"; - - stream.getTracks().forEach((track, index) => { - sdp += `m=${track.kind}\n`; - sdp += `a=mid:${index}\n`; - sdp += `a=msid:${stream.id} ${track.id}\n`; - }); - - // @ts-ignore - call.peerConn.remoteDescription = { - sdp: sdp, - }; - - stream.getTracks().forEach((track, index) => { - // @ts-ignore - call.onTrack({ - track, - streams: [stream], - transceiver: { mid: `${index}`, receiver: { track } }, - } as TrackEvent); - }); + runOnTrackForStream(call, stream); const feed = groupCall.getUserMediaFeed(call.invitee!, call.getOpponentDeviceId()!); expect(feed!.isAudioMuted()).toBe(false); From 6043a13c4e6c9b7d63fb36168eeb6c5ef4c2bbcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 28 Feb 2023 15:26:37 +0100 Subject: [PATCH 134/137] Use util MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- spec/unit/webrtc/call.spec.ts | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index f78778b577b..6729e9197d4 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -911,32 +911,7 @@ describe("Call", function () { new MockMediaStreamTrack("track1", "audio"), new MockMediaStreamTrack("track2", "video"), ]); - let sdp = - "v=0\n" + - "o=- 7135465365607179083 2 IN IP4 127.0.0.1\n" + - "s=-\n" + - "t=0 0\n" + - "a=group:BUNDLE 0 1 2\n"; - - stream.getTracks().forEach((track, index) => { - sdp += `m=${track.kind}\n`; - sdp += `a=mid:${index}\n`; - sdp += `a=msid:${stream.id} ${track.id}\n`; - }); - - // @ts-ignore - call.peerConn.remoteDescription = { - sdp: sdp, - }; - - stream.getTracks().forEach((track, index) => { - // @ts-ignore - call.onTrack({ - track, - streams: [stream], - transceiver: { mid: `${index}`, receiver: { track } }, - } as TrackEvent); - }); + runOnTrackForStream(call, stream); }; it("should handle incoming sdp_stream_metadata_changed with audio muted", async () => { From 7fa429438ae775dcc738bca89e76a940a665f64b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 6 Mar 2023 17:24:03 +0100 Subject: [PATCH 135/137] Abstract publishing (#3183) --- spec/unit/webrtc/call.spec.ts | 11 +- spec/unit/webrtc/groupCall.spec.ts | 2 +- src/webrtc/call.ts | 91 +++++++++++--- src/webrtc/callFeed.ts | 9 +- src/webrtc/callTrack.ts | 12 +- src/webrtc/feedPublication.ts | 73 +++++++++++ src/webrtc/localCallFeed.ts | 84 +++++++------ src/webrtc/localCallTrack.ts | 196 ++++++----------------------- src/webrtc/remoteCallFeed.ts | 4 + src/webrtc/remoteCallTrack.ts | 5 + src/webrtc/trackPublication.ts | 123 ++++++++++++++++++ 11 files changed, 381 insertions(+), 229 deletions(-) create mode 100644 src/webrtc/feedPublication.ts create mode 100644 src/webrtc/trackPublication.ts diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 6729e9197d4..9f7678af7e5 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -325,7 +325,7 @@ describe("Call", function () { ]).typed(), ); - const feed = call.getFeeds().find((feed) => feed.streamId === "remote_stream"); + const feed = call.getRemoteFeeds().find((feed) => feed.streamId === "remote_stream"); expect(feed?.purpose).toBe(SDPStreamMetadataPurpose.Usermedia); // @ts-ignore expect(feed?.audioMuted).toBeTruthy(); @@ -452,7 +452,7 @@ describe("Call", function () { }, }); //(call as any).pushRemoteStream(new MockMediaStream("remote_stream", [])); - const feed = call.getFeeds().find((feed) => feed.streamId === "remote_stream"); + const feed = call.getRemoteFeeds().find((feed) => feed.streamId === "remote_stream"); call.onSDPStreamMetadataChangedReceived( makeMockEvent("@test:foo", { @@ -911,7 +911,7 @@ describe("Call", function () { new MockMediaStreamTrack("track1", "audio"), new MockMediaStreamTrack("track2", "video"), ]); - runOnTrackForStream(call, stream); + runOnTrackForStream(call, stream.typed()); }; it("should handle incoming sdp_stream_metadata_changed with audio muted", async () => { @@ -1300,8 +1300,9 @@ describe("Call", function () { MockRTCPeerConnection.triggerAllNegotiations(); // @ts-ignore - const mockVideoSender = call.peerConn.getSenders().find((s) => s.track!.kind === "video"); - const mockReplaceTrack = (mockVideoSender!.replaceTrack = jest.fn()); + const mockVideoSender = call.peerConn.getSenders().find((s) => s.track!.kind === "video")!; + jest.spyOn(mockVideoSender, "replaceTrack"); + const mockReplaceTrack = mocked(mockVideoSender?.replaceTrack); await call.setScreensharingEnabled(true); diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index 51664709544..ba6c3396e60 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -955,7 +955,7 @@ describe("Group Call", function () { call.getOpponentMember = () => ({ userId: call.invitee } as RoomMember); call.onSDPStreamMetadataChangedReceived(metadataEvent); - runOnTrackForStream(call, stream); + runOnTrackForStream(call, stream.typed()); const feed = groupCall.getUserMediaFeed(call.invitee!, call.getOpponentDeviceId()!); expect(feed!.isAudioMuted()).toBe(false); diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index e1badcdc55c..e486ffa53ae 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -62,6 +62,8 @@ import { MatrixError } from "../http-api"; import { RemoteCallFeed } from "./remoteCallFeed"; import { LocalCallFeed } from "./localCallFeed"; import { LocalCallTrack } from "./localCallTrack"; +import { TrackPublication } from "./trackPublication"; +import { FeedPublication } from "./feedPublication"; interface CallOpts { // The room ID for this call. @@ -350,6 +352,8 @@ export class MatrixCall extends TypedEventEmitter = []; + private trackPublications: TrackPublication[] = []; + private feedPublications: FeedPublication[] = []; private inviteOrAnswerSent = false; private waitForLocalAVStream = false; @@ -540,11 +544,15 @@ export class MatrixCall extends TypedEventEmitter t.kind === "audio")?.sender); + return this.trackPublications.some( + ({ track }) => track.purpose === SDPStreamMetadataPurpose.Usermedia && track.isAudio, + ); } private get hasUserMediaVideoSender(): boolean { - return Boolean(this.localUsermediaFeed?.tracks?.find((t) => t.kind === "video")?.sender); + return this.trackPublications.some( + ({ track }) => track.purpose === SDPStreamMetadataPurpose.Usermedia && track.isVideo, + ); } public get localUsermediaFeed(): LocalCallFeed | undefined { @@ -579,7 +587,7 @@ export class MatrixCall extends TypedEventEmitter { if (transceiver.sender.track) return false; if (!transceiver.mid) return false; @@ -699,10 +707,10 @@ export class MatrixCall extends TypedEventEmitter { - if (!feed.streamId) return metadata; + return this.feedPublications.reduce((metadata: SDPStreamMetadata, publication: FeedPublication) => { + if (!publication.streamId) return metadata; - metadata[feed.streamId] = feed.metadata; + metadata[publication.streamId] = publication.metadata; return metadata; }, {}); } @@ -761,15 +769,23 @@ export class MatrixCall extends TypedEventEmitter publication.track === track)) { + throw new Error("Cannot publish a track that is already published"); + } + logger.info( + `Call ${this.callId} publishTrackUsingNewTransceiver() running (id=${track.id}, kind=${track.kind})`, + ); const { stream, encodings } = track; @@ -797,16 +813,55 @@ export class MatrixCall extends TypedEventEmitter publication.track === track)) { + throw new Error("Cannot publish a track that is already published"); } - logger.info(`Call ${this.callId} unpublishTrack() running (id=${track.id}, kind=${track.kind})`); + logger.info(`Call ${this.callId} publishTrack() running (id=${track.id}, kind=${track.kind})`); + + // XXX: We try to re-use transceivers here, but we don't re-use + // transceivers with the SFU: this is to work around + // https://github.com/matrix-org/waterfall/issues/98 - see the bug for + // more. Since we use WebRTC data channels to renegotiate with the SFU, + // we're not limited to the size of a Matrix event, so it's 'ok' if the + // SDP grows indefinitely (although presumably this would break if we + // tried to do an ICE restart over to-device messages after you'd turned + // screen sharing on & off too many times...) + let transceiver = this.getFreeTransceiverByKind(track.kind); + if (!transceiver || (this.isFocus && track.purpose === SDPStreamMetadataPurpose.Screenshare)) { + transceiver = this.publishTrackOnNewTransceiver(track); + } + + const publication = new TrackPublication({ + call: this, + track, + transceiver, + }); + + this.trackPublications.push(publication); + return publication; + } + + public unpublishTrack(publication: TrackPublication): void { + logger.info(`Call ${this.callId} unpublishTrack() running (${publication.logInfo})`); - this.peerConn?.removeTrack(track.sender); + this.unpublishTrackOnTransceiver(publication); + this.trackPublications = this.trackPublications.splice(this.trackPublications.indexOf(publication), 1); } /** @@ -815,7 +870,7 @@ export class MatrixCall extends TypedEventEmitter track.purpose === SDPStreamMetadataPurpose.Screenshare && track.isVideo, + )?.transceiver; if (screensharingVideoTransceiver) screensharingVideoTransceiver.setCodecPreferences(codecs); } diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 7dfe1857355..37a19ce7fdd 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -20,7 +20,6 @@ import { acquireContext, releaseContext } from "./audioContext"; import { MatrixClient } from "../client"; import { RoomMember } from "../models/room-member"; import { TypedEventEmitter } from "../models/typed-event-emitter"; -import { MatrixCall } from "./call"; import { CallTrack } from "./callTrack"; import { randomString } from "../randomstring"; @@ -55,9 +54,14 @@ type EventHandlerMap = { [CallFeedEvent.Disposed]: () => void; }; +/** + * CallFeed is a wrapper around a MediaStream. It includes useful information + * such as the userId and deviceId of the stream's sender, mute state, volume + * activity etc. This class would be usually used to display the video tiles in + * the UI. + */ export abstract class CallFeed extends TypedEventEmitter { public abstract get id(): string; - public abstract get streamId(): string | undefined; public abstract get purpose(): SDPStreamMetadataPurpose; public abstract get connected(): boolean; public abstract get userId(): string; @@ -71,7 +75,6 @@ export abstract class CallFeed extends TypedEventEmitter { + if (!publication.trackId) return metadata; + + metadata[publication.trackId] = publication.metadata; + return metadata; + }, + {}, + ), + }; + } + + public get streamId(): string | undefined { + return this.trackPublications[0]?.streamId; + } + + public addTrackPublication(publication: TrackPublication): void { + this.trackPublications.push(publication); + } + + public removeTrackPublication(publication: TrackPublication): void { + this.trackPublications.splice(this.trackPublications.indexOf(publication), 1); + } +} diff --git a/src/webrtc/localCallFeed.ts b/src/webrtc/localCallFeed.ts index c2b5ff1fa4c..6726710e742 100644 --- a/src/webrtc/localCallFeed.ts +++ b/src/webrtc/localCallFeed.ts @@ -16,8 +16,9 @@ limitations under the License. import { logger } from "../logger"; import { MatrixCall } from "./call"; -import { SDPStreamMetadataObject, SDPStreamMetadataPurpose, SDPStreamMetadataTracks } from "./callEventTypes"; +import { SDPStreamMetadataPurpose } from "./callEventTypes"; import { CallFeed, CallFeedEvent, ICallFeedOpts } from "./callFeed"; +import { FeedPublication } from "./feedPublication"; import { LocalCallTrack } from "./localCallTrack"; export interface LocalCallFeedOpts extends ICallFeedOpts { @@ -25,8 +26,15 @@ export interface LocalCallFeedOpts extends ICallFeedOpts { stream: MediaStream; } +/** + * LocalCallFeed is a wrapper around MediaStream. It represents a stream that + * we've created locally by getting user/display media. N.B. that this is not + * linked to a specific peer connection, a FeedPublication is used for that + * purpose. + */ export class LocalCallFeed extends CallFeed { protected _tracks: LocalCallTrack[] = []; + protected publications: FeedPublication[] = []; private _purpose: SDPStreamMetadataPurpose; protected _stream: MediaStream; @@ -62,22 +70,6 @@ export class LocalCallFeed extends CallFeed { return super.videoTracks as LocalCallTrack[]; } - public get metadata(): SDPStreamMetadataObject { - return { - user_id: this.userId, - device_id: this.deviceId, - purpose: this.purpose, - audio_muted: this.isAudioMuted(), - video_muted: this.isVideoMuted(), - tracks: this._tracks.reduce((metadata: SDPStreamMetadataTracks, track: LocalCallTrack) => { - if (!track.trackId) return metadata; - - metadata[track.trackId] = track.metadata; - return metadata; - }, {}), - }; - } - public get purpose(): SDPStreamMetadataPurpose { return this._purpose; } @@ -90,10 +82,6 @@ export class LocalCallFeed extends CallFeed { return this.client.getDeviceId() ?? undefined; } - public get streamId(): string | undefined { - return this._tracks[0]?.streamId; - } - /** * Set one or both of feed's internal audio and video video mute state * Either value may be null to leave it as-is @@ -146,18 +134,16 @@ export class LocalCallFeed extends CallFeed { protected updateStream(oldStream?: MediaStream, newStream?: MediaStream): void { super.updateStream(oldStream, newStream); - if (!newStream) return; - // First, remove tracks which won't be used anymore for (const track of this._tracks) { - if (!newStream.getTracks().some((streamTrack) => streamTrack.kind === track.kind)) { + if (!newStream?.getTracks().some((streamTrack) => streamTrack.kind === track.kind)) { this._tracks.splice(this._tracks.indexOf(track), 1); - if (track.published) { - track.unpublish(); - } + this.publications.forEach((publication) => this.unpublishTrack(track, publication)); } } + if (!newStream) return; + // Then, replace old track where we can and add new tracks for (const streamTrack of newStream.getTracks()) { let track = this._tracks.find((track) => track.kind === streamTrack.kind); @@ -171,24 +157,42 @@ export class LocalCallFeed extends CallFeed { track: streamTrack, }); this._tracks.push(track); - - if (this.call) { - track.publish(this.call); - } + this.publications.forEach((publication) => this.publishTrack(track!, publication)); } } - public publish(call: MatrixCall): void { - this.call = call; - for (const track of this._tracks) { - track.publish(call); + public publish(call: MatrixCall): FeedPublication { + if (this.publications.some((publication) => publication.call === call)) { + throw new Error("Cannot publish a feed that is already published"); } + + const feedPublication = new FeedPublication({ + feed: this, + call, + }); + this.tracks.forEach((track) => this.publishTrack(track, feedPublication)); + + this.publications.push(feedPublication); + return feedPublication; } - public unpublish(): void { - this.call = undefined; - for (const track of this._tracks) { - track.unpublish(); - } + public unpublish(call: MatrixCall): void { + const feedPublication = this.publications.find((publication) => publication.call === call); + if (!feedPublication) return; + + this.publications.splice(this.publications.indexOf(feedPublication), 1); + this.tracks.forEach((track) => this.unpublishTrack(track, feedPublication)); + } + + private publishTrack(track: LocalCallTrack, feedPublication: FeedPublication): void { + const trackPublication = track.publish(feedPublication.call); + if (!trackPublication) return; + feedPublication.addTrackPublication(trackPublication); + } + + private unpublishTrack(track: LocalCallTrack, feedPublication: FeedPublication): void { + const trackPublication = track.unpublish(feedPublication.call); + if (!trackPublication) return; + feedPublication.removeTrackPublication(trackPublication); } } diff --git a/src/webrtc/localCallTrack.ts b/src/webrtc/localCallTrack.ts index 8d046dc5abe..2a97d36b094 100644 --- a/src/webrtc/localCallTrack.ts +++ b/src/webrtc/localCallTrack.ts @@ -16,9 +16,10 @@ limitations under the License. import { logger } from "../logger"; import { MatrixCall } from "./call"; -import { SDPStreamMetadataPurpose, SDPStreamMetadataTrack } from "./callEventTypes"; +import { SDPStreamMetadataPurpose } from "./callEventTypes"; import { CallTrack, CallTrackOpts } from "./callTrack"; import { LocalCallFeed } from "./localCallFeed"; +import { TrackPublication } from "./trackPublication"; export enum SimulcastResolution { Full = "f", @@ -96,10 +97,16 @@ export interface LocalCallTrackOpts extends CallTrackOpts { track: MediaStreamTrack; } +/** + * LocalCallTrack is a wrapper around a MediaStream. It represents a track of a + * stream which we retrieved using get user/display media. N.B. that this is not + * linked to a specific peer connection, a TrackPublication is used for that + * purpose. + */ export class LocalCallTrack extends CallTrack { private _track: MediaStreamTrack; private feed: LocalCallFeed; - private call?: MatrixCall; + private publications: TrackPublication[] = []; public constructor(opts: LocalCallTrackOpts) { super(opts); @@ -109,40 +116,13 @@ export class LocalCallTrack extends CallTrack { } private get logInfo(): string { - return `streamId=${this.streamId}, trackId=${this.trackId}, mid=${this.mid} kind=${this.kind}`; + return `kind=${this.kind}`; } public get id(): string | undefined { return this._id; } - public get metadata(): SDPStreamMetadataTrack { - const trackMetadata: SDPStreamMetadataTrack = { - kind: this.track.kind, - }; - - if (this.isVideo) { - trackMetadata.width = this.track.getSettings().width; - trackMetadata.height = this.track.getSettings().height; - } - - return trackMetadata; - } - - public get mid(): string | undefined { - return this._transceiver?.mid ?? undefined; - } - - public get trackId(): string | undefined { - const mid = this._transceiver?.mid; - return mid ? this.call?.getLocalTrackIdByMid(mid) : undefined; - } - - public get streamId(): string | undefined { - const mid = this._transceiver?.mid; - return mid ? this.call?.getLocalStreamIdByMid(mid) : undefined; - } - public get track(): MediaStreamTrack { return this._track; } @@ -167,153 +147,59 @@ export class LocalCallTrack extends CallTrack { return this.feed.stream; } - public get sender(): RTCRtpSender | undefined { - return this._transceiver?.sender; - } - public get encodings(): RTCRtpEncodingParameters[] { return getSimulcastEncodings(this.purpose); } - public get published(): boolean { - if (!this._transceiver?.sender) return false; - if (!this.call) return false; - - return true; - } - - public publish(call: MatrixCall): void { - if (this.published) { - throw new Error("Cannot publish already published track"); - } - - const oldCall = this.call; - - // We try to re-use transceivers here - const transceiver = call.getFreeTransceiverByKind(this.kind); - if (transceiver) { - try { - this._transceiver = transceiver; - this.call = call; - this.replaceTrackOnPeerConnection(); - } catch (error) { - this._transceiver = undefined; - this.call = oldCall; - } - } else { - try { - this.call = call; - this.addTrackToPeerConnection(); - } catch (error) { - this.call = undefined; - logger.warn( - `LocalCallTrack ${this.id} publish() failed to publish track to call (callId${call.callId})`, - ); - } - } - } - - public unpublish(): void { - const call = this.call; - if (!this.published || !call) { - throw new Error("Cannot unpublish track that is not published"); + public publish(call: MatrixCall): TrackPublication | undefined { + if (this.publications.some((publication) => publication.call === call)) { + throw new Error("Cannot publish a track that is already published"); } try { - call.unpublishTrack(this); - this.call = undefined; - this._transceiver = undefined; + const publication = call.publishTrack(this); + this.publications.push(publication); + return publication; } catch (error) { - logger.warn( - `LocalCallTrack ${this.id} unpublish() failed to publish track to call (callId${call.callId})`, + logger.error( + `LocalCallTrack ${this.id} publish() failed to publish track to call (callId=${call.callId}):`, error, ); } } - public setNewTrack(track: MediaStreamTrack): void { - logger.log(`LocalCallTrack ${this.id} setNewTrack() running (${this.logInfo})`); - - const oldTrack = this._track; - this._track = track; - - // If the track is not published, we don't need to try to publish the - // new one either - if (!this.published || !this.call) return; + public unpublish(call: MatrixCall): TrackPublication | undefined { + const publication = this.publications.find((publication) => publication.call === call); + if (!publication) return; try { - // XXX: We don't re-use transceivers with the SFU: this is to work around - // https://github.com/matrix-org/waterfall/issues/98 - see the bug for more. - // Since we use WebRTC data channels to renegotiate with the SFU, we're not - // limited to the size of a Matrix event, so it's 'ok' if the SDP grows - // indefinitely (although presumably this would break if we tried to do - // an ICE restart over to-device messages after you'd turned screen sharing - // on & off too many times...) - if (!this.sender || (this.call.isFocus && this.purpose === SDPStreamMetadataPurpose.Screenshare)) { - this.unpublish(); - this.addTrackToPeerConnection(); - } else { - this.replaceTrackOnPeerConnection(); - } - - logger.log(`LocalCallTrack ${this.id} setNewTrack() updated published track (${this.logInfo})`); + publication?.unpublish(); + this.publications.splice(this.publications.indexOf(publication), 1); + return publication; } catch (error) { - this._track = oldTrack; - logger.log( - `LocalCallTrack ${this.id} setNewTrack() failed to update published track (${this.logInfo})`, + logger.error( + `LocalCallTrack ${this.id} unpublish() failed to unpublish track to call (callId=${publication.call.callId})`, error, ); } } - private addTrackToPeerConnection(): void { - if (!this.call) { - throw new Error("Called without a call"); - } - this._transceiver = this.call.publishTrack(this); - } - - private replaceTrackOnPeerConnection(): void { - const stream = this.stream; - const sender = this.sender; - const transceiver = this._transceiver; - - if (!stream || !transceiver || !sender || !this.track) { - throw new Error("Called without"); - } - - logger.log(`LocalCallTrack LocalCallTrack ${this.id} replaceTrack() running (${this.logInfo})`); - - try { - // We already have a sender, so we re-use it. We try to - // re-use transceivers as much as possible because they - // can't be removed once added, so otherwise they just - // accumulate which makes the SDP very large very quickly: - // in fact it only takes about 6 video tracks to exceed the - // maximum size of an Olm-encrypted Matrix event - Dave - - // setStreams() is currently not supported by Firefox but we - // try to use it at least in other browsers (once we switch - // to using mids and throw away streamIds we will be able to - // throw this away) - if (sender.setStreams && stream) sender.setStreams(stream); - - sender.replaceTrack(this.track); - - // We don't need to set simulcast encodings in here since we - // have already done that the first time we added the - // transceiver + public setNewTrack(track: MediaStreamTrack): void { + logger.log(`LocalCallTrack ${this.id} setNewTrack() running (${this.logInfo})`); + this._track = track; - // Set the direction of the transceiver to indicate we're - // going to be sending. This may trigger re-negotiation, if - // we weren't sending until now - transceiver.direction = transceiver.direction === "inactive" ? "sendonly" : "sendrecv"; - } catch (error) { - logger.warn( - `LocalCallTrack ${this.id} setNewTrack() failed to replace track: falling back to adding a new one (${this.logInfo})`, - error, - ); - this.addTrackToPeerConnection(); + for (const publication of this.publications) { + try { + publication.updateSenderTrack(); + logger.log( + `LocalCallTrack ${this.id} setNewTrack() updated published track (callId=${publication.call.callId}, ${this.logInfo})`, + ); + } catch (error) { + logger.log( + `LocalCallTrack ${this.id} setNewTrack() failed to update published track (callId=${publication.call.callId}, ${this.logInfo})`, + error, + ); + } } } } diff --git a/src/webrtc/remoteCallFeed.ts b/src/webrtc/remoteCallFeed.ts index af4a57cd6af..316f76e9162 100644 --- a/src/webrtc/remoteCallFeed.ts +++ b/src/webrtc/remoteCallFeed.ts @@ -31,6 +31,10 @@ export interface RemoteCallFeedOpts extends ICallFeedOpts { stream?: MediaStream; } +/** + * RemoteCallFeed is a wrapper around MediaStream. It represents an incoming + * stream. + */ export class RemoteCallFeed extends CallFeed { private _connected = false; private _metadata?: SDPStreamMetadataObject; diff --git a/src/webrtc/remoteCallTrack.ts b/src/webrtc/remoteCallTrack.ts index e56abc7de00..39b16ad4aa7 100644 --- a/src/webrtc/remoteCallTrack.ts +++ b/src/webrtc/remoteCallTrack.ts @@ -26,10 +26,15 @@ export interface RemoteCallTrackOpts extends CallTrackOpts { metadataMuted?: boolean; } +/** + * RemoteCallTrack is a wrapper around MediaStreamTrack. It represent an + * incoming track. + */ export class RemoteCallTrack extends CallTrack { private readonly _trackId?: string; private _metadata?: SDPStreamMetadataTrack; private _metadataMuted?: boolean; + private _transceiver?: RTCRtpTransceiver; private call: MatrixCall; public constructor(opts: RemoteCallTrackOpts) { diff --git a/src/webrtc/trackPublication.ts b/src/webrtc/trackPublication.ts new file mode 100644 index 00000000000..5a83185c9b5 --- /dev/null +++ b/src/webrtc/trackPublication.ts @@ -0,0 +1,123 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { logger } from "../logger"; +import { MatrixCall } from "./call"; +import { SDPStreamMetadataTrack } from "./callEventTypes"; +import { LocalCallTrack } from "./localCallTrack"; + +interface TrackPublicationOpts { + call: MatrixCall; + track: LocalCallTrack; + transceiver: RTCRtpTransceiver; +} + +/** + * TrackPublication represents a LocalCallTrack being published to a specific peer + * connection. + */ +export class TrackPublication { + public readonly call: MatrixCall; + public readonly track: LocalCallTrack; + private _transceiver: RTCRtpTransceiver; + + public constructor(opts: TrackPublicationOpts) { + this.call = opts.call; + this.track = opts.track; + this._transceiver = opts.transceiver; + + this.updateSenderTrack(); + } + + public get logInfo(): string { + return `streamId=${this.streamId}, trackId=${this.trackId}, mid=${this.mid} kind=${this.track.kind}`; + } + + public get metadata(): SDPStreamMetadataTrack { + const track = this.track; + const trackMetadata: SDPStreamMetadataTrack = { + kind: this.track.kind, + }; + + if (track.isVideo) { + trackMetadata.width = track.track.getSettings().width; + trackMetadata.height = track.track.getSettings().height; + } + + return trackMetadata; + } + + public get mid(): string | undefined { + return this.transceiver?.mid ?? undefined; + } + + public get trackId(): string | undefined { + const mid = this.transceiver?.mid; + return mid ? this.call?.getLocalTrackIdByMid(mid) : undefined; + } + + public get streamId(): string | undefined { + const mid = this.transceiver?.mid; + return mid ? this.call?.getLocalStreamIdByMid(mid) : undefined; + } + + public get transceiver(): RTCRtpTransceiver { + return this._transceiver; + } + + public unpublish(): void { + this.call.unpublishTrack(this); + } + + public updateSenderTrack(): void { + const { stream, track } = this.track; + const transceiver = this.transceiver; + const sender = this.transceiver.sender; + const parameters = sender.getParameters(); + + // No need to update the track + if (sender.track === track) return; + + // setStreams() is currently not supported by Firefox but we + // try to use it at least in other browsers (once we switch + // to using mids and throw away streamIds we will be able to + // throw this away) + if (sender.setStreams && stream) sender.setStreams(stream); + + try { + sender.replaceTrack(track); + + // Does this even work, where does it work? + transceiver.sender.setParameters({ + ...parameters, + encodings: this.track.encodings, + }); + + // Set the direction of the transceiver to indicate we're + // going to be sending. This may trigger re-negotiation, if + // we weren't sending until now + transceiver.direction = transceiver.direction === "inactive" ? "sendonly" : "sendrecv"; + } catch (error) { + logger.warn( + `TrackPublication ${this.trackId} updateSenderTrack() failed to replace track - publishing on new transceiver:`, + error, + ); + + this.call.unpublishTrackOnTransceiver(this); + this._transceiver = this.call.publishTrackOnNewTransceiver(this.track); + } + } +} From 363b910dab48b6a83da40e8636d00f8922c3a98b Mon Sep 17 00:00:00 2001 From: Enrico Schwendig Date: Mon, 20 Mar 2023 09:55:03 +0100 Subject: [PATCH 136/137] Add WebRTC media stats collector (#3205) * webrtc-stats: Add stats collector * webrtc-stats: Switch from ssrc to track based stats Co-authored-by: David Baker * webrtc-stats: add ssrc map for firefox * webrtc-stats: do not start stats multiple times * webrtc-stats: build transport and conn reporter * webrtc-stats: add track handler with tests * webrtc-stats: separate media handler from stats * webrtc-stats: build track stats handler * webrtc-stats: calculate track stats * webrtc-stats: add separate stats report builder --------- Co-authored-by: David Baker --- .gitignore | 1 + spec/test-utils/webrtc.ts | 123 ++++++++++++ .../stats/connectionStatsReporter.spec.ts | 46 +++++ spec/unit/webrtc/stats/groupCallStats.spec.ts | 135 +++++++++++++ .../stats/media/mediaSsrcHandler.spec.ts | 41 ++++ .../stats/media/mediaTrackHandler.spec.ts | 113 +++++++++++ .../media/mediaTrackStatsHandler.spec.ts | 83 ++++++++ spec/unit/webrtc/stats/statsCollector.spec.ts | 58 ++++++ .../webrtc/stats/statsReportBuilder.spec.ts | 115 +++++++++++ .../webrtc/stats/statsReportEmitter.spec.ts | 48 +++++ .../webrtc/stats/statsValueFormatter.spec.ts | 28 +++ .../webrtc/stats/trackStatsReporter.spec.ts | 132 +++++++++++++ .../stats/transportStatsReporter.spec.ts | 126 ++++++++++++ src/webrtc/call.ts | 7 + src/webrtc/groupCall.ts | 18 ++ src/webrtc/stats/connectionStats.ts | 47 +++++ src/webrtc/stats/connectionStatsReporter.ts | 28 +++ src/webrtc/stats/groupCallStats.ts | 64 +++++++ src/webrtc/stats/media/mediaSsrcHandler.ts | 57 ++++++ src/webrtc/stats/media/mediaTrackHandler.ts | 71 +++++++ src/webrtc/stats/media/mediaTrackStats.ts | 104 ++++++++++ .../stats/media/mediaTrackStatsHandler.ts | 86 +++++++++ src/webrtc/stats/statsCollector.ts | 181 ++++++++++++++++++ src/webrtc/stats/statsReport.ts | 56 ++++++ src/webrtc/stats/statsReportBuilder.ts | 110 +++++++++++ src/webrtc/stats/statsReportEmitter.ts | 33 ++++ src/webrtc/stats/statsValueFormatter.ts | 30 +++ src/webrtc/stats/trackStatsReporter.ts | 117 +++++++++++ src/webrtc/stats/transportStats.ts | 26 +++ src/webrtc/stats/transportStatsReporter.ts | 48 +++++ 30 files changed, 2132 insertions(+) create mode 100644 spec/unit/webrtc/stats/connectionStatsReporter.spec.ts create mode 100644 spec/unit/webrtc/stats/groupCallStats.spec.ts create mode 100644 spec/unit/webrtc/stats/media/mediaSsrcHandler.spec.ts create mode 100644 spec/unit/webrtc/stats/media/mediaTrackHandler.spec.ts create mode 100644 spec/unit/webrtc/stats/media/mediaTrackStatsHandler.spec.ts create mode 100644 spec/unit/webrtc/stats/statsCollector.spec.ts create mode 100644 spec/unit/webrtc/stats/statsReportBuilder.spec.ts create mode 100644 spec/unit/webrtc/stats/statsReportEmitter.spec.ts create mode 100644 spec/unit/webrtc/stats/statsValueFormatter.spec.ts create mode 100644 spec/unit/webrtc/stats/trackStatsReporter.spec.ts create mode 100644 spec/unit/webrtc/stats/transportStatsReporter.spec.ts create mode 100644 src/webrtc/stats/connectionStats.ts create mode 100644 src/webrtc/stats/connectionStatsReporter.ts create mode 100644 src/webrtc/stats/groupCallStats.ts create mode 100644 src/webrtc/stats/media/mediaSsrcHandler.ts create mode 100644 src/webrtc/stats/media/mediaTrackHandler.ts create mode 100644 src/webrtc/stats/media/mediaTrackStats.ts create mode 100644 src/webrtc/stats/media/mediaTrackStatsHandler.ts create mode 100644 src/webrtc/stats/statsCollector.ts create mode 100644 src/webrtc/stats/statsReport.ts create mode 100644 src/webrtc/stats/statsReportBuilder.ts create mode 100644 src/webrtc/stats/statsReportEmitter.ts create mode 100644 src/webrtc/stats/statsValueFormatter.ts create mode 100644 src/webrtc/stats/trackStatsReporter.ts create mode 100644 src/webrtc/stats/transportStats.ts create mode 100644 src/webrtc/stats/transportStatsReporter.ts diff --git a/.gitignore b/.gitignore index 6bc9edb40a3..ccc8e35a6d8 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ out .vscode .vscode/ +.idea diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index 48ceef555ec..7121e2f3b12 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -614,6 +614,8 @@ export class MockMatrixCall extends TypedEventEmitter { + describe("should on bandwidth stats", () => { + it("build bandwidth report if chromium starts attributes available", () => { + const stats = { + availableIncomingBitrate: 1000, + availableOutgoingBitrate: 2000, + } as RTCIceCandidatePairStats; + expect(ConnectionStatsReporter.buildBandwidthReport(stats)).toEqual({ download: 1, upload: 2 }); + }); + it("build empty bandwidth report if chromium starts attributes not available", () => { + const stats = {} as RTCIceCandidatePairStats; + expect(ConnectionStatsReporter.buildBandwidthReport(stats)).toEqual({ download: 0, upload: 0 }); + }); + }); + + describe("should on connection stats", () => { + it("build bandwidth report if chromium starts attributes available", () => { + const stats = { + availableIncomingBitrate: 1000, + availableOutgoingBitrate: 2000, + } as RTCIceCandidatePairStats; + expect(ConnectionStatsReporter.buildBandwidthReport(stats)).toEqual({ download: 1, upload: 2 }); + }); + it("build empty bandwidth report if chromium starts attributes not available", () => { + const stats = {} as RTCIceCandidatePairStats; + expect(ConnectionStatsReporter.buildBandwidthReport(stats)).toEqual({ download: 0, upload: 0 }); + }); + }); +}); diff --git a/spec/unit/webrtc/stats/groupCallStats.spec.ts b/spec/unit/webrtc/stats/groupCallStats.spec.ts new file mode 100644 index 00000000000..14a6d806622 --- /dev/null +++ b/spec/unit/webrtc/stats/groupCallStats.spec.ts @@ -0,0 +1,135 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import { GroupCallStats } from "../../../../src/webrtc/stats/groupCallStats"; + +const GROUP_CALL_ID = "GROUP_ID"; +const LOCAL_USER_ID = "LOCAL_USER_ID"; +const TIME_INTERVAL = 10000; + +describe("GroupCallStats", () => { + let stats: GroupCallStats; + beforeEach(() => { + stats = new GroupCallStats(GROUP_CALL_ID, LOCAL_USER_ID, TIME_INTERVAL); + }); + + describe("should on adding a stats collector", () => { + it("creating a new one if not existing.", async () => { + expect(stats.addStatsCollector("CALL_ID", "USER_ID", mockRTCPeerConnection())).toBeTruthy(); + }); + + it("creating only one when trying add the same collector multiple times.", async () => { + expect(stats.addStatsCollector("CALL_ID", "USER_ID", mockRTCPeerConnection())).toBeTruthy(); + expect(stats.addStatsCollector("CALL_ID", "USER_ID", mockRTCPeerConnection())).toBeFalsy(); + // The User ID is not relevant! Because for stats the call is needed and the user id is for monitoring + expect(stats.addStatsCollector("CALL_ID", "SOME_OTHER_USER_ID", mockRTCPeerConnection())).toBeFalsy(); + }); + }); + + describe("should on removing a stats collector", () => { + it("returning `true` if the collector exists", async () => { + expect(stats.addStatsCollector("CALL_ID", "USER_ID", mockRTCPeerConnection())).toBeTruthy(); + expect(stats.removeStatsCollector("CALL_ID")).toBeTruthy(); + }); + it("returning false if the collector not exists", async () => { + expect(stats.removeStatsCollector("CALL_ID_NOT_EXIST")).toBeFalsy(); + }); + }); + + describe("should on get stats collector", () => { + it("returning `undefined` if collector not existing", async () => { + expect(stats.getStatsCollector("CALL_ID")).toBeUndefined(); + }); + + it("returning Collector if collector existing", async () => { + expect(stats.addStatsCollector("CALL_ID", "USER_ID", mockRTCPeerConnection())).toBeTruthy(); + expect(stats.getStatsCollector("CALL_ID")).toBeDefined(); + }); + }); + + describe("should on start", () => { + beforeEach(() => { + jest.useFakeTimers(); + }); + afterEach(() => { + jest.useRealTimers(); + }); + + it("starting processing as well without stats collectors", async () => { + // @ts-ignore + stats.processStats = jest.fn(); + stats.start(); + jest.advanceTimersByTime(TIME_INTERVAL); + // @ts-ignore + expect(stats.processStats).toHaveBeenCalled(); + }); + + it("starting processing and calling the collectors", async () => { + stats.addStatsCollector("CALL_ID", "USER_ID", mockRTCPeerConnection()); + const collector = stats.getStatsCollector("CALL_ID"); + if (collector) { + const processStatsSpy = jest.spyOn(collector, "processStats"); + stats.start(); + jest.advanceTimersByTime(TIME_INTERVAL); + expect(processStatsSpy).toHaveBeenCalledWith(GROUP_CALL_ID, LOCAL_USER_ID); + } else { + throw new Error("Test failed, because no Collector found!"); + } + }); + + it("doing nothing if process already running", async () => { + // @ts-ignore + jest.spyOn(global, "setInterval").mockReturnValue(22); + stats.start(); + expect(setInterval).toHaveBeenCalledTimes(1); + stats.start(); + stats.start(); + stats.start(); + stats.start(); + expect(setInterval).toHaveBeenCalledTimes(1); + }); + }); + + describe("should on stop", () => { + beforeEach(() => { + jest.useFakeTimers(); + }); + afterEach(() => { + jest.useRealTimers(); + }); + it("finish stats process if was started", async () => { + // @ts-ignore + jest.spyOn(global, "setInterval").mockReturnValue(22); + jest.spyOn(global, "clearInterval"); + stats.start(); + expect(setInterval).toHaveBeenCalledTimes(1); + stats.stop(); + expect(clearInterval).toHaveBeenCalledWith(22); + }); + + it("do nothing if stats process was not started", async () => { + jest.spyOn(global, "clearInterval"); + stats.stop(); + expect(clearInterval).not.toHaveBeenCalled(); + }); + }); +}); + +const mockRTCPeerConnection = (): RTCPeerConnection => { + const pc = {} as RTCPeerConnection; + pc.addEventListener = jest.fn(); + pc.getStats = jest.fn().mockResolvedValue(null); + return pc; +}; diff --git a/spec/unit/webrtc/stats/media/mediaSsrcHandler.spec.ts b/spec/unit/webrtc/stats/media/mediaSsrcHandler.spec.ts new file mode 100644 index 00000000000..4b6e93179a4 --- /dev/null +++ b/spec/unit/webrtc/stats/media/mediaSsrcHandler.spec.ts @@ -0,0 +1,41 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import { Mid, Ssrc, MediaSsrcHandler } from "../../../../../src/webrtc/stats/media/mediaSsrcHandler"; +import { REMOTE_SFU_DESCRIPTION } from "../../../../test-utils/webrtc"; + +describe("MediaSsrcHandler", () => { + const remoteMap = new Map([ + ["0", ["2963372119"]], + ["1", ["1212931603"]], + ]); + let handler: MediaSsrcHandler; + beforeEach(() => { + handler = new MediaSsrcHandler(); + }); + describe("should parse description", () => { + it("and build mid ssrc map", () => { + handler.parse(REMOTE_SFU_DESCRIPTION, "remote"); + expect(handler.getSsrcToMidMap("remote")).toEqual(remoteMap); + }); + }); + + describe("should on find mid by ssrc", () => { + it("and return mid if mapping exists.", () => { + handler.parse(REMOTE_SFU_DESCRIPTION, "remote"); + expect(handler.findMidBySsrc("2963372119", "remote")).toEqual("0"); + }); + }); +}); diff --git a/spec/unit/webrtc/stats/media/mediaTrackHandler.spec.ts b/spec/unit/webrtc/stats/media/mediaTrackHandler.spec.ts new file mode 100644 index 00000000000..66dcc5ebf03 --- /dev/null +++ b/spec/unit/webrtc/stats/media/mediaTrackHandler.spec.ts @@ -0,0 +1,113 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import { MediaTrackHandler } from "../../../../../src/webrtc/stats/media/mediaTrackHandler"; + +describe("TrackHandler", () => { + let pc: RTCPeerConnection; + let handler: MediaTrackHandler; + beforeEach(() => { + pc = { + getTransceivers: (): RTCRtpTransceiver[] => [mockTransceiver("1", "audio"), mockTransceiver("2", "video")], + } as RTCPeerConnection; + handler = new MediaTrackHandler(pc); + }); + describe("should get local tracks", () => { + it("returns video track", () => { + expect(handler.getLocalTracks("video")).toEqual([ + { + id: `sender-track-2`, + kind: "video", + } as MediaStreamTrack, + ]); + }); + + it("returns audio track", () => { + expect(handler.getLocalTracks("audio")).toEqual([ + { + id: `sender-track-1`, + kind: "audio", + } as MediaStreamTrack, + ]); + }); + }); + + describe("should get local track by mid", () => { + it("returns video track", () => { + expect(handler.getLocalTrackIdByMid("2")).toEqual("sender-track-2"); + }); + + it("returns audio track", () => { + expect(handler.getLocalTrackIdByMid("1")).toEqual("sender-track-1"); + }); + + it("returns undefined if not exists", () => { + expect(handler.getLocalTrackIdByMid("3")).toBeUndefined(); + }); + }); + + describe("should get remote track by mid", () => { + it("returns video track", () => { + expect(handler.getRemoteTrackIdByMid("2")).toEqual("receiver-track-2"); + }); + + it("returns audio track", () => { + expect(handler.getRemoteTrackIdByMid("1")).toEqual("receiver-track-1"); + }); + + it("returns undefined if not exists", () => { + expect(handler.getRemoteTrackIdByMid("3")).toBeUndefined(); + }); + }); + + describe("should get track by id", () => { + it("returns remote track", () => { + expect(handler.getTackById("receiver-track-2")).toEqual({ + id: `receiver-track-2`, + kind: "video", + } as MediaStreamTrack); + }); + + it("returns local track", () => { + expect(handler.getTackById("sender-track-1")).toEqual({ + id: `sender-track-1`, + kind: "audio", + } as MediaStreamTrack); + }); + + it("returns undefined if not exists", () => { + expect(handler.getTackById("sender-track-3")).toBeUndefined(); + }); + }); + + describe("should get simulcast track count", () => { + it("returns 2", () => { + expect(handler.getActiveSimulcastStreams()).toEqual(3); + }); + }); +}); + +const mockTransceiver = (mid: string, kind: "video" | "audio"): RTCRtpTransceiver => { + return { + mid, + currentDirection: "sendrecv", + sender: { + track: { id: `sender-track-${mid}`, kind } as MediaStreamTrack, + } as RTCRtpSender, + receiver: { + track: { id: `receiver-track-${mid}`, kind } as MediaStreamTrack, + } as RTCRtpReceiver, + } as RTCRtpTransceiver; +}; diff --git a/spec/unit/webrtc/stats/media/mediaTrackStatsHandler.spec.ts b/spec/unit/webrtc/stats/media/mediaTrackStatsHandler.spec.ts new file mode 100644 index 00000000000..d263786fda2 --- /dev/null +++ b/spec/unit/webrtc/stats/media/mediaTrackStatsHandler.spec.ts @@ -0,0 +1,83 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import { MediaTrackHandler } from "../../../../../src/webrtc/stats/media/mediaTrackHandler"; +import { MediaTrackStatsHandler } from "../../../../../src/webrtc/stats/media/mediaTrackStatsHandler"; +import { MediaSsrcHandler } from "../../../../../src/webrtc/stats/media/mediaSsrcHandler"; + +describe("MediaTrackStatsHandler", () => { + let statsHandler: MediaTrackStatsHandler; + let ssrcHandler: MediaSsrcHandler; + let trackHandler: MediaTrackHandler; + beforeEach(() => { + ssrcHandler = {} as MediaSsrcHandler; + trackHandler = {} as MediaTrackHandler; + trackHandler.getLocalTrackIdByMid = jest.fn().mockReturnValue("2222"); + trackHandler.getRemoteTrackIdByMid = jest.fn().mockReturnValue("5555"); + trackHandler.getLocalTracks = jest.fn().mockReturnValue([{ id: "2222" } as MediaStreamTrack]); + trackHandler.getTackById = jest.fn().mockReturnValue([{ id: "2222", kind: "audio" } as MediaStreamTrack]); + statsHandler = new MediaTrackStatsHandler(ssrcHandler, trackHandler); + }); + describe("should find track stats", () => { + it("and returns stats if `trackIdentifier` exists in report", () => { + const report = { trackIdentifier: "123" }; + expect(statsHandler.findTrack2Stats(report, "remote")?.trackId).toEqual("123"); + }); + it("and returns stats if `mid` exists in report", () => { + const reportIn = { mid: "1", type: "inbound-rtp" }; + expect(statsHandler.findTrack2Stats(reportIn, "remote")?.trackId).toEqual("5555"); + const reportOut = { mid: "1", type: "outbound-rtp" }; + expect(statsHandler.findTrack2Stats(reportOut, "local")?.trackId).toEqual("2222"); + }); + it("and returns undefined if `ssrc` exists in report but not on connection", () => { + const report = { ssrc: "142443", type: "inbound-rtp" }; + ssrcHandler.findMidBySsrc = jest.fn().mockReturnValue(undefined); + expect(statsHandler.findTrack2Stats(report, "local")?.trackId).toBeUndefined(); + }); + it("and returns undefined if `ssrc` exists in inbound-rtp report", () => { + const report = { ssrc: "142443", type: "inbound-rtp" }; + ssrcHandler.findMidBySsrc = jest.fn().mockReturnValue("2"); + expect(statsHandler.findTrack2Stats(report, "remote")?.trackId).toEqual("5555"); + }); + it("and returns undefined if `ssrc` exists in outbound-rtp report", () => { + const report = { ssrc: "142443", type: "outbound-rtp" }; + ssrcHandler.findMidBySsrc = jest.fn().mockReturnValue("2"); + expect(statsHandler.findTrack2Stats(report, "local")?.trackId).toEqual("2222"); + }); + it("and returns undefined if needed property not existing", () => { + const report = {}; + expect(statsHandler.findTrack2Stats(report, "remote")?.trackId).toBeUndefined(); + }); + }); + describe("should find local video track stats", () => { + it("and returns stats if `trackIdentifier` exists in report", () => { + const report = { trackIdentifier: "2222" }; + expect(statsHandler.findLocalVideoTrackStats(report)?.trackId).toEqual("2222"); + }); + it("and returns stats if `mid` exists in report", () => { + const report = { mid: "1" }; + expect(statsHandler.findLocalVideoTrackStats(report)?.trackId).toEqual("2222"); + }); + it("and returns undefined if `ssrc` exists", () => { + const report = { ssrc: "142443", type: "outbound-rtp" }; + ssrcHandler.findMidBySsrc = jest.fn().mockReturnValue("2"); + expect(statsHandler.findTrack2Stats(report, "local")?.trackId).toEqual("2222"); + }); + it("and returns undefined if needed property not existing", () => { + const report = {}; + expect(statsHandler.findTrack2Stats(report, "remote")?.trackId).toBeUndefined(); + }); + }); +}); diff --git a/spec/unit/webrtc/stats/statsCollector.spec.ts b/spec/unit/webrtc/stats/statsCollector.spec.ts new file mode 100644 index 00000000000..2397f092183 --- /dev/null +++ b/spec/unit/webrtc/stats/statsCollector.spec.ts @@ -0,0 +1,58 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { StatsCollector } from "../../../../src/webrtc/stats/statsCollector"; +import { StatsReportEmitter } from "../../../../src/webrtc/stats/statsReportEmitter"; + +const CALL_ID = "CALL_ID"; +const USER_ID = "USER_ID"; + +describe("StatsCollector", () => { + let collector: StatsCollector; + let rtcSpy: RTCPeerConnection; + let emitter: StatsReportEmitter; + beforeEach(() => { + rtcSpy = { getStats: () => new Promise(() => null) } as RTCPeerConnection; + rtcSpy.addEventListener = jest.fn(); + emitter = new StatsReportEmitter(); + collector = new StatsCollector(CALL_ID, USER_ID, rtcSpy, emitter); + }); + + describe("on process stats", () => { + it("if active calculate stats reports", async () => { + const getStats = jest.spyOn(rtcSpy, "getStats"); + getStats.mockResolvedValue({} as RTCStatsReport); + await collector.processStats("GROUP_CALL_ID", "LOCAL_USER_ID"); + expect(getStats).toHaveBeenCalled(); + }); + + it("if not active do not calculate stats reports", async () => { + collector.setActive(false); + const getStats = jest.spyOn(rtcSpy, "getStats"); + await collector.processStats("GROUP_CALL_ID", "LOCAL_USER_ID"); + expect(getStats).not.toHaveBeenCalled(); + }); + + it("if get reports fails, the collector becomes inactive", async () => { + expect(collector.getActive()).toBeTruthy(); + const getStats = jest.spyOn(rtcSpy, "getStats"); + getStats.mockRejectedValue(new Error("unknown")); + await collector.processStats("GROUP_CALL_ID", "LOCAL_USER_ID"); + expect(getStats).toHaveBeenCalled(); + expect(collector.getActive()).toBeFalsy(); + }); + }); +}); diff --git a/spec/unit/webrtc/stats/statsReportBuilder.spec.ts b/spec/unit/webrtc/stats/statsReportBuilder.spec.ts new file mode 100644 index 00000000000..b1843bbfc9a --- /dev/null +++ b/spec/unit/webrtc/stats/statsReportBuilder.spec.ts @@ -0,0 +1,115 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { TrackID } from "../../../../src/webrtc/stats/statsReport"; +import { MediaTrackStats } from "../../../../src/webrtc/stats/media/mediaTrackStats"; +import { StatsReportBuilder } from "../../../../src/webrtc/stats/statsReportBuilder"; + +describe("StatsReportBuilder", () => { + const LOCAL_VIDEO_TRACK_ID = "LOCAL_VIDEO_TRACK_ID"; + const LOCAL_AUDIO_TRACK_ID = "LOCAL_AUDIO_TRACK_ID"; + const REMOTE_AUDIO_TRACK_ID = "REMOTE_AUDIO_TRACK_ID"; + const REMOTE_VIDEO_TRACK_ID = "REMOTE_VIDEO_TRACK_ID"; + const localAudioTrack = new MediaTrackStats(LOCAL_AUDIO_TRACK_ID, "local", "audio"); + const localVideoTrack = new MediaTrackStats(LOCAL_VIDEO_TRACK_ID, "local", "video"); + const remoteAudioTrack = new MediaTrackStats(REMOTE_AUDIO_TRACK_ID, "remote", "audio"); + const remoteVideoTrack = new MediaTrackStats(REMOTE_VIDEO_TRACK_ID, "remote", "video"); + const stats = new Map([ + [LOCAL_AUDIO_TRACK_ID, localAudioTrack], + [LOCAL_VIDEO_TRACK_ID, localVideoTrack], + [REMOTE_AUDIO_TRACK_ID, remoteAudioTrack], + [REMOTE_VIDEO_TRACK_ID, remoteVideoTrack], + ]); + beforeEach(() => { + buildData(); + }); + + describe("should build stats", () => { + it("by media track stats.", async () => { + expect(StatsReportBuilder.build(stats)).toEqual({ + bitrate: { + audio: { + download: 4000, + upload: 5000, + }, + download: 5004000, + upload: 3005000, + video: { + download: 5000000, + upload: 3000000, + }, + }, + codec: { + local: new Map([ + ["LOCAL_AUDIO_TRACK_ID", "opus"], + ["LOCAL_VIDEO_TRACK_ID", "v8"], + ]), + remote: new Map([ + ["REMOTE_AUDIO_TRACK_ID", "opus"], + ["REMOTE_VIDEO_TRACK_ID", "v9"], + ]), + }, + framerate: { + local: new Map([ + ["LOCAL_AUDIO_TRACK_ID", 0], + ["LOCAL_VIDEO_TRACK_ID", 30], + ]), + remote: new Map([ + ["REMOTE_AUDIO_TRACK_ID", 0], + ["REMOTE_VIDEO_TRACK_ID", 60], + ]), + }, + packetLoss: { + download: 7, + total: 15, + upload: 28, + }, + resolution: { + local: new Map([ + ["LOCAL_AUDIO_TRACK_ID", { height: -1, width: -1 }], + ["LOCAL_VIDEO_TRACK_ID", { height: 460, width: 780 }], + ]), + remote: new Map([ + ["REMOTE_AUDIO_TRACK_ID", { height: -1, width: -1 }], + ["REMOTE_VIDEO_TRACK_ID", { height: 960, width: 1080 }], + ]), + }, + }); + }); + }); + + const buildData = (): void => { + localAudioTrack.setCodec("opus"); + localAudioTrack.setLoss({ packetsTotal: 10, packetsLost: 5, isDownloadStream: false }); + localAudioTrack.setBitrate({ download: 0, upload: 5000 }); + + remoteAudioTrack.setCodec("opus"); + remoteAudioTrack.setLoss({ packetsTotal: 20, packetsLost: 0, isDownloadStream: true }); + remoteAudioTrack.setBitrate({ download: 4000, upload: 0 }); + + localVideoTrack.setCodec("v8"); + localVideoTrack.setLoss({ packetsTotal: 30, packetsLost: 6, isDownloadStream: false }); + localVideoTrack.setBitrate({ download: 0, upload: 3000000 }); + localVideoTrack.setFramerate(30); + localVideoTrack.setResolution({ width: 780, height: 460 }); + + remoteVideoTrack.setCodec("v9"); + remoteVideoTrack.setLoss({ packetsTotal: 40, packetsLost: 4, isDownloadStream: true }); + remoteVideoTrack.setBitrate({ download: 5000000, upload: 0 }); + remoteVideoTrack.setFramerate(60); + remoteVideoTrack.setResolution({ width: 1080, height: 960 }); + }; +}); diff --git a/spec/unit/webrtc/stats/statsReportEmitter.spec.ts b/spec/unit/webrtc/stats/statsReportEmitter.spec.ts new file mode 100644 index 00000000000..715ea1bef4b --- /dev/null +++ b/spec/unit/webrtc/stats/statsReportEmitter.spec.ts @@ -0,0 +1,48 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import { StatsReportEmitter } from "../../../../src/webrtc/stats/statsReportEmitter"; +import { ByteSendStatsReport, ConnectionStatsReport, StatsReport } from "../../../../src/webrtc/stats/statsReport"; + +describe("StatsReportEmitter", () => { + let emitter: StatsReportEmitter; + beforeEach(() => { + emitter = new StatsReportEmitter(); + }); + + it("should emit and receive ByteSendStatsReport", async () => { + const report = {} as ByteSendStatsReport; + return new Promise((resolve, _) => { + emitter.on(StatsReport.BYTE_SENT_STATS, (r) => { + expect(r).toBe(report); + resolve(null); + return; + }); + emitter.emitByteSendReport(report); + }); + }); + + it("should emit and receive ConnectionStatsReport", async () => { + const report = {} as ConnectionStatsReport; + return new Promise((resolve, _) => { + emitter.on(StatsReport.CONNECTION_STATS, (r) => { + expect(r).toBe(report); + resolve(null); + return; + }); + emitter.emitConnectionStatsReport(report); + }); + }); +}); diff --git a/spec/unit/webrtc/stats/statsValueFormatter.spec.ts b/spec/unit/webrtc/stats/statsValueFormatter.spec.ts new file mode 100644 index 00000000000..1ce563e91d6 --- /dev/null +++ b/spec/unit/webrtc/stats/statsValueFormatter.spec.ts @@ -0,0 +1,28 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import { StatsValueFormatter } from "../../../../src/webrtc/stats/statsValueFormatter"; + +describe("StatsValueFormatter", () => { + describe("on get non negative values", () => { + it("formatter shod return number", async () => { + expect(StatsValueFormatter.getNonNegativeValue("2")).toEqual(2); + expect(StatsValueFormatter.getNonNegativeValue(0)).toEqual(0); + expect(StatsValueFormatter.getNonNegativeValue("-2")).toEqual(0); + expect(StatsValueFormatter.getNonNegativeValue("")).toEqual(0); + expect(StatsValueFormatter.getNonNegativeValue(NaN)).toEqual(0); + }); + }); +}); diff --git a/spec/unit/webrtc/stats/trackStatsReporter.spec.ts b/spec/unit/webrtc/stats/trackStatsReporter.spec.ts new file mode 100644 index 00000000000..6a1bb5bf21a --- /dev/null +++ b/spec/unit/webrtc/stats/trackStatsReporter.spec.ts @@ -0,0 +1,132 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import { TrackStatsReporter } from "../../../../src/webrtc/stats/trackStatsReporter"; +import { MediaTrackStats } from "../../../../src/webrtc/stats/media/mediaTrackStats"; + +describe("TrackStatsReporter", () => { + describe("should on frame and resolution stats", () => { + it("creating empty frame and resolution report, if no data available.", async () => { + const trackStats = new MediaTrackStats("1", "local", "video"); + TrackStatsReporter.buildFramerateResolution(trackStats, {}); + expect(trackStats.getFramerate()).toEqual(0); + expect(trackStats.getResolution()).toEqual({ width: -1, height: -1 }); + }); + it("creating empty frame and resolution report.", async () => { + const trackStats = new MediaTrackStats("1", "remote", "video"); + TrackStatsReporter.buildFramerateResolution(trackStats, { + framesPerSecond: 22.2, + frameHeight: 180, + frameWidth: 360, + }); + expect(trackStats.getFramerate()).toEqual(22); + expect(trackStats.getResolution()).toEqual({ width: 360, height: 180 }); + }); + }); + + describe("should on simulcast", () => { + it("creating simulcast framerate.", async () => { + const trackStats = new MediaTrackStats("1", "local", "video"); + TrackStatsReporter.calculateSimulcastFramerate( + trackStats, + { + framesSent: 100, + timestamp: 1678957001000, + }, + { + framesSent: 10, + timestamp: 1678957000000, + }, + 3, + ); + expect(trackStats.getFramerate()).toEqual(30); + }); + }); + + describe("should on bytes received stats", () => { + it("creating build bitrate received report.", async () => { + const trackStats = new MediaTrackStats("1", "remote", "video"); + TrackStatsReporter.buildBitrateReceived( + trackStats, + { + bytesReceived: 2001000, + timestamp: 1678957010, + }, + { bytesReceived: 2000000, timestamp: 1678957000 }, + ); + expect(trackStats.getBitrate()).toEqual({ download: 800, upload: 0 }); + }); + }); + + describe("should on bytes send stats", () => { + it("creating build bitrate send report.", async () => { + const trackStats = new MediaTrackStats("1", "local", "video"); + TrackStatsReporter.buildBitrateSend( + trackStats, + { + bytesSent: 2001000, + timestamp: 1678957010, + }, + { bytesSent: 2000000, timestamp: 1678957000 }, + ); + expect(trackStats.getBitrate()).toEqual({ download: 0, upload: 800 }); + }); + }); + + describe("should on codec stats", () => { + it("creating build bitrate send report.", async () => { + const trackStats = new MediaTrackStats("1", "remote", "video"); + const remote = {} as RTCStatsReport; + remote.get = jest.fn().mockReturnValue({ mimeType: "video/v8" }); + TrackStatsReporter.buildCodec(remote, trackStats, { codecId: "codecID" }); + expect(trackStats.getCodec()).toEqual("v8"); + }); + }); + + describe("should on package lost stats", () => { + it("creating build package lost on send report.", async () => { + const trackStats = new MediaTrackStats("1", "local", "video"); + TrackStatsReporter.buildPacketsLost( + trackStats, + { + type: "outbound-rtp", + packetsSent: 200, + packetsLost: 120, + }, + { + packetsSent: 100, + packetsLost: 30, + }, + ); + expect(trackStats.getLoss()).toEqual({ packetsTotal: 190, packetsLost: 90, isDownloadStream: false }); + }); + it("creating build package lost on received report.", async () => { + const trackStats = new MediaTrackStats("1", "remote", "video"); + TrackStatsReporter.buildPacketsLost( + trackStats, + { + type: "inbound-rtp", + packetsReceived: 300, + packetsLost: 100, + }, + { + packetsReceived: 100, + packetsLost: 20, + }, + ); + expect(trackStats.getLoss()).toEqual({ packetsTotal: 280, packetsLost: 80, isDownloadStream: true }); + }); + }); +}); diff --git a/spec/unit/webrtc/stats/transportStatsReporter.spec.ts b/spec/unit/webrtc/stats/transportStatsReporter.spec.ts new file mode 100644 index 00000000000..bd3288b15ae --- /dev/null +++ b/spec/unit/webrtc/stats/transportStatsReporter.spec.ts @@ -0,0 +1,126 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { TransportStatsReporter } from "../../../../src/webrtc/stats/transportStatsReporter"; +import { TransportStats } from "../../../../src/webrtc/stats/transportStats"; + +describe("TransportStatsReporter", () => { + describe("should on build report", () => { + const REMOTE_CANDIDATE_ID = "REMOTE_CANDIDATE_ID"; + const LOCAL_CANDIDATE_ID = "LOCAL_CANDIDATE_ID"; + const localIC = { ip: "88.88.99.1", port: 56670, protocol: "tcp", candidateType: "local", networkType: "lan" }; + const remoteIC = { + ip: "123.88.99.1", + port: 46670, + protocol: "udp", + candidateType: "srfx", + networkType: "wifi", + }; + const isFocus = false; + const rtt = 200000; + + it("build new transport stats if all properties there", () => { + const { report, stats } = mockStatsReport(isFocus, 0); + const conferenceStatsTransport: TransportStats[] = []; + const transportStats = TransportStatsReporter.buildReport(report, stats, conferenceStatsTransport, isFocus); + expect(transportStats).toEqual([ + { + ip: `${remoteIC.ip + 0}:${remoteIC.port}`, + type: remoteIC.protocol, + localIp: `${localIC.ip + 0}:${localIC.port}`, + isFocus, + localCandidateType: localIC.candidateType, + remoteCandidateType: remoteIC.candidateType, + networkType: localIC.networkType, + rtt, + }, + ]); + }); + + it("build next transport stats if candidates different", () => { + const mock1 = mockStatsReport(isFocus, 0); + const mock2 = mockStatsReport(isFocus, 1); + let transportStats: TransportStats[] = []; + transportStats = TransportStatsReporter.buildReport(mock1.report, mock1.stats, transportStats, isFocus); + transportStats = TransportStatsReporter.buildReport(mock2.report, mock2.stats, transportStats, isFocus); + expect(transportStats).toEqual([ + { + ip: `${remoteIC.ip + 0}:${remoteIC.port}`, + type: remoteIC.protocol, + localIp: `${localIC.ip + 0}:${localIC.port}`, + isFocus, + localCandidateType: localIC.candidateType, + remoteCandidateType: remoteIC.candidateType, + networkType: localIC.networkType, + rtt, + }, + { + ip: `${remoteIC.ip + 1}:${remoteIC.port}`, + type: remoteIC.protocol, + localIp: `${localIC.ip + 1}:${localIC.port}`, + isFocus, + localCandidateType: localIC.candidateType, + remoteCandidateType: remoteIC.candidateType, + networkType: localIC.networkType, + rtt, + }, + ]); + }); + + it("build not a second transport stats if candidates the same", () => { + const mock1 = mockStatsReport(isFocus, 0); + const mock2 = mockStatsReport(isFocus, 0); + let transportStats: TransportStats[] = []; + transportStats = TransportStatsReporter.buildReport(mock1.report, mock1.stats, transportStats, isFocus); + transportStats = TransportStatsReporter.buildReport(mock2.report, mock2.stats, transportStats, isFocus); + expect(transportStats).toEqual([ + { + ip: `${remoteIC.ip + 0}:${remoteIC.port}`, + type: remoteIC.protocol, + localIp: `${localIC.ip + 0}:${localIC.port}`, + isFocus, + localCandidateType: localIC.candidateType, + remoteCandidateType: remoteIC.candidateType, + networkType: localIC.networkType, + rtt, + }, + ]); + }); + + const mockStatsReport = ( + isFocus: boolean, + prifix: number, + ): { report: RTCStatsReport; stats: RTCIceCandidatePairStats } => { + const report = {} as RTCStatsReport; + report.get = (key: string) => { + if (key === LOCAL_CANDIDATE_ID) { + return { ...localIC, ip: localIC.ip + prifix }; + } + if (key === REMOTE_CANDIDATE_ID) { + return { ...remoteIC, ip: remoteIC.ip + prifix }; + } + // remote + return {}; + }; + const stats = { + remoteCandidateId: REMOTE_CANDIDATE_ID, + localCandidateId: LOCAL_CANDIDATE_ID, + currentRoundTripTime: 200, + } as RTCIceCandidatePairStats; + return { report, stats }; + }; + }); +}); diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index e486ffa53ae..519784e77c9 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -64,6 +64,7 @@ import { LocalCallFeed } from "./localCallFeed"; import { LocalCallTrack } from "./localCallTrack"; import { TrackPublication } from "./trackPublication"; import { FeedPublication } from "./feedPublication"; +import { GroupCallStats } from "./stats/groupCallStats"; interface CallOpts { // The room ID for this call. @@ -407,6 +408,7 @@ export class MatrixCall extends TypedEventEmitter; + private stats: GroupCallStats | undefined; /** * Construct a new Matrix Call. @@ -3031,6 +3033,7 @@ export class MatrixCall extends TypedEventEmitter, enabled: boolean): void { diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index d1a8a43e21b..f45cadb354e 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -25,6 +25,8 @@ import { GroupCallEventHandlerEvent } from "./groupCallEventHandler"; import { IScreensharingOpts } from "./mediaHandler"; import { mapsEqual } from "../utils"; import { LocalCallFeed } from "./localCallFeed"; +import { GroupCallStats } from "./stats/groupCallStats"; +import { ByteSendStatsReport, ConnectionStatsReport, StatsReport } from "./stats/statsReport"; export enum GroupCallIntent { Ring = "m.ring", @@ -214,6 +216,8 @@ export class GroupCall extends TypedEventEmitter< private initWithVideoMuted = false; private initCallFeedPromise?: Promise; + private readonly stats: GroupCallStats; + public constructor( private client: MatrixClient, public room: Room, @@ -235,6 +239,18 @@ export class GroupCall extends TypedEventEmitter< this.on(GroupCallEvent.ParticipantsChanged, this.onParticipantsChanged); this.on(GroupCallEvent.GroupCallStateChanged, this.onStateChanged); this.on(GroupCallEvent.LocalScreenshareStateChanged, this.onLocalFeedsChanged); + const userID = this.client.getUserId() || "unknown"; + this.stats = new GroupCallStats(this.groupCallId, userID); + this.stats.reports.on(StatsReport.CONNECTION_STATS, this.onConnectionStats); + this.stats.reports.on(StatsReport.BYTE_SENT_STATS, this.onByteSendStats); + } + + private onConnectionStats(_: ConnectionStatsReport): void { + // @TODO: Implement data argumentation and event broadcasting please + } + + private onByteSendStats(_: ByteSendStatsReport): void { + // @TODO: Implement data argumentation and event broadcasting please } public async create(): Promise { @@ -1078,6 +1094,8 @@ export class GroupCall extends TypedEventEmitter< this.reEmitter.reEmit(call, Object.values(CallEvent)); + call.initStats(this.stats); + onCallFeedsChanged(); } diff --git a/src/webrtc/stats/connectionStats.ts b/src/webrtc/stats/connectionStats.ts new file mode 100644 index 00000000000..dbde6e50327 --- /dev/null +++ b/src/webrtc/stats/connectionStats.ts @@ -0,0 +1,47 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { TransportStats } from "./transportStats"; +import { Bitrate } from "./media/mediaTrackStats"; + +export interface ConnectionStatsBandwidth { + /** + * bytes per second + */ + download: number; + /** + * bytes per second + */ + upload: number; +} + +export interface ConnectionStatsBitrate extends Bitrate { + audio?: Bitrate; + video?: Bitrate; +} + +export interface PacketLoos { + total: number; + download: number; + upload: number; +} + +export class ConnectionStats { + public bandwidth: ConnectionStatsBitrate = {} as ConnectionStatsBitrate; + public bitrate: ConnectionStatsBitrate = {} as ConnectionStatsBitrate; + public packetLoss: PacketLoos = {} as PacketLoos; + public transport: TransportStats[] = []; +} diff --git a/src/webrtc/stats/connectionStatsReporter.ts b/src/webrtc/stats/connectionStatsReporter.ts new file mode 100644 index 00000000000..c43b9b40c19 --- /dev/null +++ b/src/webrtc/stats/connectionStatsReporter.ts @@ -0,0 +1,28 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import { Bitrate } from "./media/mediaTrackStats"; + +export class ConnectionStatsReporter { + public static buildBandwidthReport(now: RTCIceCandidatePairStats): Bitrate { + const availableIncomingBitrate = now.availableIncomingBitrate; + const availableOutgoingBitrate = now.availableOutgoingBitrate; + + return { + download: availableIncomingBitrate ? Math.round(availableIncomingBitrate / 1000) : 0, + upload: availableOutgoingBitrate ? Math.round(availableOutgoingBitrate / 1000) : 0, + }; + } +} diff --git a/src/webrtc/stats/groupCallStats.ts b/src/webrtc/stats/groupCallStats.ts new file mode 100644 index 00000000000..27b89346431 --- /dev/null +++ b/src/webrtc/stats/groupCallStats.ts @@ -0,0 +1,64 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import { StatsCollector } from "./statsCollector"; +import { StatsReportEmitter } from "./statsReportEmitter"; + +export class GroupCallStats { + private timer: undefined | ReturnType; + private readonly collectors: Map = new Map(); + public readonly reports = new StatsReportEmitter(); + + public constructor(private groupCallId: string, private userId: string, private interval: number = 10000) {} + + public start(): void { + if (this.timer === undefined) { + this.timer = setInterval(() => { + this.processStats(); + }, this.interval); + } + } + + public stop(): void { + if (this.timer !== undefined) { + clearInterval(this.timer); + this.collectors.forEach((c) => c.stopProcessingStats()); + } + } + + public hasStatsCollector(callId: string): boolean { + return this.collectors.has(callId); + } + + public addStatsCollector(callId: string, userId: string, peerConnection: RTCPeerConnection): boolean { + if (this.hasStatsCollector(callId)) { + return false; + } + this.collectors.set(callId, new StatsCollector(callId, userId, peerConnection, this.reports)); + return true; + } + + public removeStatsCollector(callId: string): boolean { + return this.collectors.delete(callId); + } + + public getStatsCollector(callId: string): StatsCollector | undefined { + return this.hasStatsCollector(callId) ? this.collectors.get(callId) : undefined; + } + + private processStats(): void { + this.collectors.forEach((c) => c.processStats(this.groupCallId, this.userId)); + } +} diff --git a/src/webrtc/stats/media/mediaSsrcHandler.ts b/src/webrtc/stats/media/mediaSsrcHandler.ts new file mode 100644 index 00000000000..e60605152c9 --- /dev/null +++ b/src/webrtc/stats/media/mediaSsrcHandler.ts @@ -0,0 +1,57 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { parse as parseSdp } from "sdp-transform"; + +export type Mid = string; +export type Ssrc = string; +export type MapType = "local" | "remote"; + +export class MediaSsrcHandler { + private readonly ssrcToMid = { local: new Map(), remote: new Map() }; + + public findMidBySsrc(ssrc: Ssrc, type: "local" | "remote"): Mid | undefined { + let mid: Mid | undefined; + this.ssrcToMid[type].forEach((ssrcs, m) => { + if (ssrcs.find((s) => s == ssrc)) { + mid = m; + return; + } + }); + return mid; + } + + public parse(description: string, type: MapType): void { + const sdp = parseSdp(description); + const ssrcToMid = new Map(); + sdp.media.forEach((m) => { + if ((!!m.mid && m.type === "video") || m.type === "audio") { + const ssrcs: Ssrc[] = []; + m.ssrcs?.forEach((ssrc) => { + if (ssrc.attribute === "cname") { + ssrcs.push(`${ssrc.id}`); + } + }); + ssrcToMid.set(`${m.mid}`, ssrcs); + } + }); + this.ssrcToMid[type] = ssrcToMid; + } + + public getSsrcToMidMap(type: MapType): Map { + return this.ssrcToMid[type]; + } +} diff --git a/src/webrtc/stats/media/mediaTrackHandler.ts b/src/webrtc/stats/media/mediaTrackHandler.ts new file mode 100644 index 00000000000..32580b1228a --- /dev/null +++ b/src/webrtc/stats/media/mediaTrackHandler.ts @@ -0,0 +1,71 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +export type TrackId = string; + +export class MediaTrackHandler { + public constructor(private readonly pc: RTCPeerConnection) {} + + public getLocalTracks(kind: "audio" | "video"): MediaStreamTrack[] { + const isNotNullAndKind = (track: MediaStreamTrack | null): boolean => { + return track !== null && track.kind === kind; + }; + // @ts-ignore The linter don't get it + return this.pc + .getTransceivers() + .filter((t) => t.currentDirection === "sendonly" || t.currentDirection === "sendrecv") + .filter((t) => t.sender !== null) + .map((t) => t.sender) + .map((s) => s.track) + .filter(isNotNullAndKind); + } + + public getTackById(trackId: string): MediaStreamTrack | undefined { + return this.pc + .getTransceivers() + .map((t) => { + if (t?.sender.track !== null && t.sender.track.id === trackId) { + return t.sender.track; + } + if (t?.receiver.track !== null && t.receiver.track.id === trackId) { + return t.receiver.track; + } + return undefined; + }) + .find((t) => t !== undefined); + } + + public getLocalTrackIdByMid(mid: string): string | undefined { + const transceiver = this.pc.getTransceivers().find((t) => t.mid === mid); + if (transceiver !== undefined && !!transceiver.sender && !!transceiver.sender.track) { + return transceiver.sender.track.id; + } + return undefined; + } + + public getRemoteTrackIdByMid(mid: string): string | undefined { + const transceiver = this.pc.getTransceivers().find((t) => t.mid === mid); + if (transceiver !== undefined && !!transceiver.receiver && !!transceiver.receiver.track) { + return transceiver.receiver.track.id; + } + return undefined; + } + + public getActiveSimulcastStreams(): number { + //@TODO implement this right.. Check how many layer configured + return 3; + } +} diff --git a/src/webrtc/stats/media/mediaTrackStats.ts b/src/webrtc/stats/media/mediaTrackStats.ts new file mode 100644 index 00000000000..69ee9bdfadf --- /dev/null +++ b/src/webrtc/stats/media/mediaTrackStats.ts @@ -0,0 +1,104 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { TrackId } from "./mediaTrackHandler"; + +export interface PacketLoss { + packetsTotal: number; + packetsLost: number; + isDownloadStream: boolean; +} + +export interface Bitrate { + /** + * bytes per second + */ + download: number; + /** + * bytes per second + */ + upload: number; +} + +export interface Resolution { + width: number; + height: number; +} + +export type TrackStatsType = "local" | "remote"; + +export class MediaTrackStats { + private loss: PacketLoss = { packetsTotal: 0, packetsLost: 0, isDownloadStream: false }; + private bitrate: Bitrate = { download: 0, upload: 0 }; + private resolution: Resolution = { width: -1, height: -1 }; + private framerate = 0; + private codec = ""; + + public constructor( + public readonly trackId: TrackId, + public readonly type: TrackStatsType, + public readonly kind: "audio" | "video", + ) {} + + public getType(): TrackStatsType { + return this.type; + } + + public setLoss(loos: PacketLoss): void { + this.loss = loos; + } + + public getLoss(): PacketLoss { + return this.loss; + } + + public setResolution(resolution: Resolution): void { + this.resolution = resolution; + } + + public getResolution(): Resolution { + return this.resolution; + } + + public setFramerate(framerate: number): void { + this.framerate = framerate; + } + + public getFramerate(): number { + return this.framerate; + } + + public setBitrate(bitrate: Bitrate): void { + this.bitrate = bitrate; + } + + public getBitrate(): Bitrate { + return this.bitrate; + } + + public setCodec(codecShortType: string): boolean { + this.codec = codecShortType; + return true; + } + + public getCodec(): string { + return this.codec; + } + + public resetBitrate(): void { + this.bitrate = { download: 0, upload: 0 }; + } +} diff --git a/src/webrtc/stats/media/mediaTrackStatsHandler.ts b/src/webrtc/stats/media/mediaTrackStatsHandler.ts new file mode 100644 index 00000000000..6fb119c8a75 --- /dev/null +++ b/src/webrtc/stats/media/mediaTrackStatsHandler.ts @@ -0,0 +1,86 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import { TrackID } from "../statsReport"; +import { MediaTrackStats } from "./mediaTrackStats"; +import { MediaTrackHandler } from "./mediaTrackHandler"; +import { MediaSsrcHandler } from "./mediaSsrcHandler"; + +export class MediaTrackStatsHandler { + private readonly track2stats = new Map(); + + public constructor( + public readonly mediaSsrcHandler: MediaSsrcHandler, + public readonly mediaTrackHandler: MediaTrackHandler, + ) {} + + /** + * Find tracks by rtc stats + * Argument report is any because the stats api is not consistent: + * For example `trackIdentifier`, `mid` not existing in every implementations + * https://www.w3.org/TR/webrtc-stats/#dom-rtcinboundrtpstreamstats + * https://developer.mozilla.org/en-US/docs/Web/API/RTCInboundRtpStreamStats + */ + public findTrack2Stats(report: any, type: "remote" | "local"): MediaTrackStats | undefined { + let trackID; + if (report.trackIdentifier) { + trackID = report.trackIdentifier; + } else if (report.mid) { + trackID = + type === "remote" + ? this.mediaTrackHandler.getRemoteTrackIdByMid(report.mid) + : this.mediaTrackHandler.getLocalTrackIdByMid(report.mid); + } else if (report.ssrc) { + const mid = this.mediaSsrcHandler.findMidBySsrc(report.ssrc, type); + if (!mid) { + return undefined; + } + trackID = + type === "remote" + ? this.mediaTrackHandler.getRemoteTrackIdByMid(report.mid) + : this.mediaTrackHandler.getLocalTrackIdByMid(report.mid); + } + + if (!trackID) { + return undefined; + } + + let trackStats = this.track2stats.get(trackID); + + if (!trackStats) { + const track = this.mediaTrackHandler.getTackById(trackID); + if (track !== undefined) { + const kind: "audio" | "video" = track.kind === "audio" ? track.kind : "video"; + trackStats = new MediaTrackStats(trackID, type, kind); + this.track2stats.set(trackID, trackStats); + } else { + return undefined; + } + } + return trackStats; + } + + public findLocalVideoTrackStats(report: any): MediaTrackStats | undefined { + const localVideoTracks = this.mediaTrackHandler.getLocalTracks("video"); + if (localVideoTracks.length === 0) { + return undefined; + } + return this.findTrack2Stats(report, "local"); + } + + public getTrack2stats(): Map { + return this.track2stats; + } +} diff --git a/src/webrtc/stats/statsCollector.ts b/src/webrtc/stats/statsCollector.ts new file mode 100644 index 00000000000..302004a5bb1 --- /dev/null +++ b/src/webrtc/stats/statsCollector.ts @@ -0,0 +1,181 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { ConnectionStats } from "./connectionStats"; +import { StatsReportEmitter } from "./statsReportEmitter"; +import { ByteSend, ByteSendStatsReport, TrackID } from "./statsReport"; +import { ConnectionStatsReporter } from "./connectionStatsReporter"; +import { TransportStatsReporter } from "./transportStatsReporter"; +import { MediaSsrcHandler } from "./media/mediaSsrcHandler"; +import { MediaTrackHandler } from "./media/mediaTrackHandler"; +import { MediaTrackStatsHandler } from "./media/mediaTrackStatsHandler"; +import { TrackStatsReporter } from "./trackStatsReporter"; +import { StatsReportBuilder } from "./statsReportBuilder"; +import { StatsValueFormatter } from "./statsValueFormatter"; + +export class StatsCollector { + private isActive = true; + private previousStatsReport: RTCStatsReport | undefined; + private currentStatsReport: RTCStatsReport | undefined; + private readonly connectionStats = new ConnectionStats(); + + private readonly trackStats: MediaTrackStatsHandler; + + // private readonly ssrcToMid = { local: new Map(), remote: new Map() }; + + public constructor( + public readonly callId: string, + public readonly remoteUserId: string, + private readonly pc: RTCPeerConnection, + private readonly emitter: StatsReportEmitter, + private readonly isFocus = true, + ) { + pc.addEventListener("signalingstatechange", this.onSignalStateChange.bind(this)); + this.trackStats = new MediaTrackStatsHandler(new MediaSsrcHandler(), new MediaTrackHandler(pc)); + } + + public async processStats(groupCallId: string, localUserId: string): Promise { + if (this.isActive) { + return this.pc + .getStats() + .then((report) => { + // @ts-ignore + this.currentStatsReport = typeof report?.result === "function" ? report.result() : report; + try { + this.processStatsReport(groupCallId, localUserId); + } catch (error) { + this.isActive = false; + return false; + // logger.error('Processing of RTP stats failed:', error); + } + + this.previousStatsReport = this.currentStatsReport; + return true; + }) + .catch((error) => { + this.handleError(error); + return false; + }); + } + return Promise.resolve(false); + } + + private processStatsReport(groupCallId: string, localUserId: string): void { + const byteSentStats: ByteSendStatsReport = new Map(); + + this.currentStatsReport?.forEach((now) => { + const before = this.previousStatsReport ? this.previousStatsReport.get(now.id) : null; + // RTCIceCandidatePairStats - https://w3c.github.io/webrtc-stats/#candidatepair-dict* + if (now.type === "candidate-pair" && now.nominated && now.state === "succeeded") { + this.connectionStats.bandwidth = ConnectionStatsReporter.buildBandwidthReport(now); + this.connectionStats.transport = TransportStatsReporter.buildReport( + this.currentStatsReport, + now, + this.connectionStats.transport, + this.isFocus, + ); + + // RTCReceivedRtpStreamStats + // https://w3c.github.io/webrtc-stats/#receivedrtpstats-dict* + // RTCSentRtpStreamStats + // https://w3c.github.io/webrtc-stats/#sentrtpstats-dict* + } else if (now.type === "inbound-rtp" || now.type === "outbound-rtp") { + const trackStats = this.trackStats.findTrack2Stats( + now, + now.type === "inbound-rtp" ? "remote" : "local", + ); + if (!trackStats) { + return; + } + + if (before) { + TrackStatsReporter.buildPacketsLost(trackStats, now, before); + } + + // Get the resolution and framerate for only remote video sources here. For the local video sources, + // 'track' stats will be used since they have the updated resolution based on the simulcast streams + // currently being sent. Promise based getStats reports three 'outbound-rtp' streams and there will be + // more calculations needed to determine what is the highest resolution stream sent by the client if the + // 'outbound-rtp' stats are used. + if (now.type === "inbound-rtp") { + TrackStatsReporter.buildFramerateResolution(trackStats, now); + if (before) { + TrackStatsReporter.buildBitrateReceived(trackStats, now, before); + } + } else if (before) { + byteSentStats.set(trackStats.trackId, StatsValueFormatter.getNonNegativeValue(now.bytesSent)); + TrackStatsReporter.buildBitrateSend(trackStats, now, before); + } + TrackStatsReporter.buildCodec(this.currentStatsReport, trackStats, now); + } else if (now.type === "track" && now.kind === "video" && !now.remoteSource) { + const trackStats = this.trackStats.findLocalVideoTrackStats(now); + if (!trackStats) { + return; + } + TrackStatsReporter.buildFramerateResolution(trackStats, now); + TrackStatsReporter.calculateSimulcastFramerate( + trackStats, + now, + before, + this.trackStats.mediaTrackHandler.getActiveSimulcastStreams(), + ); + } + }); + + this.emitter.emitByteSendReport(byteSentStats); + this.processAndEmitReport(); + } + + public setActive(isActive: boolean): void { + this.isActive = isActive; + } + + public getActive(): boolean { + return this.isActive; + } + + private handleError(_: any): void { + this.isActive = false; + } + + private processAndEmitReport(): void { + const report = StatsReportBuilder.build(this.trackStats.getTrack2stats()); + + this.connectionStats.bandwidth = report.bandwidth; + this.connectionStats.bitrate = report.bitrate; + this.connectionStats.packetLoss = report.packetLoss; + + this.emitter.emitConnectionStatsReport({ + ...report, + transport: this.connectionStats.transport, + }); + + this.connectionStats.transport = []; + } + + public stopProcessingStats(): void {} + + private onSignalStateChange(): void { + if (this.pc.signalingState === "stable") { + if (this.pc.currentRemoteDescription) { + this.trackStats.mediaSsrcHandler.parse(this.pc.currentRemoteDescription.sdp, "remote"); + } + if (this.pc.currentLocalDescription) { + this.trackStats.mediaSsrcHandler.parse(this.pc.currentLocalDescription.sdp, "local"); + } + } + } +} diff --git a/src/webrtc/stats/statsReport.ts b/src/webrtc/stats/statsReport.ts new file mode 100644 index 00000000000..ab6db696bee --- /dev/null +++ b/src/webrtc/stats/statsReport.ts @@ -0,0 +1,56 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { ConnectionStatsBandwidth, ConnectionStatsBitrate, PacketLoos } from "./connectionStats"; +import { TransportStats } from "./transportStats"; +import { Resolution } from "./media/mediaTrackStats"; + +export enum StatsReport { + CONNECTION_STATS = "connection_stats", + BYTE_SENT_STATS = "byte_sent_stats", +} + +export type TrackID = string; +export type ByteSend = number; + +export interface ByteSendStatsReport extends Map { + // is a map: `local trackID` => byte send +} + +export interface ConnectionStatsReport { + bandwidth: ConnectionStatsBandwidth; + bitrate: ConnectionStatsBitrate; + packetLoss: PacketLoos; + resolution: ResolutionMap; + framerate: FramerateMap; + codec: CodecMap; + transport: TransportStats[]; +} + +export interface ResolutionMap { + local: Map; + remote: Map; +} + +export interface FramerateMap { + local: Map; + remote: Map; +} + +export interface CodecMap { + local: Map; + remote: Map; +} diff --git a/src/webrtc/stats/statsReportBuilder.ts b/src/webrtc/stats/statsReportBuilder.ts new file mode 100644 index 00000000000..c1af471ce30 --- /dev/null +++ b/src/webrtc/stats/statsReportBuilder.ts @@ -0,0 +1,110 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import { CodecMap, ConnectionStatsReport, FramerateMap, ResolutionMap, TrackID } from "./statsReport"; +import { MediaTrackStats, Resolution } from "./media/mediaTrackStats"; + +export class StatsReportBuilder { + public static build(stats: Map): ConnectionStatsReport { + const report = {} as ConnectionStatsReport; + + // process stats + const totalPackets = { + download: 0, + upload: 0, + }; + const lostPackets = { + download: 0, + upload: 0, + }; + let bitrateDownload = 0; + let bitrateUpload = 0; + const resolutions: ResolutionMap = { + local: new Map(), + remote: new Map(), + }; + const framerates: FramerateMap = { local: new Map(), remote: new Map() }; + const codecs: CodecMap = { local: new Map(), remote: new Map() }; + + let audioBitrateDownload = 0; + let audioBitrateUpload = 0; + let videoBitrateDownload = 0; + let videoBitrateUpload = 0; + + for (const [trackId, trackStats] of stats) { + // process packet loss stats + const loss = trackStats.getLoss(); + const type = loss.isDownloadStream ? "download" : "upload"; + + totalPackets[type] += loss.packetsTotal; + lostPackets[type] += loss.packetsLost; + + // process bitrate stats + bitrateDownload += trackStats.getBitrate().download; + bitrateUpload += trackStats.getBitrate().upload; + + // collect resolutions and framerates + if (trackStats.kind === "audio") { + audioBitrateDownload += trackStats.getBitrate().download; + audioBitrateUpload += trackStats.getBitrate().upload; + } else { + videoBitrateDownload += trackStats.getBitrate().download; + videoBitrateUpload += trackStats.getBitrate().upload; + } + + resolutions[trackStats.getType()].set(trackId, trackStats.getResolution()); + framerates[trackStats.getType()].set(trackId, trackStats.getFramerate()); + codecs[trackStats.getType()].set(trackId, trackStats.getCodec()); + + trackStats.resetBitrate(); + } + + report.bitrate = { + upload: bitrateUpload, + download: bitrateDownload, + }; + + report.bitrate.audio = { + upload: audioBitrateUpload, + download: audioBitrateDownload, + }; + + report.bitrate.video = { + upload: videoBitrateUpload, + download: videoBitrateDownload, + }; + + report.packetLoss = { + total: StatsReportBuilder.calculatePacketLoss( + lostPackets.download + lostPackets.upload, + totalPackets.download + totalPackets.upload, + ), + download: StatsReportBuilder.calculatePacketLoss(lostPackets.download, totalPackets.download), + upload: StatsReportBuilder.calculatePacketLoss(lostPackets.upload, totalPackets.upload), + }; + report.framerate = framerates; + report.resolution = resolutions; + report.codec = codecs; + return report; + } + + private static calculatePacketLoss(lostPackets: number, totalPackets: number): number { + if (!totalPackets || totalPackets <= 0 || !lostPackets || lostPackets <= 0) { + return 0; + } + + return Math.round((lostPackets / totalPackets) * 100); + } +} diff --git a/src/webrtc/stats/statsReportEmitter.ts b/src/webrtc/stats/statsReportEmitter.ts new file mode 100644 index 00000000000..e888ec1e849 --- /dev/null +++ b/src/webrtc/stats/statsReportEmitter.ts @@ -0,0 +1,33 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { TypedEventEmitter } from "../../models/typed-event-emitter"; +import { ByteSendStatsReport, ConnectionStatsReport, StatsReport } from "./statsReport"; + +export type StatsReportHandlerMap = { + [StatsReport.BYTE_SENT_STATS]: (report: ByteSendStatsReport) => void; + [StatsReport.CONNECTION_STATS]: (report: ConnectionStatsReport) => void; +}; + +export class StatsReportEmitter extends TypedEventEmitter { + public emitByteSendReport(byteSentStats: ByteSendStatsReport): void { + this.emit(StatsReport.BYTE_SENT_STATS, byteSentStats); + } + + public emitConnectionStatsReport(report: ConnectionStatsReport): void { + this.emit(StatsReport.CONNECTION_STATS, report); + } +} diff --git a/src/webrtc/stats/statsValueFormatter.ts b/src/webrtc/stats/statsValueFormatter.ts new file mode 100644 index 00000000000..b377a409b5b --- /dev/null +++ b/src/webrtc/stats/statsValueFormatter.ts @@ -0,0 +1,30 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +export class StatsValueFormatter { + public static getNonNegativeValue(imput: any): number { + let value = imput; + + if (typeof value !== "number") { + value = Number(value); + } + + if (isNaN(value)) { + return 0; + } + + return Math.max(0, value); + } +} diff --git a/src/webrtc/stats/trackStatsReporter.ts b/src/webrtc/stats/trackStatsReporter.ts new file mode 100644 index 00000000000..1f6fcd6d1ce --- /dev/null +++ b/src/webrtc/stats/trackStatsReporter.ts @@ -0,0 +1,117 @@ +import { MediaTrackStats } from "./media/mediaTrackStats"; +import { StatsValueFormatter } from "./statsValueFormatter"; + +export class TrackStatsReporter { + public static buildFramerateResolution(trackStats: MediaTrackStats, now: any): void { + const resolution = { + height: now.frameHeight, + width: now.frameWidth, + }; + const frameRate = now.framesPerSecond; + + if (resolution.height && resolution.width) { + trackStats.setResolution(resolution); + } + trackStats.setFramerate(Math.round(frameRate || 0)); + } + + public static calculateSimulcastFramerate(trackStats: MediaTrackStats, now: any, before: any, layer: number): void { + let frameRate = trackStats.getFramerate(); + if (!frameRate) { + if (before) { + const timeMs = now.timestamp - before.timestamp; + + if (timeMs > 0 && now.framesSent) { + const numberOfFramesSinceBefore = now.framesSent - before.framesSent; + + frameRate = (numberOfFramesSinceBefore / timeMs) * 1000; + } + } + + if (!frameRate) { + return; + } + } + + // Reset frame rate to 0 when video is suspended as a result of endpoint falling out of last-n. + frameRate = layer ? Math.round(frameRate / layer) : 0; + trackStats.setFramerate(frameRate); + } + + public static buildCodec(report: RTCStatsReport | undefined, trackStats: MediaTrackStats, now: any): void { + const codec = report?.get(now.codecId); + + if (codec) { + /** + * The mime type has the following form: video/VP8 or audio/ISAC, + * so we what to keep just the type after the '/', audio and video + * keys will be added on the processing side. + */ + const codecShortType = codec.mimeType.split("/")[1]; + + codecShortType && trackStats.setCodec(codecShortType); + } + } + + public static buildBitrateReceived(trackStats: MediaTrackStats, now: any, before: any): void { + trackStats.setBitrate({ + download: TrackStatsReporter.calculateBitrate( + now.bytesReceived, + before.bytesReceived, + now.timestamp, + before.timestamp, + ), + upload: 0, + }); + } + + public static buildBitrateSend(trackStats: MediaTrackStats, now: any, before: any): void { + trackStats.setBitrate({ + download: 0, + upload: this.calculateBitrate(now.bytesSent, before.bytesSent, now.timestamp, before.timestamp), + }); + } + + public static buildPacketsLost(trackStats: MediaTrackStats, now: any, before: any): void { + const key = now.type === "outbound-rtp" ? "packetsSent" : "packetsReceived"; + + let packetsNow = now[key]; + if (!packetsNow || packetsNow < 0) { + packetsNow = 0; + } + + const packetsBefore = StatsValueFormatter.getNonNegativeValue(before[key]); + const packetsDiff = Math.max(0, packetsNow - packetsBefore); + + const packetsLostNow = StatsValueFormatter.getNonNegativeValue(now.packetsLost); + const packetsLostBefore = StatsValueFormatter.getNonNegativeValue(before.packetsLost); + const packetsLostDiff = Math.max(0, packetsLostNow - packetsLostBefore); + + trackStats.setLoss({ + packetsTotal: packetsDiff + packetsLostDiff, + packetsLost: packetsLostDiff, + isDownloadStream: now.type !== "outbound-rtp", + }); + } + + private static calculateBitrate( + bytesNowAny: any, + bytesBeforeAny: any, + nowTimestamp: number, + beforeTimestamp: number, + ): number { + const bytesNow = StatsValueFormatter.getNonNegativeValue(bytesNowAny); + const bytesBefore = StatsValueFormatter.getNonNegativeValue(bytesBeforeAny); + const bytesProcessed = Math.max(0, bytesNow - bytesBefore); + + const timeMs = nowTimestamp - beforeTimestamp; + let bitrateKbps = 0; + + if (timeMs > 0) { + // TODO is there any reason to round here? + bitrateKbps = Math.round((bytesProcessed * 8) / timeMs); + } + + return bitrateKbps; + } +} diff --git a/src/webrtc/stats/transportStats.ts b/src/webrtc/stats/transportStats.ts new file mode 100644 index 00000000000..2b6e975484f --- /dev/null +++ b/src/webrtc/stats/transportStats.ts @@ -0,0 +1,26 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +export interface TransportStats { + ip: string; + type: string; + localIp: string; + isFocus: boolean; + localCandidateType: string; + remoteCandidateType: string; + networkType: string; + rtt: number; +} diff --git a/src/webrtc/stats/transportStatsReporter.ts b/src/webrtc/stats/transportStatsReporter.ts new file mode 100644 index 00000000000..d419a73972b --- /dev/null +++ b/src/webrtc/stats/transportStatsReporter.ts @@ -0,0 +1,48 @@ +import { TransportStats } from "./transportStats"; + +export class TransportStatsReporter { + public static buildReport( + report: RTCStatsReport | undefined, + now: RTCIceCandidatePairStats, + conferenceStatsTransport: TransportStats[], + isFocus: boolean, + ): TransportStats[] { + const localUsedCandidate = report?.get(now.localCandidateId); + const remoteUsedCandidate = report?.get(now.remoteCandidateId); + + // RTCIceCandidateStats + // https://w3c.github.io/webrtc-stats/#icecandidate-dict* + if (remoteUsedCandidate && localUsedCandidate) { + const remoteIpAddress = + remoteUsedCandidate.ip !== undefined ? remoteUsedCandidate.ip : remoteUsedCandidate.address; + const remotePort = remoteUsedCandidate.port; + const ip = `${remoteIpAddress}:${remotePort}`; + + const localIpAddress = + localUsedCandidate.ip !== undefined ? localUsedCandidate.ip : localUsedCandidate.address; + const localPort = localUsedCandidate.port; + const localIp = `${localIpAddress}:${localPort}`; + + const type = remoteUsedCandidate.protocol; + + // Save the address unless it has been saved already. + if ( + !conferenceStatsTransport.some( + (t: TransportStats) => t.ip === ip && t.type === type && t.localIp === localIp, + ) + ) { + conferenceStatsTransport.push({ + ip, + type, + localIp, + isFocus, + localCandidateType: localUsedCandidate.candidateType, + remoteCandidateType: remoteUsedCandidate.candidateType, + networkType: localUsedCandidate.networkType, + rtt: now.currentRoundTripTime ? now.currentRoundTripTime * 1000 : NaN, + } as TransportStats); + } + } + return conferenceStatsTransport; + } +} From bcd86c21f40007ede72c24edfd235eda88feed30 Mon Sep 17 00:00:00 2001 From: Enrico Schwendig Date: Wed, 22 Mar 2023 11:30:29 +0100 Subject: [PATCH 137/137] Exporting live WebRTC metric as GroupCall Event from SDK #3220 (#3224) * develop: merge voip event from `develop` branch * stats: start statistics if a calls starts * stats: add a `GroupCallStatsReport` and fix typo * stats: add callback as anonymous fkt for right ctx * stats: check for promise for backward compatibility * stats: add test for promise check --- spec/unit/webrtc/stats/statsCollector.spec.ts | 10 +++++ .../webrtc/stats/statsReportEmitter.spec.ts | 4 +- src/webrtc/call.ts | 12 ++++- src/webrtc/groupCall.ts | 37 +++++++++++----- src/webrtc/stats/statsCollector.ts | 44 ++++++++++--------- src/webrtc/stats/statsReport.ts | 6 +-- src/webrtc/stats/statsReportEmitter.ts | 6 +-- 7 files changed, 79 insertions(+), 40 deletions(-) diff --git a/spec/unit/webrtc/stats/statsCollector.spec.ts b/spec/unit/webrtc/stats/statsCollector.spec.ts index 2397f092183..c7cc0408994 100644 --- a/spec/unit/webrtc/stats/statsCollector.spec.ts +++ b/spec/unit/webrtc/stats/statsCollector.spec.ts @@ -54,5 +54,15 @@ describe("StatsCollector", () => { expect(getStats).toHaveBeenCalled(); expect(collector.getActive()).toBeFalsy(); }); + + it("if active an RTCStatsReport not a promise the collector becomes inactive", async () => { + const getStats = jest.spyOn(rtcSpy, "getStats"); + // @ts-ignore + getStats.mockReturnValue({}); + const actual = await collector.processStats("GROUP_CALL_ID", "LOCAL_USER_ID"); + expect(actual).toBeFalsy(); + expect(getStats).toHaveBeenCalled(); + expect(collector.getActive()).toBeFalsy(); + }); }); }); diff --git a/spec/unit/webrtc/stats/statsReportEmitter.spec.ts b/spec/unit/webrtc/stats/statsReportEmitter.spec.ts index 715ea1bef4b..de75d44746d 100644 --- a/spec/unit/webrtc/stats/statsReportEmitter.spec.ts +++ b/spec/unit/webrtc/stats/statsReportEmitter.spec.ts @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ import { StatsReportEmitter } from "../../../../src/webrtc/stats/statsReportEmitter"; -import { ByteSendStatsReport, ConnectionStatsReport, StatsReport } from "../../../../src/webrtc/stats/statsReport"; +import { ByteSentStatsReport, ConnectionStatsReport, StatsReport } from "../../../../src/webrtc/stats/statsReport"; describe("StatsReportEmitter", () => { let emitter: StatsReportEmitter; @@ -23,7 +23,7 @@ describe("StatsReportEmitter", () => { }); it("should emit and receive ByteSendStatsReport", async () => { - const report = {} as ByteSendStatsReport; + const report = {} as ByteSentStatsReport; return new Promise((resolve, _) => { emitter.on(StatsReport.BYTE_SENT_STATS, (r) => { expect(r).toBe(report); diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 519784e77c9..bc519758dd9 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -310,6 +310,15 @@ function isFirefox(): boolean { return navigator.userAgent.indexOf("Firefox") !== -1; } +export interface VoipEvent { + type: "toDevice" | "sendEvent"; + eventType: string; + userId?: string; + opponentDeviceId?: string; + roomId?: string; + content: Record; +} + export type CallEventHandlerMap = { [CallEvent.DataChannel]: (channel: RTCDataChannel) => void; [CallEvent.FeedsChanged]: (feeds: CallFeed[]) => void; @@ -323,7 +332,7 @@ export type CallEventHandlerMap = { [CallEvent.AssertedIdentityChanged]: () => void; /* @deprecated */ [CallEvent.HoldUnhold]: (onHold: boolean) => void; - [CallEvent.SendVoipEvent]: (event: Record) => void; + [CallEvent.SendVoipEvent]: (event: VoipEvent) => void; }; export class MatrixCall extends TypedEventEmitter { @@ -3109,6 +3118,7 @@ export class MatrixCall extends TypedEventEmitter void; }; +export enum GroupCallStatsReportEvent { + ConnectionStats = "GroupCall.connection_stats", + ByteSentStats = "GroupCall.byte_sent_stats", +} + +export type GroupCallStatsReportEventHandlerMap = { + [GroupCallStatsReportEvent.ConnectionStats]: (report: GroupCallStatsReport) => void; + [GroupCallStatsReportEvent.ByteSentStats]: (report: GroupCallStatsReport) => void; +}; + export enum GroupCallErrorCode { NoUserMedia = "no_user_media", UnknownDevice = "unknown_device", PlaceCallFailed = "place_call_failed", } +export interface GroupCallStatsReport { + report: T; +} + export class GroupCallError extends Error { public code: string; @@ -185,8 +199,8 @@ function getCallUserId(call: MatrixCall): string | null { } export class GroupCall extends TypedEventEmitter< - GroupCallEvent | CallEvent, - GroupCallEventHandlerMap & CallEventHandlerMap + GroupCallEvent | CallEvent | GroupCallStatsReportEvent, + GroupCallEventHandlerMap & CallEventHandlerMap & GroupCallStatsReportEventHandlerMap > { // Config public activeSpeakerInterval = 1000; @@ -239,19 +253,22 @@ export class GroupCall extends TypedEventEmitter< this.on(GroupCallEvent.ParticipantsChanged, this.onParticipantsChanged); this.on(GroupCallEvent.GroupCallStateChanged, this.onStateChanged); this.on(GroupCallEvent.LocalScreenshareStateChanged, this.onLocalFeedsChanged); + const userID = this.client.getUserId() || "unknown"; this.stats = new GroupCallStats(this.groupCallId, userID); this.stats.reports.on(StatsReport.CONNECTION_STATS, this.onConnectionStats); - this.stats.reports.on(StatsReport.BYTE_SENT_STATS, this.onByteSendStats); + this.stats.reports.on(StatsReport.BYTE_SENT_STATS, this.onByteSentStats); } - private onConnectionStats(_: ConnectionStatsReport): void { - // @TODO: Implement data argumentation and event broadcasting please - } + private onConnectionStats = (report: ConnectionStatsReport): void => { + // @TODO: Implement data argumentation + this.emit(GroupCallStatsReportEvent.ConnectionStats, { report }); + }; - private onByteSendStats(_: ByteSendStatsReport): void { - // @TODO: Implement data argumentation and event broadcasting please - } + private onByteSentStats = (report: ByteSentStatsReport): void => { + // @TODO: Implement data argumentation + this.emit(GroupCallStatsReportEvent.ByteSentStats, { report }); + }; public async create(): Promise { this.creationTs = Date.now(); diff --git a/src/webrtc/stats/statsCollector.ts b/src/webrtc/stats/statsCollector.ts index 302004a5bb1..b58201183d7 100644 --- a/src/webrtc/stats/statsCollector.ts +++ b/src/webrtc/stats/statsCollector.ts @@ -16,7 +16,7 @@ limitations under the License. import { ConnectionStats } from "./connectionStats"; import { StatsReportEmitter } from "./statsReportEmitter"; -import { ByteSend, ByteSendStatsReport, TrackID } from "./statsReport"; +import { ByteSend, ByteSentStatsReport, TrackID } from "./statsReport"; import { ConnectionStatsReporter } from "./connectionStatsReporter"; import { TransportStatsReporter } from "./transportStatsReporter"; import { MediaSsrcHandler } from "./media/mediaSsrcHandler"; @@ -49,32 +49,34 @@ export class StatsCollector { public async processStats(groupCallId: string, localUserId: string): Promise { if (this.isActive) { - return this.pc - .getStats() - .then((report) => { - // @ts-ignore - this.currentStatsReport = typeof report?.result === "function" ? report.result() : report; - try { - this.processStatsReport(groupCallId, localUserId); - } catch (error) { - this.isActive = false; + const statsPromise = this.pc.getStats(); + if (typeof statsPromise?.then === "function") { + return statsPromise + .then((report) => { + // @ts-ignore + this.currentStatsReport = typeof report?.result === "function" ? report.result() : report; + try { + this.processStatsReport(groupCallId, localUserId); + } catch (error) { + this.isActive = false; + return false; + } + + this.previousStatsReport = this.currentStatsReport; + return true; + }) + .catch((error) => { + this.handleError(error); return false; - // logger.error('Processing of RTP stats failed:', error); - } - - this.previousStatsReport = this.currentStatsReport; - return true; - }) - .catch((error) => { - this.handleError(error); - return false; - }); + }); + } + this.isActive = false; } return Promise.resolve(false); } private processStatsReport(groupCallId: string, localUserId: string): void { - const byteSentStats: ByteSendStatsReport = new Map(); + const byteSentStats: ByteSentStatsReport = new Map(); this.currentStatsReport?.forEach((now) => { const before = this.previousStatsReport ? this.previousStatsReport.get(now.id) : null; diff --git a/src/webrtc/stats/statsReport.ts b/src/webrtc/stats/statsReport.ts index ab6db696bee..56d6c4b2e48 100644 --- a/src/webrtc/stats/statsReport.ts +++ b/src/webrtc/stats/statsReport.ts @@ -19,14 +19,14 @@ import { TransportStats } from "./transportStats"; import { Resolution } from "./media/mediaTrackStats"; export enum StatsReport { - CONNECTION_STATS = "connection_stats", - BYTE_SENT_STATS = "byte_sent_stats", + CONNECTION_STATS = "StatsReport.connection_stats", + BYTE_SENT_STATS = "StatsReport.byte_sent_stats", } export type TrackID = string; export type ByteSend = number; -export interface ByteSendStatsReport extends Map { +export interface ByteSentStatsReport extends Map { // is a map: `local trackID` => byte send } diff --git a/src/webrtc/stats/statsReportEmitter.ts b/src/webrtc/stats/statsReportEmitter.ts index e888ec1e849..cf014708e89 100644 --- a/src/webrtc/stats/statsReportEmitter.ts +++ b/src/webrtc/stats/statsReportEmitter.ts @@ -15,15 +15,15 @@ limitations under the License. */ import { TypedEventEmitter } from "../../models/typed-event-emitter"; -import { ByteSendStatsReport, ConnectionStatsReport, StatsReport } from "./statsReport"; +import { ByteSentStatsReport, ConnectionStatsReport, StatsReport } from "./statsReport"; export type StatsReportHandlerMap = { - [StatsReport.BYTE_SENT_STATS]: (report: ByteSendStatsReport) => void; + [StatsReport.BYTE_SENT_STATS]: (report: ByteSentStatsReport) => void; [StatsReport.CONNECTION_STATS]: (report: ConnectionStatsReport) => void; }; export class StatsReportEmitter extends TypedEventEmitter { - public emitByteSendReport(byteSentStats: ByteSendStatsReport): void { + public emitByteSendReport(byteSentStats: ByteSentStatsReport): void { this.emit(StatsReport.BYTE_SENT_STATS, byteSentStats); }