From 620bbf49c8d412777c881acf96e14844b6c3cfe4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Ro=C5=BCnawski?= Date: Mon, 7 Oct 2024 15:24:04 +0200 Subject: [PATCH 1/4] Ignore missing senders --- packages/ts-client/src/FishjamClient.ts | 2 +- .../ts-client/src/webrtc/CommandsQueue.ts | 1 + .../ts-client/src/webrtc/ConnectionManager.ts | 31 ++----------------- packages/ts-client/src/webrtc/tracks/Local.ts | 14 ++++++--- .../ts-client/src/webrtc/tracks/LocalTrack.ts | 5 +-- .../ts-client/src/webrtc/tracks/Remote.ts | 7 ++--- .../ts-client/src/webrtc/webRTCEndpoint.ts | 24 +++++++------- 7 files changed, 32 insertions(+), 52 deletions(-) diff --git a/packages/ts-client/src/FishjamClient.ts b/packages/ts-client/src/FishjamClient.ts index 26074520..3269babb 100644 --- a/packages/ts-client/src/FishjamClient.ts +++ b/packages/ts-client/src/FishjamClient.ts @@ -29,7 +29,7 @@ export type Component = Omit( endpoint: Endpoint, -): endpoint is Peer => endpoint.type === 'webrtc'; +): endpoint is Peer => endpoint.type === 'webrtc' || endpoint.type === 'exwebrtc'; const isComponent = ( endpoint: Endpoint, ): endpoint is Component => diff --git a/packages/ts-client/src/webrtc/CommandsQueue.ts b/packages/ts-client/src/webrtc/CommandsQueue.ts index e8b5d35c..fe8ed972 100644 --- a/packages/ts-client/src/webrtc/CommandsQueue.ts +++ b/packages/ts-client/src/webrtc/CommandsQueue.ts @@ -38,6 +38,7 @@ export class CommandsQueue { this.processNextCommand(); } }; + const onIceConnectionStateChange = () => { if (connection.getConnection().iceConnectionState === 'connected') { this.processNextCommand(); diff --git a/packages/ts-client/src/webrtc/ConnectionManager.ts b/packages/ts-client/src/webrtc/ConnectionManager.ts index ffbf401a..afcc8236 100644 --- a/packages/ts-client/src/webrtc/ConnectionManager.ts +++ b/packages/ts-client/src/webrtc/ConnectionManager.ts @@ -1,21 +1,12 @@ import type { MediaStreamTrackId } from './types'; -export type TurnServer = { - transport: string; - password: string; - serverAddr: string; - serverPort: string; - username: string; -}; - export class ConnectionManager { private readonly connection: RTCPeerConnection; - constructor(turnServers: TurnServer[]) { + constructor() { this.connection = new RTCPeerConnection({ bundlePolicy: 'max-bundle', - iceServers: this.getIceServers(turnServers), - iceTransportPolicy: 'relay', + iceTransportPolicy: 'all', }); } @@ -29,24 +20,6 @@ export class ConnectionManager { return isSignalingUnstable && isConnectionNotConnected && isIceNotConnected; }; - /** - * Configures TURN servers for WebRTC connections by adding them to the provided RTCConfiguration object. - */ - private getIceServers = (turnServers: TurnServer[]): RTCIceServer[] => { - return turnServers.map((turnServer: TurnServer) => { - const transport = turnServer.transport === 'tls' ? 'tcp' : turnServer.transport; - const uri = turnServer.transport === 'tls' ? 'turns' : 'turn'; - const address = turnServer.serverAddr; - const port = turnServer.serverPort; - - return { - credential: turnServer.password, - urls: uri.concat(':', address, ':', port, '?transport=', transport), - username: turnServer.username, - }; - }); - }; - public getConnection = (): RTCPeerConnection => { return this.connection; }; diff --git a/packages/ts-client/src/webrtc/tracks/Local.ts b/packages/ts-client/src/webrtc/tracks/Local.ts index 072e56b5..ac2d9759 100644 --- a/packages/ts-client/src/webrtc/tracks/Local.ts +++ b/packages/ts-client/src/webrtc/tracks/Local.ts @@ -334,10 +334,16 @@ export class Local { private getTrackIdToTrackBitrates = (): Record => { return Object.values(this.localTracks).reduce( - (previousValue, localTrack) => ({ - ...previousValue, - [localTrack.id]: localTrack.getTrackBitrates(), - }), + (previousValue, localTrack) => { + const bitrates = localTrack.getTrackBitrates(); + if (bitrates) { + return { + ...previousValue, + [localTrack.id]: bitrates, + } + } + return previousValue; + }, {} as Record, ); }; diff --git a/packages/ts-client/src/webrtc/tracks/LocalTrack.ts b/packages/ts-client/src/webrtc/tracks/LocalTrack.ts index 75387530..73dae64b 100644 --- a/packages/ts-client/src/webrtc/tracks/LocalTrack.ts +++ b/packages/ts-client/src/webrtc/tracks/LocalTrack.ts @@ -243,7 +243,7 @@ export class LocalTrack implements TrackCommon private isNotSimulcastTrack = (encodings: RTCRtpEncodingParameters[]): boolean => encodings.length === 1 && !encodings[0]!.rid; - public getTrackBitrates = (): Bitrates => { + public getTrackBitrates = (): Bitrates | undefined => { const trackContext = this.trackContext; const kind = this.trackContext.track?.kind as TrackKind | undefined; @@ -255,7 +255,8 @@ export class LocalTrack implements TrackCommon return defaultBitrates[trackContext.trackKind]; } - if (!this.sender) throw new Error(`RTCRtpSender for track ${this.id} not found`); + if (!this.sender) + return undefined; const encodings = this.sender.getParameters().encodings; diff --git a/packages/ts-client/src/webrtc/tracks/Remote.ts b/packages/ts-client/src/webrtc/tracks/Remote.ts index 03781998..b11a2926 100644 --- a/packages/ts-client/src/webrtc/tracks/Remote.ts +++ b/packages/ts-client/src/webrtc/tracks/Remote.ts @@ -56,7 +56,6 @@ export class Remote { public addTracks = ( endpointId: EndpointId, tracks: Record, - trackIdToMetadata: Record, ) => { const endpoint: EndpointWithTrackContext | undefined = this.remoteEndpoints[endpointId]; @@ -64,11 +63,11 @@ export class Remote { if (!endpoint) throw new Error(`Endpoint ${endpointId} not found`); Object.entries(tracks || {}) - .map(([trackId, { simulcastConfig }]) => { + .map(([trackId, { simulcastConfig, metadata }]) => { const trackContext = new TrackContextImpl( endpoint, trackId, - trackIdToMetadata[trackId], + metadata, simulcastConfig, this.trackMetadataParser, ); @@ -115,7 +114,7 @@ export class Remote { this.updateEndpointMetadata(newEndpoint, endpoint?.metadata?.peer); this.addEndpoint(newEndpoint); - this.addTracks(newEndpoint.id, endpoint.tracks, endpoint.trackIdToMetadata); + this.addTracks(newEndpoint.id, endpoint.tracks); if (sendNotification) { this.emit('endpointAdded', endpoint); diff --git a/packages/ts-client/src/webrtc/webRTCEndpoint.ts b/packages/ts-client/src/webrtc/webRTCEndpoint.ts index 7ed1a386..ab7a35c0 100644 --- a/packages/ts-client/src/webrtc/webRTCEndpoint.ts +++ b/packages/ts-client/src/webrtc/webRTCEndpoint.ts @@ -20,7 +20,6 @@ import { LocalTrackManager } from './tracks/LocalTrackManager'; import { CommandsQueue } from './CommandsQueue'; import { Remote } from './tracks/Remote'; import { Local } from './tracks/Local'; -import type { TurnServer } from './ConnectionManager'; import { ConnectionManager } from './ConnectionManager'; /** @@ -208,7 +207,7 @@ export class WebRTCEndpoint extends if (this.getEndpointId() === data.endpointId) return; - this.remote.addTracks(data.endpointId, data.tracks, data.trackIdToMetadata); + this.remote.addTracks(data.endpointId, data.tracks); break; } case 'tracksRemoved': { @@ -224,9 +223,9 @@ export class WebRTCEndpoint extends } case 'sdpAnswer': + this.localTrackManager.ongoingRenegotiation = false; await this.onSdpAnswer(deserializedMediaEvent.data); - this.localTrackManager.ongoingRenegotiation = false; this.commandsQueue.processNextCommand(); break; @@ -689,6 +688,7 @@ export class WebRTCEndpoint extends // todo change to private public sendMediaEvent = (mediaEvent: MediaEvent) => { + console.log("send ME", mediaEvent) const serializedMediaEvent = serializeMediaEvent(mediaEvent); this.emit('sendMediaEvent', serializedMediaEvent); }; @@ -698,6 +698,8 @@ export class WebRTCEndpoint extends if (!connection) return; try { + this.localTrackManager.updateSenders(); + const offer = await connection.getConnection().createOffer(); if (!this.connectionManager) { @@ -723,10 +725,8 @@ export class WebRTCEndpoint extends private onOfferData = async (offerData: MediaEvent) => { const connection = this.connectionManager; - if (connection) { - connection.getConnection().restartIce(); - } else { - this.setConnection(offerData.data.integratedTurnServers); + if (!connection) { + this.setConnection(); const onIceCandidate = (event: RTCPeerConnectionIceEvent) => this.onLocalCandidate(event); const onIceCandidateError = (event: RTCPeerConnectionIceErrorEvent) => this.onIceCandidateError(event); @@ -753,8 +753,6 @@ export class WebRTCEndpoint extends this.local.addAllTracksToConnection(); } - this.localTrackManager.updateSenders(); - const tracks = new Map(Object.entries(offerData.data.tracksTypes)); this.connectionManager?.addTransceiversIfNeeded(tracks); @@ -762,8 +760,8 @@ export class WebRTCEndpoint extends await this.createAndSendOffer(); }; - private setConnection = (turnServers: TurnServer[]) => { - this.connectionManager = new ConnectionManager(turnServers); + private setConnection = () => { + this.connectionManager = new ConnectionManager(); this.localTrackManager.updateConnection(this.connectionManager); this.local.updateConnection(this.connectionManager); @@ -788,6 +786,8 @@ export class WebRTCEndpoint extends data: { candidate: event.candidate.candidate, sdpMLineIndex: event.candidate.sdpMLineIndex, + sdpMid: event.candidate.sdpMid, + usernameFragment: event.candidate.usernameFragment, }, }); this.sendMediaEvent(mediaEvent); @@ -815,7 +815,7 @@ export class WebRTCEndpoint extends console.warn('ICE connection: disconnected'); // Requesting renegotiation on ICE connection state failed fixes RTCPeerConnection // when the user changes their WiFi network. - this.sendMediaEvent(generateCustomEvent({ type: 'renegotiateTracks' })); + // this.sendMediaEvent(generateCustomEvent({ type: 'renegotiateTracks' })); break; case 'failed': this.emit('connectionError', { From afdd4055d6f1537e5837c48112d1190455f19c52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Ro=C5=BCnawski?= Date: Mon, 7 Oct 2024 15:24:04 +0200 Subject: [PATCH 2/4] Fix some yarn problems --- packages/ts-client/src/webrtc/webRTCEndpoint.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ts-client/src/webrtc/webRTCEndpoint.ts b/packages/ts-client/src/webrtc/webRTCEndpoint.ts index ab7a35c0..34639daa 100644 --- a/packages/ts-client/src/webrtc/webRTCEndpoint.ts +++ b/packages/ts-client/src/webrtc/webRTCEndpoint.ts @@ -334,13 +334,13 @@ export class WebRTCEndpoint extends .map((mid) => this.local.getTrackByMidOrNull(mid)) .filter((localTrack) => localTrack !== null) .forEach((localTrack) => { - const trackContext = localTrack.trackContext; + const trackContext = localTrack!.trackContext; trackContext.negotiationStatus = 'done'; if (trackContext.pendingMetadataUpdate) { const mediaEvent = generateMediaEvent('updateTrackMetadata', { - trackId: localTrack.id, + trackId: localTrack!.id, trackMetadata: trackContext.metadata, }); this.sendMediaEvent(mediaEvent); From ebe9305bc501d6c883829004bbdc58d494177f8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Ro=C5=BCnawski?= Date: Mon, 7 Oct 2024 15:24:04 +0200 Subject: [PATCH 3/4] stuns --- packages/ts-client/src/webrtc/ConnectionManager.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/ts-client/src/webrtc/ConnectionManager.ts b/packages/ts-client/src/webrtc/ConnectionManager.ts index afcc8236..e209dbff 100644 --- a/packages/ts-client/src/webrtc/ConnectionManager.ts +++ b/packages/ts-client/src/webrtc/ConnectionManager.ts @@ -7,6 +7,7 @@ export class ConnectionManager { this.connection = new RTCPeerConnection({ bundlePolicy: 'max-bundle', iceTransportPolicy: 'all', + iceServers: [{ urls: 'stun:stun.l.google.com:19302' }, { urls: "stun:stun.l.google.com:5349" }] }); } From 49508322eed3958cb78d4715d5fe5f5a50b671da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Ro=C5=BCnawski?= Date: Mon, 7 Oct 2024 15:35:25 +0200 Subject: [PATCH 4/4] Logs + address --- examples/ts-client/simple-app/src/main.ts | 4 +++- packages/ts-client/src/webrtc/webRTCEndpoint.ts | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/examples/ts-client/simple-app/src/main.ts b/examples/ts-client/simple-app/src/main.ts index 9871a136..ed499114 100644 --- a/examples/ts-client/simple-app/src/main.ts +++ b/examples/ts-client/simple-app/src/main.ts @@ -353,7 +353,9 @@ client.on("bandwidthEstimationChanged", (_estimation) => {}); client.on("tracksPriorityChanged", (_enabledTracks, _disabledTracks) => {}); -const FISHJAM_URL = "ws://localhost:5002"; +// const FISHJAM_URL = "https://cloud.fishjam.work/api/v1/connect/d541bcac1431418a80f20990377aa819"; +const FISHJAM_URL = "wss://cloud.fishjam.work/api/v1/connect/d1cfbbd138884749bc411356062df0d8"; +// const FISHJAM_URL = "ws://localhost:5002"; connectButton.addEventListener("click", () => { console.log("Connect"); diff --git a/packages/ts-client/src/webrtc/webRTCEndpoint.ts b/packages/ts-client/src/webrtc/webRTCEndpoint.ts index 34639daa..d073ff89 100644 --- a/packages/ts-client/src/webrtc/webRTCEndpoint.ts +++ b/packages/ts-client/src/webrtc/webRTCEndpoint.ts @@ -196,6 +196,8 @@ export class WebRTCEndpoint extends } private handleMediaEvent = async (deserializedMediaEvent: MediaEvent) => { + console.log("incoming me", { deserializedMediaEvent }); + switch (deserializedMediaEvent.type) { case 'offerData': { await this.onOfferData(deserializedMediaEvent); @@ -296,6 +298,10 @@ export class WebRTCEndpoint extends break; case 'error': + console.log("signaling error", { + message: deserializedMediaEvent.data.message, + }); + this.emit('signalingError', { message: deserializedMediaEvent.data.message, }); @@ -799,6 +805,7 @@ export class WebRTCEndpoint extends }; private onConnectionStateChange = (event: Event) => { + console.log("onConnectionStateChange", event, this.localTrackManager.connection?.getConnection().connectionState); switch (this.localTrackManager.connection?.getConnection().connectionState) { case 'failed': this.emit('connectionError', {