Comply with ex_webrtc engine endpoint

examples/ts-client/simple-app/src/main.ts

@@ -353,7 +353,9 @@ client.on("bandwidthEstimationChanged", (_estimation) => {});
 
 client.on("tracksPriorityChanged", (_enabledTracks, _disabledTracks) => {});
 
-const FISHJAM_URL = "ws://localhost:5002";
+// const FISHJAM_URL = "https://cloud.fishjam.work/api/v1/connect/d541bcac1431418a80f20990377aa819";
+const FISHJAM_URL = "wss://cloud.fishjam.work/api/v1/connect/d1cfbbd138884749bc411356062df0d8";
+// const FISHJAM_URL = "ws://localhost:5002";
 
 connectButton.addEventListener("click", () => {
   console.log("Connect");

packages/ts-client/src/FishjamClient.ts

@@ -29,7 +29,7 @@ export type Component<PeerMetadata, TrackMetadata> = Omit<Endpoint<PeerMetadata,
 
 const isPeer = <PeerMetadata, TrackMetadata>(
   endpoint: Endpoint<PeerMetadata, TrackMetadata>,
-): endpoint is Peer<PeerMetadata, TrackMetadata> => endpoint.type === 'webrtc';
+): endpoint is Peer<PeerMetadata, TrackMetadata> => endpoint.type === 'webrtc' || endpoint.type === 'exwebrtc';
 const isComponent = <PeerMetadata, TrackMetadata>(
   endpoint: Endpoint<PeerMetadata, TrackMetadata>,
 ): endpoint is Component<PeerMetadata, TrackMetadata> =>

packages/ts-client/src/webrtc/CommandsQueue.ts

@@ -38,6 +38,7 @@ export class CommandsQueue<EndpointMetadata, TrackMetadata> {
         this.processNextCommand();
       }
     };
+
     const onIceConnectionStateChange = () => {
       if (connection.getConnection().iceConnectionState === 'connected') {
         this.processNextCommand();

packages/ts-client/src/webrtc/ConnectionManager.ts

@@ -1,21 +1,13 @@
 import type { MediaStreamTrackId } from './types';
 
-export type TurnServer = {
-  transport: string;
-  password: string;
-  serverAddr: string;
-  serverPort: string;
-  username: string;
-};
-
 export class ConnectionManager {
   private readonly connection: RTCPeerConnection;
 
-  constructor(turnServers: TurnServer[]) {
+  constructor() {
     this.connection = new RTCPeerConnection({
       bundlePolicy: 'max-bundle',
-      iceServers: this.getIceServers(turnServers),
-      iceTransportPolicy: 'relay',
+      iceTransportPolicy: 'all',
+      iceServers: [{ urls: 'stun:stun.l.google.com:19302' }, { urls: "stun:stun.l.google.com:5349" }]
     });
   }
 
@@ -29,24 +21,6 @@ export class ConnectionManager {
     return isSignalingUnstable && isConnectionNotConnected && isIceNotConnected;
   };
 
-  /**
-   * Configures TURN servers for WebRTC connections by adding them to the provided RTCConfiguration object.
-   */
-  private getIceServers = (turnServers: TurnServer[]): RTCIceServer[] => {
-    return turnServers.map((turnServer: TurnServer) => {
-      const transport = turnServer.transport === 'tls' ? 'tcp' : turnServer.transport;
-      const uri = turnServer.transport === 'tls' ? 'turns' : 'turn';
-      const address = turnServer.serverAddr;
-      const port = turnServer.serverPort;
-
-      return {
-        credential: turnServer.password,
-        urls: uri.concat(':', address, ':', port, '?transport=', transport),
-        username: turnServer.username,
-      };
-    });
-  };
-
   public getConnection = (): RTCPeerConnection => {
     return this.connection;
   };

packages/ts-client/src/webrtc/tracks/Local.ts

@@ -334,10 +334,16 @@ export class Local<EndpointMetadata, TrackMetadata> {
 
   private getTrackIdToTrackBitrates = (): Record<TrackId, Bitrates> => {
     return Object.values(this.localTracks).reduce(
-      (previousValue, localTrack) => ({
-        ...previousValue,
-        [localTrack.id]: localTrack.getTrackBitrates(),
-      }),
+      (previousValue, localTrack) => {
+        const bitrates = localTrack.getTrackBitrates();
+        if (bitrates) {
+          return {
+            ...previousValue,
+            [localTrack.id]: bitrates,
+          }
+        }
+        return previousValue;
+      },
       {} as Record<TrackId, Bitrates>,
     );
   };

packages/ts-client/src/webrtc/tracks/LocalTrack.ts

@@ -243,7 +243,7 @@ export class LocalTrack<EndpointMetadata, TrackMetadata> implements TrackCommon
   private isNotSimulcastTrack = (encodings: RTCRtpEncodingParameters[]): boolean =>
     encodings.length === 1 && !encodings[0]!.rid;
 
-  public getTrackBitrates = (): Bitrates => {
+  public getTrackBitrates = (): Bitrates | undefined => {
     const trackContext = this.trackContext;
     const kind = this.trackContext.track?.kind as TrackKind | undefined;
 
@@ -255,7 +255,8 @@ export class LocalTrack<EndpointMetadata, TrackMetadata> implements TrackCommon
       return defaultBitrates[trackContext.trackKind];
     }
 
-    if (!this.sender) throw new Error(`RTCRtpSender for track ${this.id} not found`);
+    if (!this.sender)
+      return undefined;
 
     const encodings = this.sender.getParameters().encodings;
 

packages/ts-client/src/webrtc/tracks/Remote.ts

@@ -56,19 +56,18 @@ export class Remote<EndpointMetadata, TrackMetadata> {
   public addTracks = (
     endpointId: EndpointId,
     tracks: Record<TrackId, SDPTrack>,
-    trackIdToMetadata: Record<TrackId, any>,
   ) => {
     const endpoint: EndpointWithTrackContext<EndpointMetadata, TrackMetadata> | undefined =
       this.remoteEndpoints[endpointId];
 
     if (!endpoint) throw new Error(`Endpoint ${endpointId} not found`);
 
     Object.entries(tracks || {})
-      .map(([trackId, { simulcastConfig }]) => {
+      .map(([trackId, { simulcastConfig, metadata }]) => {
         const trackContext = new TrackContextImpl(
           endpoint,
           trackId,
-          trackIdToMetadata[trackId],
+          metadata,
           simulcastConfig,
           this.trackMetadataParser,
         );
@@ -115,7 +114,7 @@ export class Remote<EndpointMetadata, TrackMetadata> {
     this.updateEndpointMetadata(newEndpoint, endpoint?.metadata?.peer);
 
     this.addEndpoint(newEndpoint);
-    this.addTracks(newEndpoint.id, endpoint.tracks, endpoint.trackIdToMetadata);
+    this.addTracks(newEndpoint.id, endpoint.tracks);
 
     if (sendNotification) {
       this.emit('endpointAdded', endpoint);

packages/ts-client/src/webrtc/webRTCEndpoint.ts

@@ -20,7 +20,6 @@ import { LocalTrackManager } from './tracks/LocalTrackManager';
 import { CommandsQueue } from './CommandsQueue';
 import { Remote } from './tracks/Remote';
 import { Local } from './tracks/Local';
-import type { TurnServer } from './ConnectionManager';
 import { ConnectionManager } from './ConnectionManager';
 
 /**
@@ -197,6 +196,8 @@ export class WebRTCEndpoint<EndpointMetadata = any, TrackMetadata = any> extends
   }
 
   private handleMediaEvent = async (deserializedMediaEvent: MediaEvent) => {
+    console.log("incoming me", { deserializedMediaEvent });
+
     switch (deserializedMediaEvent.type) {
       case 'offerData': {
         await this.onOfferData(deserializedMediaEvent);
@@ -208,7 +209,7 @@ export class WebRTCEndpoint<EndpointMetadata = any, TrackMetadata = any> extends
 
         if (this.getEndpointId() === data.endpointId) return;
 
-        this.remote.addTracks(data.endpointId, data.tracks, data.trackIdToMetadata);
+        this.remote.addTracks(data.endpointId, data.tracks);
         break;
       }
       case 'tracksRemoved': {
@@ -224,9 +225,9 @@ export class WebRTCEndpoint<EndpointMetadata = any, TrackMetadata = any> extends
       }
 
       case 'sdpAnswer':
+        this.localTrackManager.ongoingRenegotiation = false;
         await this.onSdpAnswer(deserializedMediaEvent.data);
 
-        this.localTrackManager.ongoingRenegotiation = false;
         this.commandsQueue.processNextCommand();
         break;
 
@@ -297,6 +298,10 @@ export class WebRTCEndpoint<EndpointMetadata = any, TrackMetadata = any> extends
         break;
 
       case 'error':
+        console.log("signaling error", {
+          message: deserializedMediaEvent.data.message,
+        });
+
         this.emit('signalingError', {
           message: deserializedMediaEvent.data.message,
         });
@@ -335,13 +340,13 @@ export class WebRTCEndpoint<EndpointMetadata = any, TrackMetadata = any> extends
       .map((mid) => this.local.getTrackByMidOrNull(mid))
       .filter((localTrack) => localTrack !== null)
       .forEach((localTrack) => {
-        const trackContext = localTrack.trackContext;
+        const trackContext = localTrack!.trackContext;
 
         trackContext.negotiationStatus = 'done';
 
         if (trackContext.pendingMetadataUpdate) {
           const mediaEvent = generateMediaEvent('updateTrackMetadata', {
-            trackId: localTrack.id,
+            trackId: localTrack!.id,
             trackMetadata: trackContext.metadata,
           });
           this.sendMediaEvent(mediaEvent);
@@ -689,6 +694,7 @@ export class WebRTCEndpoint<EndpointMetadata = any, TrackMetadata = any> extends
 
   // todo change to private
   public sendMediaEvent = (mediaEvent: MediaEvent) => {
+    console.log("send ME", mediaEvent)
     const serializedMediaEvent = serializeMediaEvent(mediaEvent);
     this.emit('sendMediaEvent', serializedMediaEvent);
   };
@@ -698,6 +704,8 @@ export class WebRTCEndpoint<EndpointMetadata = any, TrackMetadata = any> extends
     if (!connection) return;
 
     try {
+      this.localTrackManager.updateSenders();
+
       const offer = await connection.getConnection().createOffer();
 
       if (!this.connectionManager) {
@@ -723,10 +731,8 @@ export class WebRTCEndpoint<EndpointMetadata = any, TrackMetadata = any> extends
   private onOfferData = async (offerData: MediaEvent) => {
     const connection = this.connectionManager;
 
-    if (connection) {
-      connection.getConnection().restartIce();
-    } else {
-      this.setConnection(offerData.data.integratedTurnServers);
+    if (!connection) {
+      this.setConnection();
 
       const onIceCandidate = (event: RTCPeerConnectionIceEvent) => this.onLocalCandidate(event);
       const onIceCandidateError = (event: RTCPeerConnectionIceErrorEvent) => this.onIceCandidateError(event);
@@ -753,17 +759,15 @@ export class WebRTCEndpoint<EndpointMetadata = any, TrackMetadata = any> extends
       this.local.addAllTracksToConnection();
     }
 
-    this.localTrackManager.updateSenders();
-
     const tracks = new Map<string, number>(Object.entries(offerData.data.tracksTypes));
 
     this.connectionManager?.addTransceiversIfNeeded(tracks);
 
     await this.createAndSendOffer();
   };
 
-  private setConnection = (turnServers: TurnServer[]) => {
-    this.connectionManager = new ConnectionManager(turnServers);
+  private setConnection = () => {
+    this.connectionManager = new ConnectionManager();
 
     this.localTrackManager.updateConnection(this.connectionManager);
     this.local.updateConnection(this.connectionManager);
@@ -788,6 +792,8 @@ export class WebRTCEndpoint<EndpointMetadata = any, TrackMetadata = any> extends
         data: {
           candidate: event.candidate.candidate,
           sdpMLineIndex: event.candidate.sdpMLineIndex,
+          sdpMid: event.candidate.sdpMid,
+          usernameFragment: event.candidate.usernameFragment,
         },
       });
       this.sendMediaEvent(mediaEvent);
@@ -799,6 +805,7 @@ export class WebRTCEndpoint<EndpointMetadata = any, TrackMetadata = any> extends
   };
 
   private onConnectionStateChange = (event: Event) => {
+    console.log("onConnectionStateChange", event, this.localTrackManager.connection?.getConnection().connectionState);
     switch (this.localTrackManager.connection?.getConnection().connectionState) {
       case 'failed':
         this.emit('connectionError', {
@@ -815,7 +822,7 @@ export class WebRTCEndpoint<EndpointMetadata = any, TrackMetadata = any> extends
         console.warn('ICE connection: disconnected');
         // Requesting renegotiation on ICE connection state failed fixes RTCPeerConnection
         // when the user changes their WiFi network.
-        this.sendMediaEvent(generateCustomEvent({ type: 'renegotiateTracks' }));
+        // this.sendMediaEvent(generateCustomEvent({ type: 'renegotiateTracks' }));
         break;
       case 'failed':
         this.emit('connectionError', {