Merge pull request #375 from boostcampwm-2024/fix/#367/stream

[Fix] Safari에서 비디오 스트림을 전송할 때 다른 브라우저에서 스트림 정보가 오지 않던 오류 수정
boostcampwm-2024 · Dec 5, 2024 · d6fcdf0 · d6fcdf0
2 parents ffab126 + b66ff06
commit d6fcdf0
Show file tree

Hide file tree

Showing 8 changed files with 70 additions and 48 deletions.
diff --git a/README.md b/README.md
@@ -147,6 +147,7 @@
 
 ## 프론트엔드
 
+<h4><a href='https://simeunseo.notion.site/5d4f6f4ab2124d19940e21e19ac3f04b?pvs=4'>🔗 네트워크 상태에 따른 스트림 품질 변경</a></h4>
 <h4><a href='https://simeunseo.notion.site/14a599a6f0d2800cb40add7d399525aa?pvs=4'>🔗 스트리밍을 최적화 해보자</a></h4>
 
 > 최대한 많은 유저가 들어와 화상 서비스를 이용하는 것을 목적으로 하는 만큼 소켓 이벤트, 기존 할당된 자원에 대한 관리를 진행하며 최대한 적은 자원으로 나은 환경을 제공하기 위해 최적화를 진행하였습니다.

diff --git a/apps/media/src/mediasoup/config.ts b/apps/media/src/mediasoup/config.ts
@@ -29,10 +29,22 @@ export class MediasoupConfig {
           useinbandfec: 1,
         },
       },
+      {
+        kind: 'video',
+        mimeType: 'video/H264',
+        clockRate: 90000,
+        parameters: {
+          'packetization-mode': 1,
+          'profile-level-id': '42e01f',
+        },
+      },
       {
         kind: 'video',
         mimeType: 'video/VP8',
         clockRate: 90000,
+        parameters: {
+          'x-google-start-bitrate': 10000,
+        },
       },
     ] as RtpCodecCapability[],
   };

diff --git a/apps/media/src/mediasoup/mediasoup.service.ts b/apps/media/src/mediasoup/mediasoup.service.ts
@@ -113,10 +113,6 @@ export class MediasoupService implements OnModuleInit {
     const peer = room.getPeer(socketId);
     const transport = peer.getTransport(transportId);
 
-    if (appData.mediaTypes !== 'audio') {
-      rtpParameters.encodings = server.PRODUCER_OPTIONS.encodings;
-    }
-
     const producer = await transport.produce({
       kind,
       rtpParameters,
@@ -269,6 +265,8 @@ export class MediasoupService implements OnModuleInit {
     const peer = room.peers.get(socketId);
     const consumer = peer.getConsumer(consumerId);
 
+    if (!consumer) return;
+
     consumer?.pause();
 
     return { paused: true, consumerId, producerId: consumer.producerId };
@@ -279,6 +277,8 @@ export class MediasoupService implements OnModuleInit {
     const peer = room.peers.get(socketId);
     const consumer = peer.getConsumer(consumerId);
 
+    if (!consumer) return;
+
     if (consumer?.producerPaused) {
       return { paused: true, consumerId, producerId: consumer.producerId };
     }
@@ -289,11 +289,15 @@ export class MediasoupService implements OnModuleInit {
   }
 
   pauseConsumers(socketId: string, roomId: string, consumerIds: string[]) {
-    return consumerIds.map((consumerId) => this.pauseConsumer(socketId, consumerId, roomId));
+    return consumerIds
+      .map((consumerId) => this.pauseConsumer(socketId, consumerId, roomId))
+      .filter(Boolean);
   }
 
   resumeConsumers(socketId: string, roomId: string, consumerIds: string[]) {
-    return consumerIds.map((consumerId) => this.resumeConsumer(socketId, consumerId, roomId));
+    return consumerIds
+      .map((consumerId) => this.resumeConsumer(socketId, consumerId, roomId))
+      .filter(Boolean);
   }
 
   changeConsumerPreferredLayers(
@@ -307,10 +311,9 @@ export class MediasoupService implements OnModuleInit {
 
       const consumer = peer.getConsumer(consumerId);
 
-      consumer?.setPreferredLayers({
-        spatialLayer: networkQuality,
-        temporalLayer: networkQuality,
-      });
+      if (!consumer || consumer.closed || consumer.paused) return;
+
+      consumer.setPreferredLayers({ spatialLayer: networkQuality });
     });
   }
 

diff --git a/apps/web/src/hooks/mediasoup/useNetworkMonitor.ts b/apps/web/src/hooks/mediasoup/useNetworkMonitor.ts
@@ -13,7 +13,7 @@ const QUALITY_LEVEL = {
   },
   poor: {
     quality: 0,
-    options: { packetLossRate: 5, jitter: 30, frameDropRate: 10, averageRTT: 300, nackCount: 50 },
+    options: { packetLossRate: 10, jitter: 30, frameDropRate: 10, averageRTT: 300, nackCount: 50 },
   },
 } as const;
 
@@ -89,12 +89,12 @@ const useNetworkMonitor = ({ streams }: UseNetworkMonitorProps) => {
     []
   );
 
-  const checkNetworkQuality = async (streams: client.RemoteStream[]) => {
+  const checkNetworkQualities = async (streams: client.RemoteStream[]) => {
     const networkQualities = await Promise.all(
       streams.map(async (data) => {
         const { consumer } = data;
 
-        if (!consumer) return;
+        if (!consumer || consumer.closed || consumer.paused) return;
 
         let networkQuality = 2; // 0: poor, 1: average, 2: good
 
@@ -131,7 +131,15 @@ const useNetworkMonitor = ({ streams }: UseNetworkMonitorProps) => {
       })
     );
 
-    return networkQualities;
+    return networkQualities.filter(Boolean).reduce(
+      (acc, cur) => {
+        if (!cur) return acc;
+        if (acc.some((data) => data.consumerId === cur.consumerId)) return acc;
+
+        return [...acc, cur];
+      },
+      [] as { consumerId: string; networkQuality: number }[]
+    );
   };
 
   useEffect(() => {
@@ -142,7 +150,9 @@ const useNetworkMonitor = ({ streams }: UseNetworkMonitorProps) => {
     const interval = setInterval(async () => {
       const notPausedStreams = getNotPausedStreams(streams);
 
-      const networkQualities = await checkNetworkQuality(notPausedStreams);
+      const networkQualities = await checkNetworkQualities(notPausedStreams);
+
+      if (!networkQualities || !networkQualities.length) return;
 
       socket.emit(SOCKET_EVENTS.changeConsumerPreferredLayers, {
         roomId: ticleId,

diff --git a/apps/web/src/hooks/mediasoup/useRemoteStream.ts b/apps/web/src/hooks/mediasoup/useRemoteStream.ts
@@ -249,7 +249,7 @@ const useRemoteStream = () => {
       const newStreams = [...prevStreams];
       const stream = newStreams.find((stream) => stream.consumer?.producerId === producerId);
 
-      if (!stream) {
+      if (!stream || stream.consumer?.closed) {
         return prevStreams;
       }
 
@@ -280,7 +280,7 @@ const useRemoteStream = () => {
         const newStreams = [...prevStreams];
         const stream = newStreams.find((stream) => stream.consumer?.producerId === producerId);
 
-        if (!stream) {
+        if (!stream || stream.consumer?.closed) {
           return prevStreams;
         }
 

diff --git a/apps/web/src/hooks/useMediaTracks.ts b/apps/web/src/hooks/useMediaTracks.ts
@@ -12,7 +12,7 @@ const DEFAULT_LOCAL_STREAM = {
 
 const getMediaDevices = (kind: MediaDeviceKind, devices: MediaDeviceInfo[]) => {
   return devices
-    .filter((device) => device.kind === kind && device.deviceId && device.deviceId !== 'default')
+    .filter((device) => device.kind === kind && device.deviceId)
     .map((device) => ({ label: device.label, value: device.deviceId }));
 };
 
@@ -116,16 +116,22 @@ const useMediaTracks = () => {
         setAudioDevices(audioInputs);
         setAudioOutputDevices(audioOutputs);
 
-        if (videoInputs[0]) setSelectedVideoDeviceId(videoInputs[0].value);
-        if (audioInputs[0]) setSelectedAudioDeviceId(audioInputs[0].value);
-        if (audioOutputs[0]) setSelectedAudioOutputDeviceId(audioOutputs[0].value);
+        if (videoInputs[0] && !selectedVideoDeviceId) {
+          setSelectedVideoDeviceId(videoInputs[0].value);
+        }
+        if (audioInputs[0] && !selectedAudioDeviceId) {
+          setSelectedAudioDeviceId(audioInputs[0].value);
+        }
+        if (audioOutputs[0] && !selectedAudioOutputDeviceId) {
+          setSelectedAudioOutputDeviceId(audioOutputs[0].value);
+        }
       } catch (_) {
         toast('미디어 정보를 가져올 수 없습니다.');
       }
     };
 
     fetchMediaDevices();
-  }, []);
+  }, [video, audio]);
 
   return {
     video,

diff --git a/packages/mediasoup/src/client/index.ts b/packages/mediasoup/src/client/index.ts
@@ -65,25 +65,26 @@ export interface ResumeConsumersRes {
   paused: boolean;
 }
 
-export const PRODUCER_OPTIONS: ProducerOptions = {
-  encodings: [
-    { rid: 'r0', maxBitrate: 50000, scalabilityMode: 'S1T3' },
-    { rid: 'r1', maxBitrate: 150000, scalabilityMode: 'S1T3' },
-    { rid: 'r2', maxBitrate: 500000, scalabilityMode: 'S1T3' },
-  ],
-  codecOptions: {
-    videoGoogleStartBitrate: 1000,
-  },
-};
-
 export const VIDEO_PRODUCER_OPTIONS: ProducerOptions = {
   encodings: [
-    { rid: 'r0', maxBitrate: 50000, scalabilityMode: 'S1T3' },
-    { rid: 'r1', maxBitrate: 150000, scalabilityMode: 'S1T3' },
-    { rid: 'r2', maxBitrate: 500000, scalabilityMode: 'S1T3' },
+    {
+      rid: 'r0',
+      maxBitrate: 750000,
+      maxFramerate: 30,
+    },
+    {
+      rid: 'r1',
+      maxBitrate: 2000000,
+      maxFramerate: 30,
+    },
+    {
+      rid: 'r2',
+      maxBitrate: 3500000,
+      maxFramerate: 30,
+    },
   ],
   codecOptions: {
-    videoGoogleStartBitrate: 1000,
+    videoGoogleStartBitrate: 100000,
     opusDtx: true,
   },
 };

diff --git a/packages/mediasoup/src/server/index.ts b/packages/mediasoup/src/server/index.ts
@@ -77,14 +77,3 @@ export interface ChangeConsumerPreferredLayersDto {
   roomId: string;
   networkQualities: NetworkQualityDto[];
 }
-
-export const PRODUCER_OPTIONS = {
-  encodings: [
-    { rid: 'r0', maxBitrate: 50000, scalabilityMode: 'S1T3', active: true, dtx: false },
-    { rid: 'r1', maxBitrate: 150000, scalabilityMode: 'S1T3', active: true, dtx: false },
-    { rid: 'r2', maxBitrate: 500000, scalabilityMode: 'S1T3', active: true, dtx: false },
-  ],
-  codecOptions: {
-    videoGoogleStartBitrate: 1000,
-  },
-};