Reconnect to video streams on disconnect

frontend/src/api/ApiCaller.tsx

@@ -13,6 +13,7 @@ import { MissionDefinition, PlantInfo } from 'models/MissionDefinition'
 import { MissionDefinitionUpdateForm } from 'models/MissionDefinitionUpdateForm'
 import { Deck } from 'models/Deck'
 import { ApiError, isApiError } from './ApiError'
+import { MediaStreamConfig } from 'models/VideoStream'
 
 /** Implements the request sent to the backend api. */
 export class BackendAPICaller {
@@ -140,6 +141,12 @@ export class BackendAPICaller {
         return result.content
     }
 
+    static async getRobotMediaConfig(robotId: string): Promise<MediaStreamConfig> {
+        const path: string = 'media-stream/' + robotId
+        const result = await this.GET<MediaStreamConfig>(path).catch(BackendAPICaller.handleError('GET', path))
+        return result.content
+    }
+
     static async getMissionRuns(parameters: MissionRunQueryParameters): Promise<PaginatedResponse<Mission>> {
         let path: string = 'missions/runs?'
 

frontend/src/components/Contexts/MediaStreamContext.tsx

@@ -1,22 +1,19 @@
 import { createContext, FC, useContext, useEffect, useState } from 'react'
-import { SignalREventLabels, useSignalRContext } from './SignalRContext'
-import { useRobotContext } from './RobotContext'
-import {
-    ConnectionState,
-    RemoteParticipant,
-    RemoteTrack,
-    RemoteTrackPublication,
-    Room,
-    RoomEvent,
-} from 'livekit-client'
+import { ConnectionState, Room, RoomEvent } from 'livekit-client'
 import { MediaConnectionType, MediaStreamConfig } from 'models/VideoStream'
+import { BackendAPICaller } from 'api/ApiCaller'
 
 type MediaStreamDictionaryType = {
-    [robotId: string]: MediaStreamConfig & { streams: MediaStreamTrack[] }
+    [robotId: string]: { isLoading: boolean } & MediaStreamConfig & { streams: MediaStreamTrack[] }
+}
+
+type MediaStreamConfigDictionaryType = {
+    [robotId: string]: MediaStreamConfig
 }
 
 interface IMediaStreamContext {
     mediaStreams: MediaStreamDictionaryType
+    addMediaStreamConfigIfItDoesNotExist: (robotId: string) => void
 }
 
 interface Props {
@@ -25,6 +22,7 @@ interface Props {
 
 const defaultMediaStreamInterface = {
     mediaStreams: {},
+    addMediaStreamConfigIfItDoesNotExist: (robotId: string) => {},
 }
 
 const MediaStreamContext = createContext<IMediaStreamContext>(defaultMediaStreamInterface)
@@ -33,76 +31,128 @@ export const MediaStreamProvider: FC<Props> = ({ children }) => {
     const [mediaStreams, setMediaStreams] = useState<MediaStreamDictionaryType>(
         defaultMediaStreamInterface.mediaStreams
     )
-    const { registerEvent, connectionReady } = useSignalRContext()
-    const { enabledRobots } = useRobotContext()
+    const [cachedConfigs] = useState<MediaStreamConfigDictionaryType>(
+        JSON.parse(window.localStorage.getItem('mediaConfigs') ?? '{}')
+    )
+
+    useEffect(() => {
+        // Here we maintain the localstorage with the connection details
+        let updatedConfigs: MediaStreamConfigDictionaryType = {}
+        Object.keys(mediaStreams).forEach((robotId) => {
+            const conf = mediaStreams[robotId]
+
+            if (conf.streams.length === 0 && !conf.isLoading) refreshRobotMediaConfig(robotId)
+            updatedConfigs[robotId] = {
+                url: conf.url,
+                token: conf.token,
+                mediaConnectionType: conf.mediaConnectionType,
+                robotId: conf.robotId,
+            }
+        })
+        window.localStorage.setItem('mediaConfigs', JSON.stringify(updatedConfigs))
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [mediaStreams])
 
     const addTrackToConnection = (newTrack: MediaStreamTrack, robotId: string) => {
         setMediaStreams((oldStreams) => {
-            if (!Object.keys(oldStreams).includes(robotId)) {
+            if (
+                !Object.keys(oldStreams).includes(robotId) ||
+                oldStreams[robotId].streams.find((s) => s.id === newTrack.id)
+            ) {
                 return oldStreams
             } else {
-                const newStreams = { ...oldStreams }
                 return {
                     ...oldStreams,
-                    [robotId]: { ...newStreams[robotId], streams: [...oldStreams[robotId].streams, newTrack] },
+                    [robotId]: {
+                        ...oldStreams[robotId],
+                        streams: [...oldStreams[robotId].streams, newTrack],
+                        isLoading: false,
+                    },
                 }
             }
         })
     }
 
-    const createLiveKitConnection = async (config: MediaStreamConfig) => {
+    const createLiveKitConnection = async (config: MediaStreamConfig, cachedConfig: boolean = false) => {
         const room = new Room()
-        room.on(RoomEvent.TrackSubscribed, handleTrackSubscribed)
-
-        function handleTrackSubscribed(
-            track: RemoteTrack,
-            publication: RemoteTrackPublication,
-            participant: RemoteParticipant
-        ) {
-            addTrackToConnection(track.mediaStreamTrack, config.robotId)
-        }
+
+        window.addEventListener('unload', async () => room.disconnect())
+
+        room.on(RoomEvent.TrackSubscribed, (track) => addTrackToConnection(track.mediaStreamTrack, config.robotId))
+        room.on(RoomEvent.TrackUnpublished, (e) => {
+            setMediaStreams((oldStreams) => {
+                let streamsCopy = { ...oldStreams }
+                if (!Object.keys(streamsCopy).includes(config.robotId) || streamsCopy[config.robotId].isLoading)
+                    return streamsCopy
+
+                let streamList = streamsCopy[config.robotId].streams
+                const streamIndex = streamList.findIndex((s) => s.id === e.trackSid)
+
+                if (streamIndex < 0) return streamsCopy
+
+                streamList.splice(streamIndex, 1)
+                streamsCopy[config.robotId].streams = streamList
+
+                if (streamList.length === 0) room.disconnect()
+
+                return streamsCopy
+            })
+        })
+
         if (room.state === ConnectionState.Disconnected) {
             room.connect(config.url, config.token)
-                .then(() => console.log(JSON.stringify(room.state)))
-                .catch((error) => console.warn('Error connecting to LiveKit Room, may already be connected:', error))
+                .then(() => console.log('LiveKit room status: ', JSON.stringify(room.state)))
+                .catch((error) => {
+                    if (cachedConfig) refreshRobotMediaConfig(config.robotId)
+                    else console.error('Failed to connect to LiveKit room: ', error)
+                })
         }
     }
 
-    const createMediaConnection = async (config: MediaStreamConfig) => {
+    const createMediaConnection = async (config: MediaStreamConfig, cachedConfig: boolean = false) => {
         switch (config.mediaConnectionType) {
             case MediaConnectionType.LiveKit:
-                return await createLiveKitConnection(config)
+                return await createLiveKitConnection(config, cachedConfig)
             default:
                 console.error('Invalid media connection type received')
         }
         return undefined
     }
 
-    // Register a signalR event handler that listens for new media stream connections
-    useEffect(() => {
-        if (connectionReady) {
-            registerEvent(SignalREventLabels.mediaStreamConfigReceived, (username: string, message: string) => {
-                const newMediaConfig: MediaStreamConfig = JSON.parse(message)
-                setMediaStreams((oldStreams) => {
-                    if (Object.keys(oldStreams).includes(newMediaConfig.robotId)) {
-                        return oldStreams
-                    } else {
-                        createMediaConnection(newMediaConfig)
-                        return {
-                            ...oldStreams,
-                            [newMediaConfig.robotId]: { ...newMediaConfig, streams: [] },
-                        }
-                    }
-                })
-            })
+    const addConfigToMediaStreams = (conf: MediaStreamConfig, cachedConfig: boolean = false) => {
+        setMediaStreams((oldStreams) => {
+            createMediaConnection(conf, cachedConfig)
+            return {
+                ...oldStreams,
+                [conf.robotId]: { ...conf, streams: [], isLoading: true },
+            }
+        })
+    }
+
+    const addMediaStreamConfigIfItDoesNotExist = (robotId: string) => {
+        if (Object.keys(mediaStreams).includes(robotId)) {
+            const currentStream = mediaStreams[robotId]
+            if (currentStream.isLoading || currentStream.streams.find((stream) => stream.enabled)) return
+        } else if (Object.keys(cachedConfigs).includes(robotId)) {
+            const config = cachedConfigs[robotId]
+            addConfigToMediaStreams(config, true)
+            return
         }
-        // eslint-disable-next-line react-hooks/exhaustive-deps
-    }, [registerEvent, connectionReady, enabledRobots])
+
+        refreshRobotMediaConfig(robotId)
+    }
+
+    const refreshRobotMediaConfig = (robotId: string) => {
+        BackendAPICaller.getRobotMediaConfig(robotId)
+            .then((conf: MediaStreamConfig) => addConfigToMediaStreams(conf))
+            .catch((e) => console.error(e))
+    }
 
     return (
         <MediaStreamContext.Provider
             value={{
                 mediaStreams,
+                addMediaStreamConfigIfItDoesNotExist,
             }}
         >
             {children}

frontend/src/components/Pages/MissionPage/MissionPage.tsx

@@ -41,7 +41,13 @@ export const MissionPage = () => {
     const [videoMediaStreams, setVideoMediaStreams] = useState<MediaStreamTrack[]>([])
     const [selectedMission, setSelectedMission] = useState<Mission>()
     const { registerEvent, connectionReady } = useSignalRContext()
-    const { mediaStreams } = useMediaStreamContext()
+    const { mediaStreams, addMediaStreamConfigIfItDoesNotExist } = useMediaStreamContext()
+
+    useEffect(() => {
+        if (selectedMission && !Object.keys(mediaStreams).includes(selectedMission?.robot.id))
+            addMediaStreamConfigIfItDoesNotExist(selectedMission?.robot.id)
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [selectedMission])
 
     useEffect(() => {
         if (connectionReady) {

frontend/src/components/Pages/RobotPage/RobotPage.tsx

@@ -60,10 +60,15 @@ export const RobotPage = () => {
     const { setAlert, setListAlert } = useAlertContext()
     const { robotId } = useParams()
     const { enabledRobots } = useRobotContext()
-    const { mediaStreams } = useMediaStreamContext()
+    const { mediaStreams, addMediaStreamConfigIfItDoesNotExist } = useMediaStreamContext()
     const [videoMediaStreams, setVideoMediaStreams] = useState<MediaStreamTrack[]>([])
     const { ongoingMissions } = useMissionsContext()
 
+    useEffect(() => {
+        if (robotId && !Object.keys(mediaStreams).includes(robotId)) addMediaStreamConfigIfItDoesNotExist(robotId)
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [robotId])
+
     const selectedRobot = enabledRobots.find((robot) => robot.id === robotId)
 
     const [isDialogOpen, setIsDialogOpen] = useState(false)