adding retry strategy in KurtosisUpgrader

enclave-manager/web/packages/app/src/emui/enclaves/components/KurtosisUpgrader.tsx

@@ -1,4 +1,5 @@
 import { Alert, AlertDescription, AlertIcon, Box, Button, Flex } from "@chakra-ui/react";
+import { GetCloudInstanceConfigResponse } from "enclave-manager-sdk/build/kurtosis_backend_server_api_pb";
 import { PropsWithChildren, useEffect, useState } from "react";
 import { FiDownloadCloud, FiRotateCcw } from "react-icons/fi";
 import { GoBug } from "react-icons/go";
@@ -74,7 +75,8 @@ export const KurtosisUpgrader = () => {
 
     const interval = setInterval(async () => {
       try {
-        const getCloudInstanceConfigResponse = await kurtosisClient.getCloudInstanceConfig(skipCache);
+        // Calling it with retries because when the engine is restarted it won't return a response for a few seconds
+        const getCloudInstanceConfigResponse = await getCloudInstanceConfigWithRetry(5, 2000);
         const instanceStatus = getCloudInstanceConfigResponse.status;
         if (instanceStatus === "running") {
           setUpgradeStatus(UpgradeStatus.SUCCESS);
@@ -90,6 +92,31 @@ export const KurtosisUpgrader = () => {
     return () => clearInterval(interval);
   }, [upgradeStatus, kurtosisClient]);
 
+  const getCloudInstanceConfigWithRetry = async (
+    tries: number,
+    interval: number,
+  ): Promise<GetCloudInstanceConfigResponse> => {
+    let getCloudInstanceConfigResponse: GetCloudInstanceConfigResponse = new GetCloudInstanceConfigResponse();
+    try {
+      getCloudInstanceConfigResponse = await kurtosisClient.getCloudInstanceConfig(skipCache);
+    } catch (e) {
+      const newTries = tries - 1;
+
+      if (newTries === 0) {
+        throw e;
+      }
+
+      await wait(interval);
+
+      return getCloudInstanceConfigWithRetry(newTries, interval);
+    }
+    return getCloudInstanceConfigResponse;
+  };
+
+  const wait = (ms: number) => {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  };
+
   if (upgradeStatus === UpgradeStatus.NONE) {
     return null;
   }