chore: improves timeout of waitForContainer (#195)

### What does this PR do?

* Adds a long timeout (30 minutes) for waiting for the container to
  build
* Adds retries in case the container dissapears

### Screenshot / video of UI

<!-- If this PR is changing UI, please include
screenshots or screencasts showing the difference -->

N/A

### What issues does this PR fix or reference?

<!-- Include any related issues from Podman Desktop
repository (or from another issue tracker). -->

Closes #194

### How to test this PR?

<!-- Please explain steps to reproduce -->

Covered by tests

Signed-off-by: Charlie Drage <[email protected]>

packages/backend/src/container-utils.spec.ts

@@ -88,6 +88,27 @@ test('waitForContainerToExit should wait for container to exit', async () => {
   await expect(waitForContainerToExit('1234')).resolves.toBeUndefined();
 });
 
+test('test waitForContainerToExit should throw error if container does not exit', async () => {
+  const listContainersMock = vi.fn();
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  (extensionApi.containerEngine as any).listContainers = listContainersMock;
+  listContainersMock.mockResolvedValue([]);
+
+  // Exit with a quick timeout of 1 retry
+  await expect(waitForContainerToExit('1234', 1)).rejects.toThrow('Container not found after maximum retries');
+});
+
+test('Check that listContainers was called 2 times with a retry count of 2 if container was not found', async () => {
+  const listContainersMock = vi.fn();
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  (extensionApi.containerEngine as any).listContainers = listContainersMock;
+  listContainersMock.mockResolvedValue([]);
+
+  // Exit with a quick timeout of 2 retries
+  await expect(waitForContainerToExit('1234', 2)).rejects.toThrow('Container not found after maximum retries');
+  expect(listContainersMock).toBeCalledTimes(2);
+});
+
 // Test removeContainerIfExists
 test('removeContainerIfExists should remove existing container', async () => {
   const listContainersMock = vi.fn();

packages/backend/src/container-utils.ts

@@ -121,31 +121,65 @@ export async function createAndStartContainer(engineId: string, options: Contain
     throw new Error('There was an error creating the container: ' + e);
   }
 }
-/*
-Wait for the container to exit, if it exits with a non-zero exit code, throw an error
-TODO: Add timeout?
-*/
-export async function waitForContainerToExit(containerId: string): Promise<void> {
+
+export async function waitForContainerToExit(
+  containerId: string,
+  maxRetryCount: number = 5,
+  timeoutMinutes: number = 10,
+): Promise<void> {
+  let retryCount = 0;
   let containerRunning = true;
+  const timeout = timeoutMinutes * 60 * 1000; // change to minutes
+
+  const timeoutPromise = new Promise((_, reject) =>
+    setTimeout(() => reject(new Error(`Timeout after ${timeoutMinutes} minutes`)), timeout),
+  );
 
   while (containerRunning) {
     console.log('Waiting for container to exit: ', containerId);
-    await extensionApi.containerEngine.listContainers().then(containers => {
-      containers.forEach(container => {
-        if (container.Id === containerId && container.State === 'exited') {
-          // Let's stop the loop if the container has exited / stopped
-          containerRunning = false;
-          // Container.status reports "ex. Exited (1) Less than a second ago" when it
-          // errors out, and Exited (0) when it succeeds. So we check for that.
-          if (!container.Status.includes('Exited (0)')) {
-            throw new Error('There was an error with the build, the container exited with a non-zero exit code.');
+
+    await Promise.race([
+      (async () => {
+        const containers = await extensionApi.containerEngine.listContainers();
+        let containerFound = false;
+
+        containers.forEach(container => {
+          if (container.Id === containerId) {
+            containerFound = true;
+            if (container.State === 'exited') {
+              // Stop the loop if the container has exited/stopped
+              containerRunning = false;
+              // Check for non-zero exit code
+              if (!container.Status.includes('Exited (0)')) {
+                throw new Error('Container exited with a non-zero exit code.');
+              }
+            }
           }
+        });
+
+        // Retry in case the container has been lost for any reason, such as the engine being restarted / network issues, etc.
+        // container being recreated.
+        if (!containerFound) {
+          retryCount++;
+          if (retryCount >= maxRetryCount) {
+            throw new Error('Container not found after maximum retries.');
+          }
+          // If container not found, wait a bit before retrying
+          console.log(`Container ${containerId} not found, retrying... (${retryCount}/${maxRetryCount})`);
+          await new Promise(r => setTimeout(r, 1000));
         }
-      });
+      })(),
+      timeoutPromise,
+    ]).catch((error: unknown) => {
+      // Handle both timeout and other errors
+      containerRunning = false; // Stop trying to wait for the container
+      throw error;
     });
 
-    // Check every second
-    await new Promise(r => setTimeout(r, 1000));
+    if (containerRunning) {
+      // If still running and not timed out or hit max retries, check again after 1 second
+      await new Promise(r => setTimeout(r, 1000));
+    }
   }
 }