Adjust crawler pvc on exit code 3 (out of storage) (#2375)

crawler 1.5.0 now has an exit code 3 for when crawler is actually out of
disk space. The operator should handle this by immediately adjusting the
PVC size.

Ideally, crawler will be improved to avoid this, but since this can
still happen, operator should be able to respond and fix the issue.

backend/btrixcloud/operator/crawls.py

@@ -1381,6 +1381,7 @@ async def update_crawl_state(
         )
 
         for key, value in sizes.items():
+            increase_storage = False
             value = int(value)
             if value > 0 and status.podStatus:
                 pod_info = status.podStatus[key]
@@ -1393,15 +1394,21 @@ async def update_crawl_state(
                     and pod_info.used.storage * self.min_avail_storage_ratio
                     > pod_info.allocated.storage
                 ):
-                    new_storage = math.ceil(
-                        pod_info.used.storage
-                        * self.min_avail_storage_ratio
-                        / 1_000_000_000
-                    )
-                    pod_info.newStorage = f"{new_storage}Gi"
-                    print(
-                        f"Attempting to adjust storage to {pod_info.newStorage} for {key}"
-                    )
+                    increase_storage = True
+
+            # out of storage
+            if pod_info.isNewExit and pod_info.exitCode == 3:
+                pod_info.used.storage = pod_info.allocated.storage
+                increase_storage = True
+
+            if increase_storage:
+                new_storage = math.ceil(
+                    pod_info.used.storage * self.min_avail_storage_ratio / 1_000_000_000
+                )
+                pod_info.newStorage = f"{new_storage}Gi"
+                print(
+                    f"Attempting to adjust storage to {pod_info.newStorage} for {key}"
+                )
 
         if not status.stopReason:
             status.stopReason = await self.is_crawl_stopping(crawl, status, data)