Attempt to fix timeout issue with testing-farm

fedora-llvm-team · Jul 31, 2024 · 39e309c · 39e309c
1 parent bfb7772
commit 39e309c
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 7 deletions.
diff --git a/snapshot_manager/snapshot_manager/snapshot_manager.py b/snapshot_manager/snapshot_manager/snapshot_manager.py
@@ -340,6 +340,8 @@ def check_todays_builds(self) -> None:
             if chroot in requests:
                 request = requests[chroot]
                 watch_result, artifacts_url = request.watch()
+                if watch_result is None and artifacts_url is None:
+                    continue
 
                 html = tf.render_html(request, watch_result, artifacts_url)
                 build_status_matrix = build_status_matrix.replace(
@@ -371,7 +373,9 @@ def check_todays_builds(self) -> None:
                     issue=issue,
                     copr_build_ids=current_copr_build_ids,
                 )
-                logging.info(f"Request ID: {request.request_id}")
+                logging.info(
+                    f"testing-farm request ID for {chroot}: {request.request_id}"
+                )
                 requests[chroot] = request
                 self.github.flip_test_label(issue, chroot, in_testing)
 

diff --git a/snapshot_manager/snapshot_manager/testing_farm_util.py b/snapshot_manager/snapshot_manager/testing_farm_util.py
@@ -223,12 +223,16 @@ def watch(self) -> tuple["TestingFarmWatchResult", str]:
         request_id = sanitize_request_id(request_id=self.request_id)
         cmd = f"testing-farm watch --no-wait --id {self.request_id}"
         # We ignore the exit code because in case of a test error, 1 is the exit code
-        _, stdout, stderr = util.run_cmd(cmd=cmd, timeout_secs=20)
-        watch_result, artifacts_url = TestingFarmWatchResult.from_output(stdout)
-        if watch_result is None:
-            raise SystemError(
-                f"failed to watch 'testing-farm request': {cmd}\n\nstdout: {stdout}\n\nstderr: {stderr}"
-            )
+        try:
+            _, stdout, stderr = util.run_cmd(cmd=cmd, timeout_secs=20)
+            watch_result, artifacts_url = TestingFarmWatchResult.from_output(stdout)
+            if watch_result is None:
+                raise SystemError(
+                    f"failed to watch 'testing-farm request': {cmd}\n\nstdout: {stdout}\n\nstderr: {stderr}"
+                )
+        except Exception as ex:
+            logging.warn(f"failed to watch for testing-farm result {ex}")
+            return (None, None)
         return (watch_result, artifacts_url)
 
     @classmethod