Merge pull request #19001 from mapk-amazon/random-failure-fix

Fixes random job failures in kubernetes

lib/galaxy/jobs/runners/kubernetes.py

@@ -772,7 +772,10 @@ def check_watched_item(self, job_state):
             # as probably this means that the k8s API server hasn't
             # had time to fill in the object status since the
             # job was created only too recently.
-            if len(job.obj["status"]) == 0:
+            # It is possible that k8s didn't account for the status of the pods
+            # and they are in the uncountedTerminatedPods status. In this
+            # case we also need to wait a moment
+            if len(job.obj["status"]) == 0 or job.obj["status"].get("uncountedTerminatedPods"):
                 return job_state
             if "succeeded" in job.obj["status"]:
                 succeeded = job.obj["status"]["succeeded"]