fix: add delays in ray application to mitigate flakiness

Change-Id: Ibd5f789d29dd91ecd140389b367f040bde8eb99b
GoogleCloudPlatform · Aug 28, 2024 · 9208cdf · 9208cdf
1 parent ce087aa
commit 9208cdf
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/cloudbuild.yaml b/cloudbuild.yaml
@@ -102,9 +102,11 @@ steps:
 
         # Make sure pods are running
         kubectl wait --all pods -n ml-$SHORT_SHA-$_BUILD_ID-ray --for=condition=Ready --timeout=1200s
+        # Wait for pods to be stable
+        sleep 5s
         kubectl port-forward -n ml-$SHORT_SHA-$_BUILD_ID-ray service/ray-cluster-kuberay-head-svc 8265:8265 &
         # Wait port-forwarding to take its place
-        sleep 5s
+        sleep 10s
 
         ray job submit \
         --address=http://127.0.0.1:8265 -- python -c "import ray; ray.init(); print(ray.cluster_resources())"