From 81b84f2e0bebd0161ebde2b63790331732b4ed8a Mon Sep 17 00:00:00 2001
From: genlu2011 <genlu2011@gmail.com>
Date: Wed, 28 Aug 2024 13:30:15 -0700
Subject: [PATCH] fix: add delays in ray application cloud build test (#787)

fix: add delays in ray application to mitigate flakiness

Change-Id: Ibd5f789d29dd91ecd140389b367f040bde8eb99b

Co-authored-by: Gen Lu <genlu@google.com>
---
 cloudbuild.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/cloudbuild.yaml b/cloudbuild.yaml
index d3c22f362..47f4f4765 100644
--- a/cloudbuild.yaml
+++ b/cloudbuild.yaml
@@ -102,9 +102,11 @@ steps:
 
         # Make sure pods are running
         kubectl wait --all pods -n ml-$SHORT_SHA-$_BUILD_ID-ray --for=condition=Ready --timeout=1200s
+        # Wait for pods to be stable
+        sleep 5s
         kubectl port-forward -n ml-$SHORT_SHA-$_BUILD_ID-ray service/ray-cluster-kuberay-head-svc 8265:8265 &
         # Wait port-forwarding to take its place
-        sleep 5s
+        sleep 10s
 
         ray job submit \
         --address=http://127.0.0.1:8265 -- python -c "import ray; ray.init(); print(ray.cluster_resources())"