From 392e09c135ccd45dfb277c0837290702e053f9ef Mon Sep 17 00:00:00 2001
From: Gen Lu <genlu@google.com>
Date: Thu, 29 Aug 2024 18:36:23 +0000
Subject: [PATCH] fix: increase wait time for ray head

Change-Id: Iedd59e8a78dd1fc5a53b1e3f72e0774e9004cde0
---
 cloudbuild.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/cloudbuild.yaml b/cloudbuild.yaml
index 47f4f4765..d17194b82 100644
--- a/cloudbuild.yaml
+++ b/cloudbuild.yaml
@@ -102,8 +102,8 @@ steps:
 
         # Make sure pods are running
         kubectl wait --all pods -n ml-$SHORT_SHA-$_BUILD_ID-ray --for=condition=Ready --timeout=1200s
-        # Wait for pods to be stable
-        sleep 5s
+        # Ray head's readinessProbe is not probing the head service today. Therefore the wait for ready above is not reliable.
+        sleep 45s
         kubectl port-forward -n ml-$SHORT_SHA-$_BUILD_ID-ray service/ray-cluster-kuberay-head-svc 8265:8265 &
         # Wait port-forwarding to take its place
         sleep 10s
@@ -231,6 +231,8 @@ steps:
 
         # Validate Ray: Make sure pods are running
         kubectl wait --for=condition=Ready pods -n rag-$SHORT_SHA-$_BUILD_ID -l 'component!=continuous-image-puller' --timeout=1200s
+        # Ray head's readinessProbe is not probing the head service today. Therefore the wait for ready above is not reliable.
+        sleep 45s
         kubectl port-forward -n rag-$SHORT_SHA-$_BUILD_ID service/ray-cluster-kuberay-head-svc 8262:8265 &
         # Wait port-forwarding to take its place
         sleep 5s