From 1fe7a21b10dda46b90559ced7cb9ce9a05ac7531 Mon Sep 17 00:00:00 2001 From: Kai-Hsun Chen Date: Wed, 1 Nov 2023 23:44:46 -0700 Subject: [PATCH] [Feature] Improve observability for flaky RayJob test (#1587) --- ray-operator/controllers/ray/rayjob_controller.go | 3 ++- ray-operator/controllers/ray/rayjob_controller_test.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ray-operator/controllers/ray/rayjob_controller.go b/ray-operator/controllers/ray/rayjob_controller.go index fc689a89a82..1f4cff589f3 100644 --- a/ray-operator/controllers/ray/rayjob_controller.go +++ b/ray-operator/controllers/ray/rayjob_controller.go @@ -537,12 +537,13 @@ func (r *RayJobReconciler) shouldUpdateJobStatus(oldJobStatus rayv1.JobStatus, o // make sure the priority is correct func (r *RayJobReconciler) updateState(ctx context.Context, rayJob *rayv1.RayJob, jobInfo *utils.RayJobInfo, jobStatus rayv1.JobStatus, jobDeploymentStatus rayv1.JobDeploymentStatus, err error) error { + r.Log.Info("UpdateState", "oldJobStatus", rayJob.Status.JobStatus, "newJobStatus", jobStatus, "oldJobDeploymentStatus", rayJob.Status.JobDeploymentStatus, "newJobDeploymentStatus", jobDeploymentStatus) + // Let's skip update the APIServer if it's synced. if rayJob.Status.JobStatus == jobStatus && rayJob.Status.JobDeploymentStatus == jobDeploymentStatus { return nil } - r.Log.Info("UpdateState", "oldJobStatus", rayJob.Status.JobStatus, "newJobStatus", jobStatus, "oldJobDeploymentStatus", rayJob.Status.JobDeploymentStatus, "newJobDeploymentStatus", jobDeploymentStatus) rayJob.Status.JobStatus = jobStatus rayJob.Status.JobDeploymentStatus = jobDeploymentStatus if jobInfo != nil { diff --git a/ray-operator/controllers/ray/rayjob_controller_test.go b/ray-operator/controllers/ray/rayjob_controller_test.go index 1bac798bd7c..52c1cc9906d 100644 --- a/ray-operator/controllers/ray/rayjob_controller_test.go +++ b/ray-operator/controllers/ray/rayjob_controller_test.go @@ -238,7 +238,7 @@ var _ = Context("Inside the default namespace", func() { It("Dashboard URL should be set", func() { Eventually( getDashboardURLForRayJob(ctx, myRayJob), - time.Second*3, time.Millisecond*500).Should(HavePrefix(myRayJob.Name), "Dashboard URL = %v", myRayJob.Status.DashboardURL) + time.Second*1, time.Millisecond*500).Should(HavePrefix(myRayJob.Name), "Dashboard URL = %v", myRayJob.Status.DashboardURL) }) It("test cluster selector", func() {