diff --git a/ray-operator/test/sampleyaml/raycluster_test.go b/ray-operator/test/sampleyaml/raycluster_test.go index 85dfeb51a2..1596ed3926 100644 --- a/ray-operator/test/sampleyaml/raycluster_test.go +++ b/ray-operator/test/sampleyaml/raycluster_test.go @@ -106,6 +106,24 @@ func TestRayCluster(t *testing.T) { // Check that all pods can submit jobs g.Eventually(SubmitJobsToAllPods(test, rayCluster), TestTimeoutShort).Should(Succeed()) + // Suspend RayCluster + KubectlSetRayClusterSuspend(test, namespace.Name, rayCluster.Name, true) + g.Eventually(RayCluster(test, namespace.Name, rayCluster.Name), TestTimeoutMedium). + Should(WithTransform(StatusCondition(rayv1.HeadPodReady), MatchCondition(metav1.ConditionFalse, rayv1.HeadPodNotFound))) + g.Eventually(RayCluster(test, namespace.Name, rayCluster.Name), TestTimeoutMedium). + Should(WithTransform(StatusCondition(rayv1.RayClusterProvisioned), MatchCondition(metav1.ConditionFalse, rayv1.RayClusterPodsProvisioning))) + g.Eventually(RayCluster(test, namespace.Name, rayCluster.Name), TestTimeoutMedium). + Should(WithTransform(StatusCondition(rayv1.RayClusterSuspended), MatchCondition(metav1.ConditionTrue, string(rayv1.RayClusterSuspended)))) + + // Resume RayCluster + KubectlSetRayClusterSuspend(test, namespace.Name, rayCluster.Name, false) + g.Eventually(RayCluster(test, namespace.Name, rayCluster.Name), TestTimeoutMedium). + Should(WithTransform(StatusCondition(rayv1.HeadPodReady), MatchCondition(metav1.ConditionTrue, rayv1.HeadPodRunningAndReady))) + g.Eventually(RayCluster(test, namespace.Name, rayCluster.Name), TestTimeoutMedium). + Should(WithTransform(StatusCondition(rayv1.RayClusterProvisioned), MatchCondition(metav1.ConditionTrue, rayv1.AllPodRunningAndReadyFirstTime))) + g.Eventually(RayCluster(test, namespace.Name, rayCluster.Name), TestTimeoutMedium). + Should(WithTransform(StatusCondition(rayv1.RayClusterSuspended), MatchCondition(metav1.ConditionFalse, string(rayv1.RayClusterSuspended)))) + // Delete all pods after setting quota to 0 to avoid recreating pods KubectlApplyQuota(test, namespace.Name, "--hard=cpu=0,memory=0G,pods=0") KubectlDeleteAllPods(test, namespace.Name) diff --git a/ray-operator/test/support/yaml.go b/ray-operator/test/support/yaml.go index 289b96bc50..6d6e783230 100644 --- a/ray-operator/test/support/yaml.go +++ b/ray-operator/test/support/yaml.go @@ -71,3 +71,15 @@ func KubectlDeleteAllPods(t Test, namespace string) { require.NoError(t.T(), err, "Failed to delete pods in %s", namespace) t.T().Logf("Successfully delete pods in %s", namespace) } + +func KubectlSetRayClusterSuspend(t Test, namespace, raycluster string, suspend bool) { + t.T().Helper() + patch := `{"spec":{"suspend":false}}` + if suspend { + patch = `{"spec":{"suspend":true}}` + } + kubectlCmd := exec.CommandContext(t.Ctx(), "kubectl", "patch", "raycluster", raycluster, "-n", namespace, "--type=merge", "-p", patch) + err := kubectlCmd.Run() + require.NoError(t.T(), err, "Failed to set suspend=%t to %s in %s", suspend, raycluster, namespace) + t.T().Logf("Successfully set suspend=%t to %s in %s", suspend, raycluster, namespace) +}