[RayCluster][CI] add e2e tests for the RayClusterSuspended status con…

…dition

Signed-off-by: Rueian <[email protected]>

ray-operator/test/sampleyaml/raycluster_test.go

@@ -106,6 +106,24 @@ func TestRayCluster(t *testing.T) {
 			// Check that all pods can submit jobs
 			g.Eventually(SubmitJobsToAllPods(test, rayCluster), TestTimeoutShort).Should(Succeed())
 
+			// Suspend RayCluster
+			KubectlSetRayClusterSuspend(test, namespace.Name, rayCluster.Name, true)
+			g.Eventually(RayCluster(test, namespace.Name, rayCluster.Name), TestTimeoutMedium).
+				Should(WithTransform(StatusCondition(rayv1.HeadPodReady), MatchCondition(metav1.ConditionFalse, rayv1.HeadPodNotFound)))
+			g.Eventually(RayCluster(test, namespace.Name, rayCluster.Name), TestTimeoutMedium).
+				Should(WithTransform(StatusCondition(rayv1.RayClusterProvisioned), MatchCondition(metav1.ConditionFalse, rayv1.RayClusterPodsProvisioning)))
+			g.Eventually(RayCluster(test, namespace.Name, rayCluster.Name), TestTimeoutMedium).
+				Should(WithTransform(StatusCondition(rayv1.RayClusterSuspended), MatchCondition(metav1.ConditionTrue, string(rayv1.RayClusterSuspended))))
+
+			// Resume RayCluster
+			KubectlSetRayClusterSuspend(test, namespace.Name, rayCluster.Name, false)
+			g.Eventually(RayCluster(test, namespace.Name, rayCluster.Name), TestTimeoutMedium).
+				Should(WithTransform(StatusCondition(rayv1.HeadPodReady), MatchCondition(metav1.ConditionTrue, rayv1.HeadPodRunningAndReady)))
+			g.Eventually(RayCluster(test, namespace.Name, rayCluster.Name), TestTimeoutMedium).
+				Should(WithTransform(StatusCondition(rayv1.RayClusterProvisioned), MatchCondition(metav1.ConditionTrue, rayv1.AllPodRunningAndReadyFirstTime)))
+			g.Eventually(RayCluster(test, namespace.Name, rayCluster.Name), TestTimeoutMedium).
+				Should(WithTransform(StatusCondition(rayv1.RayClusterSuspended), MatchCondition(metav1.ConditionFalse, string(rayv1.RayClusterSuspended))))
+
 			// Delete all pods after setting quota to 0 to avoid recreating pods
 			KubectlApplyQuota(test, namespace.Name, "--hard=cpu=0,memory=0G,pods=0")
 			KubectlDeleteAllPods(test, namespace.Name)

ray-operator/test/support/yaml.go

@@ -71,3 +71,15 @@ func KubectlDeleteAllPods(t Test, namespace string) {
 	require.NoError(t.T(), err, "Failed to delete pods in %s", namespace)
 	t.T().Logf("Successfully delete pods in %s", namespace)
 }
+
+func KubectlSetRayClusterSuspend(t Test, namespace, raycluster string, suspend bool) {
+	t.T().Helper()
+	patch := `{"spec":{"suspend":false}}`
+	if suspend {
+		patch = `{"spec":{"suspend":true}}`
+	}
+	kubectlCmd := exec.CommandContext(t.Ctx(), "kubectl", "patch", "raycluster", raycluster, "-n", namespace, "--type=merge", "-p", patch)
+	err := kubectlCmd.Run()
+	require.NoError(t.T(), err, "Failed to set suspend=%t to %s in %s", suspend, raycluster, namespace)
+	t.T().Logf("Successfully set suspend=%t to %s in %s", suspend, raycluster, namespace)
+}