test: check RayService ray job submit can successfully create a job

Signed-off-by: win5923 <[email protected]>

ray-operator/config/samples/ray-service.high-availability.yaml

@@ -1,76 +1,3 @@
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: redis-config
-  labels:
-    app: redis
-data:
-  redis.conf: |-
-    dir /data
-    port 6379
-    bind 0.0.0.0
-    appendonly yes
-    protected-mode no
-    requirepass 5241590000000000
-    pidfile /data/redis-6379.pid
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: redis
-  labels:
-    app: redis
-spec:
-  type: ClusterIP
-  ports:
-    - name: redis
-      port: 6379
-  selector:
-    app: redis
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: redis
-  labels:
-    app: redis
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: redis
-  template:
-    metadata:
-      labels:
-        app: redis
-    spec:
-      containers:
-        - name: redis
-          image: redis:5.0.8
-          command:
-            - "sh"
-            - "-c"
-            - "redis-server /usr/local/etc/redis/redis.conf"
-          ports:
-            - containerPort: 6379
-          volumeMounts:
-            - name: config
-              mountPath: /usr/local/etc/redis/redis.conf
-              subPath: redis.conf
-      volumes:
-        - name: config
-          configMap:
-            name: redis-config
----
-# Redis password
-apiVersion: v1
-kind: Secret
-metadata:
-  name: redis-password-secret
-type: Opaque
-data:
-  # echo -n "5241590000000000" | base64
-  password: NTI0MTU5MDAwMDAwMDAwMA==
 ---
 apiVersion: ray.io/v1
 kind: RayService
@@ -225,3 +152,77 @@ spec:
         items:
           - key: query.py
             path: query.py
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: redis-config
+  labels:
+    app: redis
+data:
+  redis.conf: |-
+    dir /data
+    port 6379
+    bind 0.0.0.0
+    appendonly yes
+    protected-mode no
+    requirepass 5241590000000000
+    pidfile /data/redis-6379.pid
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: redis
+  labels:
+    app: redis
+spec:
+  type: ClusterIP
+  ports:
+    - name: redis
+      port: 6379
+  selector:
+    app: redis
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: redis
+  labels:
+    app: redis
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: redis
+  template:
+    metadata:
+      labels:
+        app: redis
+    spec:
+      containers:
+        - name: redis
+          image: redis:5.0.8
+          command:
+            - "sh"
+            - "-c"
+            - "redis-server /usr/local/etc/redis/redis.conf"
+          ports:
+            - containerPort: 6379
+          volumeMounts:
+            - name: config
+              mountPath: /usr/local/etc/redis/redis.conf
+              subPath: redis.conf
+      volumes:
+        - name: config
+          configMap:
+            name: redis-config
+---
+# Redis password
+apiVersion: v1
+kind: Secret
+metadata:
+  name: redis-password-secret
+type: Opaque
+data:
+  # echo -n "5241590000000000" | base64
+  password: NTI0MTU5MDAwMDAwMDAwMA==

ray-operator/test/sampleyaml/rayservice_test.go

@@ -0,0 +1,76 @@
+package sampleyaml
+
+import (
+	"testing"
+
+	. "github.com/onsi/gomega"
+
+	rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
+	. "github.com/ray-project/kuberay/ray-operator/test/support"
+)
+
+func TestRayService(t *testing.T) {
+	tests := []struct {
+		name string
+	}{
+		{
+			name: "ray-service.custom-serve-service.yaml",
+		},
+		{
+			name: "ray-service.different-port.yaml",
+		},
+		{
+			name: "ray-service.high-availability.yaml",
+		},
+		{
+			name: "ray-service.sample.yaml",
+		},
+		{
+			name: "ray-service.text-ml.yaml",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			test := With(t)
+			g := NewWithT(t)
+
+			namespace := test.NewTestNamespace()
+			test.StreamKubeRayOperatorLogs()
+			rayServiceFromYaml := DeserializeRayServiceSampleYAML(test, tt.name)
+			KubectlApplyYAML(test, tt.name, namespace.Name)
+
+			rayService, err := GetRayService(test, namespace.Name, rayServiceFromYaml.Name)
+			g.Expect(err).NotTo(HaveOccurred())
+			g.Expect(rayService).NotTo(BeNil())
+
+			var rayClusterName string
+			// Wait for RayCluster name to be populated
+			g.Eventually(func() string {
+				rs, err := GetRayService(test, namespace.Name, rayServiceFromYaml.Name)
+				g.Expect(err).NotTo(HaveOccurred())
+				if rs.Status.PendingServiceStatus.RayClusterName != "" {
+					rayClusterName = rs.Status.PendingServiceStatus.RayClusterName
+				} else if rs.Status.ActiveServiceStatus.RayClusterName != "" {
+					rayClusterName = rs.Status.ActiveServiceStatus.RayClusterName
+				}
+				return rayClusterName
+			}, TestTimeoutShort).ShouldNot(BeEmpty())
+
+			test.T().Logf("Waiting for RayCluster %s/%s to be ready", namespace.Name, rayClusterName)
+			g.Eventually(RayCluster(test, namespace.Name, rayClusterName), TestTimeoutMedium).
+				Should(WithTransform(RayClusterState, Equal(rayv1.Ready)))
+			rayCluster, err := GetRayCluster(test, namespace.Name, rayClusterName)
+			g.Expect(err).NotTo(HaveOccurred())
+
+			// // Check if the head pod is ready
+			g.Eventually(HeadPod(test, rayCluster), TestTimeoutShort).Should(WithTransform(IsPodRunningAndReady, BeTrue()))
+
+			// // Check if all worker pods are ready
+			g.Eventually(WorkerPods(test, rayCluster), TestTimeoutShort).Should(WithTransform(AllPodsRunningAndReady, BeTrue()))
+
+			// // Check that all pods can submit jobs
+			g.Eventually(SubmitJobsToAllPods(test, rayCluster), TestTimeoutShort).Should(Succeed())
+		})
+	}
+}

ray-operator/test/sampleyaml/support.go

@@ -44,6 +44,16 @@ func DeserializeRayClusterSampleYAML(t Test, filename string) *rayv1.RayCluster
 	return rayCluster
 }
 
+func DeserializeRayServiceSampleYAML(t Test, filename string) *rayv1.RayService {
+	t.T().Helper()
+	yamlFileContent := readYAML(t, filename)
+	decoder := rayscheme.Codecs.UniversalDecoder()
+	rayService := &rayv1.RayService{}
+	_, _, err := decoder.Decode(yamlFileContent, nil, rayService)
+	assert.NoError(t.T(), err)
+	return rayService
+}
+
 func KubectlApplyYAML(t Test, filename string, namespace string) {
 	t.T().Helper()
 	sampleYAMLDir := getSampleYAMLDir(t)

ray-operator/test/support/ray.go

@@ -59,6 +59,10 @@ func GetRayCluster(t Test, namespace, name string) (*rayv1.RayCluster, error) {
 	return t.Client().Ray().RayV1().RayClusters(namespace).Get(t.Ctx(), name, metav1.GetOptions{})
 }
 
+func GetRayService(t Test, namespace, name string) (*rayv1.RayService, error) {
+	return t.Client().Ray().RayV1().RayServices(namespace).Get(t.Ctx(), name, metav1.GetOptions{})
+}
+
 func RayClusterState(cluster *rayv1.RayCluster) rayv1.ClusterState {
 	return cluster.Status.State //nolint:staticcheck // https://github.com/ray-project/kuberay/pull/2288
 }