From 64f414e2aa6675f09d5d015a77062dfe5f9fb1b8 Mon Sep 17 00:00:00 2001 From: Aaron Liang Date: Wed, 15 Jan 2025 10:04:50 -0800 Subject: [PATCH] Add e2e test for ray create cluster command --- .../test/e2e/kubectl_ray_create_test.go | 119 ++++++++++++++++++ .../test/e2e/kubectl_ray_job_submit_test.go | 5 +- .../test/e2e/kubectl_ray_session_test.go | 2 +- 3 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 kubectl-plugin/test/e2e/kubectl_ray_create_test.go diff --git a/kubectl-plugin/test/e2e/kubectl_ray_create_test.go b/kubectl-plugin/test/e2e/kubectl_ray_create_test.go new file mode 100644 index 00000000000..002259ed491 --- /dev/null +++ b/kubectl-plugin/test/e2e/kubectl_ray_create_test.go @@ -0,0 +1,119 @@ +package e2e + +import ( + "fmt" + "os/exec" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +const ( + rayClusterName = "testing-raycluster" + rayVersion = "2.39.0" + rayImage = "rayproject/ray:2.39.0" + headCPU = "1" + headMemory = "5Gi" + workerReplica = "2" + workerCPU = "3" + workerMemory = "6Gi" +) + +var _ = Describe("Calling ray plugin `create` command to create RayCluster", Ordered, func() { + var namespace string + + // No need to create RayCluster since this test will be creating one + BeforeEach(func() { + namespace = createTestNamespace() + DeferCleanup(func() { + deleteTestNamespace(namespace) + namespace = "" + }) + }) + + It("succeeds in creating a RayCluster with minimal parameters", func() { + cmd := exec.Command("kubectl", "ray", "create", "cluster", rayClusterName, "--namespace", namespace) + output, err := cmd.CombinedOutput() + + cleanOutput := strings.TrimSpace(string(output)) + + fmt.Printf("Command Output:\n %s", cleanOutput) + Expect(err).NotTo(HaveOccurred()) + + Expect(cleanOutput).To(Equal(fmt.Sprintf("Created Ray Cluster: %s", rayClusterName))) + + // Make sure cluster is created + cmd = exec.Command("kubectl", "ray", "get", "cluster", rayClusterName, "--namespace", namespace) + output, err = cmd.CombinedOutput() + + Expect(err).NotTo(HaveOccurred()) + Expect(strings.TrimSpace(string(output))).ToNot(BeEmpty()) + + // Cleanup + cmd = exec.Command("kubectl", "delete", "raycluster", rayClusterName, "--namespace", namespace) + _, err = cmd.CombinedOutput() + Expect(err).ToNot(HaveOccurred()) + }) + + It("succeeds in creating a RayCluster with all the parameters set", func() { + cmd := exec.Command("kubectl", "ray", "create", "cluster", rayClusterName, "--namespace", namespace, "--ray-version", rayVersion, "--head-cpu", headCPU, "--head-memory", headMemory, "--worker-replicas", workerReplica, "--worker-cpu", workerCPU, "--worker-memory", workerMemory) + output, err := cmd.CombinedOutput() + + cleanOutput := strings.TrimSpace(string(output)) + + fmt.Printf("Command Output:\n %s", cleanOutput) + Expect(err).NotTo(HaveOccurred()) + + Expect(cleanOutput).To(Equal(fmt.Sprintf("Created Ray Cluster: %s", rayClusterName))) + + // Get cluster info but also check the values + cmd = exec.Command("kubectl", "get", "raycluster", rayClusterName, "--namespace", namespace, "-o", "jsonpath={.spec.rayVersion}") + output, err = cmd.CombinedOutput() + Expect(err).ToNot(HaveOccurred()) + Expect(rayVersion).To(Equal(string(output))) + + cmd = exec.Command("kubectl", "get", "raycluster", rayClusterName, "--namespace", namespace, "-o", "jsonpath={.spec.headGroupSpec.template.spec.containers[0].image}") + output, err = cmd.CombinedOutput() + Expect(err).ToNot(HaveOccurred()) + Expect(rayImage).To(Equal(string(output))) + + cmd = exec.Command("kubectl", "get", "raycluster", rayClusterName, "--namespace", namespace, "-o", "jsonpath={.spec.headGroupSpec.template.spec.containers[0].resources.limits.cpu}") + output, err = cmd.CombinedOutput() + Expect(err).ToNot(HaveOccurred()) + Expect(headCPU).To(Equal(string(output))) + + cmd = exec.Command("kubectl", "get", "raycluster", rayClusterName, "--namespace", namespace, "-o", "jsonpath={.spec.headGroupSpec.template.spec.containers[0].resources.limits.memory}") + output, err = cmd.CombinedOutput() + Expect(err).ToNot(HaveOccurred()) + Expect(headMemory).To(Equal(string(output))) + + cmd = exec.Command("kubectl", "get", "raycluster", rayClusterName, "--namespace", namespace, "-o", "jsonpath={.spec.workerGroupSpecs[0].replicas}") + output, err = cmd.CombinedOutput() + Expect(err).ToNot(HaveOccurred()) + Expect(workerReplica).To(Equal(string(output))) + + cmd = exec.Command("kubectl", "get", "raycluster", rayClusterName, "--namespace", namespace, "-o", "jsonpath={.spec.workerGroupSpecs[0].template.spec.containers[0].resources.limits.cpu}") + output, err = cmd.CombinedOutput() + Expect(err).ToNot(HaveOccurred()) + Expect(workerCPU).To(Equal(string(output))) + + cmd = exec.Command("kubectl", "get", "raycluster", rayClusterName, "--namespace", namespace, "-o", "jsonpath={.spec.workerGroupSpecs[0].template.spec.containers[0].resources.limits.memory}") + output, err = cmd.CombinedOutput() + Expect(err).ToNot(HaveOccurred()) + Expect(workerMemory).To(Equal(string(output))) + + // Cleanup + cmd = exec.Command("kubectl", "delete", "raycluster", rayClusterName, "--namespace", namespace) + _, err = cmd.CombinedOutput() + Expect(err).ToNot(HaveOccurred()) + }) + + It("should not succeed with creating RayCluster", func() { + cmd := exec.Command("kubectl", "ray", "create", "cluster", "--namespace", namespace) + output, err := cmd.CombinedOutput() + + Expect(err).To(HaveOccurred()) + Expect(string(output)).To(ContainElement("Error: cluster [CLUSTERNAME]")) + }) +}) diff --git a/kubectl-plugin/test/e2e/kubectl_ray_job_submit_test.go b/kubectl-plugin/test/e2e/kubectl_ray_job_submit_test.go index da55c468d79..0876e56ba5f 100644 --- a/kubectl-plugin/test/e2e/kubectl_ray_job_submit_test.go +++ b/kubectl-plugin/test/e2e/kubectl_ray_job_submit_test.go @@ -1,6 +1,7 @@ package e2e import ( + "fmt" "os/exec" "path" "regexp" @@ -18,7 +19,7 @@ const ( runtimeEnvSampleFileName = "runtime-env-sample.yaml" ) -var _ = Describe("Calling ray plugin `job submit` command on Ray Job", func() { +var _ = Describe("Calling ray plugin `job submit` command on Ray Job", Ordered, func() { var namespace string BeforeEach(func() { @@ -34,6 +35,7 @@ var _ = Describe("Calling ray plugin `job submit` command on Ray Job", func() { cmd := exec.Command("kubectl", "ray", "job", "submit", "--namespace", namespace, "-f", rayJobFilePath, "--working-dir", kubectlRayJobWorkingDir, "--", "python", entrypointSampleFileName) output, err := cmd.CombinedOutput() + fmt.Printf("Command Output:\n %s", string(output)) Expect(err).NotTo(HaveOccurred()) // Retrieve the Job ID from the output regexExp := regexp.MustCompile(`'([^']*raysubmit[^']*)'`) @@ -70,6 +72,7 @@ var _ = Describe("Calling ray plugin `job submit` command on Ray Job", func() { cmd := exec.Command("kubectl", "ray", "job", "submit", "--namespace", namespace, "-f", rayJobNoEnvFilePath, "--runtime-env", runtimeEnvFilePath, "--", "python", entrypointSampleFileName) output, err := cmd.CombinedOutput() + fmt.Printf("Command Output:\n %s", string(output)) Expect(err).NotTo(HaveOccurred()) // Retrieve the Job ID from the output regexExp := regexp.MustCompile(`'([^']*raysubmit[^']*)'`) diff --git a/kubectl-plugin/test/e2e/kubectl_ray_session_test.go b/kubectl-plugin/test/e2e/kubectl_ray_session_test.go index 695dc2ba450..f9a88aa9bf1 100644 --- a/kubectl-plugin/test/e2e/kubectl_ray_session_test.go +++ b/kubectl-plugin/test/e2e/kubectl_ray_session_test.go @@ -11,7 +11,7 @@ import ( . "github.com/onsi/gomega" ) -var _ = Describe("Calling ray plugin `session` command", func() { +var _ = Describe("Calling ray plugin `session` command", Ordered, func() { var namespace string BeforeEach(func() {