diff --git a/.buildkite/test-e2e.yml b/.buildkite/test-e2e.yml index 921474c9d2..c736f4fa19 100644 --- a/.buildkite/test-e2e.yml +++ b/.buildkite/test-e2e.yml @@ -12,7 +12,7 @@ - IMG=kuberay/operator:nightly make deploy - kubectl wait --timeout=90s --for=condition=Available=true deployment kuberay-operator # Run e2e tests - - KUBERAY_TEST_TIMEOUT_SHORT=1m KUBERAY_TEST_TIMEOUT_MEDIUM=5m KUBERAY_TEST_TIMEOUT_LONG=10m go test -timeout 30m -v ./test/e2e/support.go ./test/e2e/raycluster_test.go ./test/e2e/rayjob_test.go ./test/e2e/rayjob_cluster_selector_test.go ./test/e2e/rayjob_lightweight_test.go ./test/e2e/rayjob_retry_test.go ./test/e2e/rayjob_suspend_test.go + - KUBERAY_TEST_TIMEOUT_SHORT=1m KUBERAY_TEST_TIMEOUT_MEDIUM=5m KUBERAY_TEST_TIMEOUT_LONG=10m go test -timeout 30m -v ./test/e2e # Printing KubeRay operator logs - kubectl logs --tail -1 -l app.kubernetes.io/name=kuberay @@ -30,7 +30,7 @@ - IMG=kuberay/operator:nightly make deploy - kubectl wait --timeout=90s --for=condition=Available=true deployment kuberay-operator # Run e2e tests - - KUBERAY_TEST_TIMEOUT_SHORT=1m KUBERAY_TEST_TIMEOUT_MEDIUM=5m KUBERAY_TEST_TIMEOUT_LONG=10m go test -timeout 30m -v ./test/e2e/rayservice_ha_test.go ./test/e2e/rayservice_in_place_update_test.go ./test/e2e/support.go + - KUBERAY_TEST_TIMEOUT_SHORT=1m KUBERAY_TEST_TIMEOUT_MEDIUM=5m KUBERAY_TEST_TIMEOUT_LONG=10m go test -timeout 30m -v ./test/e2erayservice # Printing KubeRay operator logs - kubectl logs --tail -1 -l app.kubernetes.io/name=kuberay diff --git a/ray-operator/test/e2e/rayjob_cluster_selector_test.go b/ray-operator/test/e2e/rayjob_cluster_selector_test.go index 6da28c2057..410d11db6a 100644 --- a/ray-operator/test/e2e/rayjob_cluster_selector_test.go +++ b/ray-operator/test/e2e/rayjob_cluster_selector_test.go @@ -21,7 +21,7 @@ func TestRayJobWithClusterSelector(t *testing.T) { namespace := test.NewTestNamespace() // Job scripts - jobsAC := newConfigMap(namespace.Name, "jobs", files(test, "counter.py", "fail.py")) + jobsAC := newConfigMap(namespace.Name, files(test, "counter.py", "fail.py")) jobs, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Apply(test.Ctx(), jobsAC, TestApplyOptions) g.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created ConfigMap %s/%s successfully", jobs.Namespace, jobs.Name) diff --git a/ray-operator/test/e2e/rayjob_lightweight_test.go b/ray-operator/test/e2e/rayjob_lightweight_test.go index 8a7328dc62..3516dff4e0 100644 --- a/ray-operator/test/e2e/rayjob_lightweight_test.go +++ b/ray-operator/test/e2e/rayjob_lightweight_test.go @@ -23,7 +23,7 @@ func TestRayJobLightWeightMode(t *testing.T) { namespace := test.NewTestNamespace() // Job scripts - jobsAC := newConfigMap(namespace.Name, "jobs", files(test, "counter.py", "fail.py", "stop.py")) + jobsAC := newConfigMap(namespace.Name, files(test, "counter.py", "fail.py", "stop.py")) jobs, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Apply(test.Ctx(), jobsAC, TestApplyOptions) g.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created ConfigMap %s/%s successfully", jobs.Namespace, jobs.Name) diff --git a/ray-operator/test/e2e/rayjob_retry_test.go b/ray-operator/test/e2e/rayjob_retry_test.go index c397747727..fa3a24be7d 100644 --- a/ray-operator/test/e2e/rayjob_retry_test.go +++ b/ray-operator/test/e2e/rayjob_retry_test.go @@ -21,7 +21,7 @@ func TestRayJobRetry(t *testing.T) { namespace := test.NewTestNamespace() // Job scripts - jobsAC := newConfigMap(namespace.Name, "jobs", files(test, "fail.py")) + jobsAC := newConfigMap(namespace.Name, files(test, "fail.py")) jobs, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Apply(test.Ctx(), jobsAC, TestApplyOptions) g.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created ConfigMap %s/%s successfully", jobs.Namespace, jobs.Name) diff --git a/ray-operator/test/e2e/rayjob_suspend_test.go b/ray-operator/test/e2e/rayjob_suspend_test.go index ab868cf39e..2292be42d6 100644 --- a/ray-operator/test/e2e/rayjob_suspend_test.go +++ b/ray-operator/test/e2e/rayjob_suspend_test.go @@ -23,7 +23,7 @@ func TestRayJobSuspend(t *testing.T) { namespace := test.NewTestNamespace() // Job scripts - jobsAC := newConfigMap(namespace.Name, "jobs", files(test, "long_running.py", "counter.py")) + jobsAC := newConfigMap(namespace.Name, files(test, "long_running.py", "counter.py")) jobs, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Apply(test.Ctx(), jobsAC, TestApplyOptions) g.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created ConfigMap %s/%s successfully", jobs.Namespace, jobs.Name) diff --git a/ray-operator/test/e2e/rayjob_test.go b/ray-operator/test/e2e/rayjob_test.go index 3b8885a91b..866eaaafb0 100644 --- a/ray-operator/test/e2e/rayjob_test.go +++ b/ray-operator/test/e2e/rayjob_test.go @@ -24,7 +24,7 @@ func TestRayJob(t *testing.T) { namespace := test.NewTestNamespace() // Job scripts - jobsAC := newConfigMap(namespace.Name, "jobs", files(test, "counter.py", "fail.py", "stop.py", "long_running.py")) + jobsAC := newConfigMap(namespace.Name, files(test, "counter.py", "fail.py", "stop.py", "long_running.py")) jobs, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Apply(test.Ctx(), jobsAC, TestApplyOptions) g.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created ConfigMap %s/%s successfully", jobs.Namespace, jobs.Name) diff --git a/ray-operator/test/e2e/support.go b/ray-operator/test/e2e/support.go index 3d2d94be92..13fe38afb1 100644 --- a/ray-operator/test/e2e/support.go +++ b/ray-operator/test/e2e/support.go @@ -1,9 +1,7 @@ package e2e import ( - "bytes" "embed" - "fmt" "github.com/stretchr/testify/assert" @@ -11,7 +9,6 @@ import ( "k8s.io/apimachinery/pkg/api/resource" corev1ac "k8s.io/client-go/applyconfigurations/core/v1" - rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" rayv1ac "github.com/ray-project/kuberay/ray-operator/pkg/client/applyconfiguration/ray/v1" . "github.com/ray-project/kuberay/ray-operator/test/support" ) @@ -44,8 +41,8 @@ func options[T any](options ...option[T]) option[T] { } } -func newConfigMap(namespace, name string, options ...option[corev1ac.ConfigMapApplyConfiguration]) *corev1ac.ConfigMapApplyConfiguration { - cmAC := corev1ac.ConfigMap(name, namespace). +func newConfigMap(namespace string, options ...option[corev1ac.ConfigMapApplyConfiguration]) *corev1ac.ConfigMapApplyConfiguration { + cmAC := corev1ac.ConfigMap("jobs", namespace). WithBinaryData(map[string][]byte{}). WithImmutable(true) @@ -178,95 +175,3 @@ func jobSubmitterPodTemplateApplyConfiguration() *corev1ac.PodTemplateSpecApplyC corev1.ResourceMemory: resource.MustParse("500Mi"), })))) } - -func curlRayServicePod( - t Test, - rayService *rayv1.RayService, - curlPod *corev1.Pod, - curlPodContainerName, - rayServicePath, - body string, -) (bytes.Buffer, bytes.Buffer) { - cmd := []string{ - "curl", - "-X", "POST", - "-H", "Content-Type: application/json", - fmt.Sprintf("%s-serve-svc.%s.svc.cluster.local:8000%s", rayService.Name, rayService.Namespace, rayServicePath), - "-d", body, - } - - return ExecPodCmd(t, curlPod, curlPodContainerName, cmd) -} - -func rayServiceSampleYamlApplyConfiguration() *rayv1ac.RayServiceSpecApplyConfiguration { - return rayv1ac.RayServiceSpec().WithServeConfigV2(`applications: - - name: fruit_app - import_path: fruit.deployment_graph - route_prefix: /fruit - runtime_env: - working_dir: "https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip" - deployments: - - name: MangoStand - num_replicas: 1 - user_config: - price: 3 - ray_actor_options: - num_cpus: 0.1 - - name: OrangeStand - num_replicas: 1 - user_config: - price: 2 - ray_actor_options: - num_cpus: 0.1 - - name: FruitMarket - num_replicas: 1 - ray_actor_options: - num_cpus: 0.1 - - name: math_app - import_path: conditional_dag.serve_dag - route_prefix: /calc - runtime_env: - working_dir: "https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip" - deployments: - - name: Adder - num_replicas: 1 - user_config: - increment: 3 - ray_actor_options: - num_cpus: 0.1 - - name: Multiplier - num_replicas: 1 - user_config: - factor: 5 - ray_actor_options: - num_cpus: 0.1 - - name: Router - ray_actor_options: - num_cpus: 0.1 - num_replicas: 1`). - WithRayClusterSpec(rayv1ac.RayClusterSpec(). - WithRayVersion(GetRayVersion()). - WithHeadGroupSpec(rayv1ac.HeadGroupSpec(). - WithRayStartParams(map[string]string{"dashboard-host": "0.0.0.0"}). - WithTemplate(corev1ac.PodTemplateSpec(). - WithSpec(corev1ac.PodSpec(). - WithContainers(corev1ac.Container(). - WithName("ray-head"). - WithImage(GetRayImage()). - WithPorts( - corev1ac.ContainerPort().WithName("gcs-server").WithContainerPort(6379), - corev1ac.ContainerPort().WithName("serve").WithContainerPort(8000), - corev1ac.ContainerPort().WithName("dashboard").WithContainerPort(8265), - corev1ac.ContainerPort().WithName("client").WithContainerPort(10001), - ). - WithResources(corev1ac.ResourceRequirements(). - WithRequests(corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("2"), - corev1.ResourceMemory: resource.MustParse("2Gi"), - }). - WithLimits(corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("2"), - corev1.ResourceMemory: resource.MustParse("2Gi"), - })))))), - ) -} diff --git a/ray-operator/test/e2e/locust_runner.py b/ray-operator/test/e2erayservice/locust_runner.py similarity index 100% rename from ray-operator/test/e2e/locust_runner.py rename to ray-operator/test/e2erayservice/locust_runner.py diff --git a/ray-operator/test/e2e/rayservice_ha_test.go b/ray-operator/test/e2erayservice/rayservice_ha_test.go similarity index 97% rename from ray-operator/test/e2e/rayservice_ha_test.go rename to ray-operator/test/e2erayservice/rayservice_ha_test.go index 2c6661ee63..0638007e1e 100644 --- a/ray-operator/test/e2e/rayservice_ha_test.go +++ b/ray-operator/test/e2erayservice/rayservice_ha_test.go @@ -26,7 +26,7 @@ func TestStaticRayService(t *testing.T) { namespace := test.NewTestNamespace() // Create a ConfigMap with Locust runner script - configMapAC := newConfigMap(namespace.Name, "locust-runner-script", files(test, "locust_runner.py")) + configMapAC := newConfigMap(namespace.Name, files(test, "locust_runner.py")) configMap, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Apply(test.Ctx(), configMapAC, TestApplyOptions) g.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created ConfigMap %s/%s successfully", configMap.Namespace, configMap.Name) @@ -76,7 +76,7 @@ func TestAutoscalingRayService(t *testing.T) { namespace := test.NewTestNamespace() // Create a ConfigMap with Locust runner script - configMapAC := newConfigMap(namespace.Name, "locust-runner-script", files(test, "locust_runner.py")) + configMapAC := newConfigMap(namespace.Name, files(test, "locust_runner.py")) configMap, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Apply(test.Ctx(), configMapAC, TestApplyOptions) g.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created ConfigMap %s/%s successfully", configMap.Namespace, configMap.Name) @@ -142,7 +142,7 @@ func TestRayServiceZeroDowntimeUpgrade(t *testing.T) { namespace := test.NewTestNamespace() // Create a ConfigMap with Locust runner script - configMapAC := newConfigMap(namespace.Name, "locust-runner-script", files(test, "locust_runner.py")) + configMapAC := newConfigMap(namespace.Name, files(test, "locust_runner.py")) configMap, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Apply(test.Ctx(), configMapAC, TestApplyOptions) g.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created ConfigMap %s/%s successfully", configMap.Namespace, configMap.Name) @@ -218,7 +218,7 @@ func TestRayServiceGCSFaultTolerance(t *testing.T) { namespace := test.NewTestNamespace() // Create a ConfigMap with Locust runner script - configMapAC := newConfigMap(namespace.Name, "locust-runner-script", files(test, "locust_runner.py")) + configMapAC := newConfigMap(namespace.Name, files(test, "locust_runner.py")) configMap, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Apply(test.Ctx(), configMapAC, TestApplyOptions) g.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created ConfigMap %s/%s successfully", configMap.Namespace, configMap.Name) diff --git a/ray-operator/test/e2e/rayservice_in_place_update_test.go b/ray-operator/test/e2erayservice/rayservice_in_place_update_test.go similarity index 100% rename from ray-operator/test/e2e/rayservice_in_place_update_test.go rename to ray-operator/test/e2erayservice/rayservice_in_place_update_test.go diff --git a/ray-operator/test/e2erayservice/support.go b/ray-operator/test/e2erayservice/support.go new file mode 100644 index 0000000000..f77a92f9fb --- /dev/null +++ b/ray-operator/test/e2erayservice/support.go @@ -0,0 +1,164 @@ +package e2e + +import ( + "bytes" + "embed" + "fmt" + + "github.com/stretchr/testify/assert" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + corev1ac "k8s.io/client-go/applyconfigurations/core/v1" + + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + rayv1ac "github.com/ray-project/kuberay/ray-operator/pkg/client/applyconfiguration/ray/v1" + . "github.com/ray-project/kuberay/ray-operator/test/support" +) + +//go:embed *.py +var _files embed.FS + +func ReadFile(t Test, fileName string) []byte { + t.T().Helper() + file, err := _files.ReadFile(fileName) + assert.NoError(t.T(), err) + return file +} + +type option[T any] func(t *T) *T + +func apply[T any](t *T, options ...option[T]) *T { + for _, opt := range options { + t = opt(t) + } + return t +} + +func options[T any](options ...option[T]) option[T] { + return func(t *T) *T { + for _, opt := range options { + t = opt(t) + } + return t + } +} + +func newConfigMap(namespace string, options ...option[corev1ac.ConfigMapApplyConfiguration]) *corev1ac.ConfigMapApplyConfiguration { + cmAC := corev1ac.ConfigMap("locust-runner-script", namespace). + WithBinaryData(map[string][]byte{}). + WithImmutable(true) + + return configMapWith(cmAC, options...) +} + +func configMapWith(configMapAC *corev1ac.ConfigMapApplyConfiguration, options ...option[corev1ac.ConfigMapApplyConfiguration]) *corev1ac.ConfigMapApplyConfiguration { + return apply(configMapAC, options...) +} + +func file(t Test, fileName string) option[corev1ac.ConfigMapApplyConfiguration] { + return func(cmAC *corev1ac.ConfigMapApplyConfiguration) *corev1ac.ConfigMapApplyConfiguration { + cmAC.WithBinaryData(map[string][]byte{fileName: ReadFile(t, fileName)}) + return cmAC + } +} + +func files(t Test, fileNames ...string) option[corev1ac.ConfigMapApplyConfiguration] { + var files []option[corev1ac.ConfigMapApplyConfiguration] + for _, fileName := range fileNames { + files = append(files, file(t, fileName)) + } + return options(files...) +} + +func curlRayServicePod( + t Test, + rayService *rayv1.RayService, + curlPod *corev1.Pod, + curlPodContainerName, + rayServicePath, + body string, +) (bytes.Buffer, bytes.Buffer) { + cmd := []string{ + "curl", + "-X", "POST", + "-H", "Content-Type: application/json", + fmt.Sprintf("%s-serve-svc.%s.svc.cluster.local:8000%s", rayService.Name, rayService.Namespace, rayServicePath), + "-d", body, + } + + return ExecPodCmd(t, curlPod, curlPodContainerName, cmd) +} + +func rayServiceSampleYamlApplyConfiguration() *rayv1ac.RayServiceSpecApplyConfiguration { + return rayv1ac.RayServiceSpec().WithServeConfigV2(`applications: + - name: fruit_app + import_path: fruit.deployment_graph + route_prefix: /fruit + runtime_env: + working_dir: "https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip" + deployments: + - name: MangoStand + num_replicas: 1 + user_config: + price: 3 + ray_actor_options: + num_cpus: 0.1 + - name: OrangeStand + num_replicas: 1 + user_config: + price: 2 + ray_actor_options: + num_cpus: 0.1 + - name: FruitMarket + num_replicas: 1 + ray_actor_options: + num_cpus: 0.1 + - name: math_app + import_path: conditional_dag.serve_dag + route_prefix: /calc + runtime_env: + working_dir: "https://github.com/ray-project/test_dag/archive/78b4a5da38796123d9f9ffff59bab2792a043e95.zip" + deployments: + - name: Adder + num_replicas: 1 + user_config: + increment: 3 + ray_actor_options: + num_cpus: 0.1 + - name: Multiplier + num_replicas: 1 + user_config: + factor: 5 + ray_actor_options: + num_cpus: 0.1 + - name: Router + ray_actor_options: + num_cpus: 0.1 + num_replicas: 1`). + WithRayClusterSpec(rayv1ac.RayClusterSpec(). + WithRayVersion(GetRayVersion()). + WithHeadGroupSpec(rayv1ac.HeadGroupSpec(). + WithRayStartParams(map[string]string{"dashboard-host": "0.0.0.0"}). + WithTemplate(corev1ac.PodTemplateSpec(). + WithSpec(corev1ac.PodSpec(). + WithContainers(corev1ac.Container(). + WithName("ray-head"). + WithImage(GetRayImage()). + WithPorts( + corev1ac.ContainerPort().WithName("gcs-server").WithContainerPort(6379), + corev1ac.ContainerPort().WithName("serve").WithContainerPort(8000), + corev1ac.ContainerPort().WithName("dashboard").WithContainerPort(8265), + corev1ac.ContainerPort().WithName("client").WithContainerPort(10001), + ). + WithResources(corev1ac.ResourceRequirements(). + WithRequests(corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }). + WithLimits(corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + })))))), + ) +} diff --git a/ray-operator/test/e2e/testdata/locust-cluster.burst.yaml b/ray-operator/test/e2erayservice/testdata/locust-cluster.burst.yaml similarity index 100% rename from ray-operator/test/e2e/testdata/locust-cluster.burst.yaml rename to ray-operator/test/e2erayservice/testdata/locust-cluster.burst.yaml diff --git a/ray-operator/test/e2e/testdata/locust-cluster.const-rate.yaml b/ray-operator/test/e2erayservice/testdata/locust-cluster.const-rate.yaml similarity index 100% rename from ray-operator/test/e2e/testdata/locust-cluster.const-rate.yaml rename to ray-operator/test/e2erayservice/testdata/locust-cluster.const-rate.yaml diff --git a/ray-operator/test/e2e/testdata/ray-service.ft.yaml b/ray-operator/test/e2erayservice/testdata/ray-service.ft.yaml similarity index 100% rename from ray-operator/test/e2e/testdata/ray-service.ft.yaml rename to ray-operator/test/e2erayservice/testdata/ray-service.ft.yaml diff --git a/ray-operator/test/e2e/testdata/rayservice.autoscaling.yaml b/ray-operator/test/e2erayservice/testdata/rayservice.autoscaling.yaml similarity index 100% rename from ray-operator/test/e2e/testdata/rayservice.autoscaling.yaml rename to ray-operator/test/e2erayservice/testdata/rayservice.autoscaling.yaml diff --git a/ray-operator/test/e2e/testdata/rayservice.static.yaml b/ray-operator/test/e2erayservice/testdata/rayservice.static.yaml similarity index 100% rename from ray-operator/test/e2e/testdata/rayservice.static.yaml rename to ray-operator/test/e2erayservice/testdata/rayservice.static.yaml