From 0a56cd4f04920ae3ee3cedb7885e9812ac139be9 Mon Sep 17 00:00:00 2001 From: Kai-Hsun Chen Date: Fri, 6 Oct 2023 15:50:12 -0700 Subject: [PATCH] [Enhancement] GPU RayCluster doesn't work on GKE Autopilot (#1470) [Enhancement] GPU RayCluster doesn't work on GKE Autopilot --- ray-operator/controllers/ray/common/pod.go | 17 +++++++++++++++-- ray-operator/controllers/ray/common/pod_test.go | 2 -- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/ray-operator/controllers/ray/common/pod.go b/ray-operator/controllers/ray/common/pod.go index a1f33a3868..8f6e928e54 100644 --- a/ray-operator/controllers/ray/common/pod.go +++ b/ray-operator/controllers/ray/common/pod.go @@ -185,8 +185,21 @@ func DefaultWorkerPodTemplate(instance rayv1alpha1.RayCluster, workerSpec rayv1a // For more details, please refer to: https://docs.ray.io/en/latest/ray-core/configure.html#tls-authentication. Env: deepCopyRayContainer.Env, VolumeMounts: deepCopyRayContainer.VolumeMounts, - // If users specify ResourceQuota for the namespace, the init container need to specify resource explicitly. - Resources: deepCopyRayContainer.Resources, + // If users specify a ResourceQuota for the namespace, the init container needs to specify resources explicitly. + // GKE's Autopilot does not support GPU-using init containers, so we explicitly specify the resources for the + // init container instead of reusing the resources of the Ray container. + Resources: v1.ResourceRequirements{ + // The init container's resource consumption remains constant, as it solely sends requests to check the GCS status at a fixed frequency. + // Therefore, hard-coding the resources is acceptable. + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("200m"), + v1.ResourceMemory: resource.MustParse("256Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("200m"), + v1.ResourceMemory: resource.MustParse("256Mi"), + }, + }, } podTemplate.Spec.InitContainers = append(podTemplate.Spec.InitContainers, initContainer) } diff --git a/ray-operator/controllers/ray/common/pod_test.go b/ray-operator/controllers/ray/common/pod_test.go index 204d9db53e..9f18381ad0 100644 --- a/ray-operator/controllers/ray/common/pod_test.go +++ b/ray-operator/controllers/ray/common/pod_test.go @@ -903,9 +903,7 @@ func TestDefaultInitContainer(t *testing.T) { } } - // The values of `Resources` should be the same in both Ray container and health-check container. assert.NotEmpty(t, rayContainer.Resources, "The test only makes sense if the Ray container has resource limit/request.") - assert.Equal(t, rayContainer.Resources, healthCheckContainer.Resources) } func TestDefaultInitContainerImagePullPolicy(t *testing.T) {