Skip to content

Commit

Permalink
[Enhancement] GPU RayCluster doesn't work on GKE Autopilot (#1470)
Browse files Browse the repository at this point in the history
[Enhancement] GPU RayCluster doesn't work on GKE Autopilot
  • Loading branch information
kevin85421 authored Oct 6, 2023
1 parent 3a1fedb commit 0a56cd4
Show file tree
Hide file tree
Showing 2 changed files with 15 additions and 4 deletions.
17 changes: 15 additions & 2 deletions ray-operator/controllers/ray/common/pod.go
Original file line number Diff line number Diff line change
Expand Up @@ -185,8 +185,21 @@ func DefaultWorkerPodTemplate(instance rayv1alpha1.RayCluster, workerSpec rayv1a
// For more details, please refer to: https://docs.ray.io/en/latest/ray-core/configure.html#tls-authentication.
Env: deepCopyRayContainer.Env,
VolumeMounts: deepCopyRayContainer.VolumeMounts,
// If users specify ResourceQuota for the namespace, the init container need to specify resource explicitly.
Resources: deepCopyRayContainer.Resources,
// If users specify a ResourceQuota for the namespace, the init container needs to specify resources explicitly.
// GKE's Autopilot does not support GPU-using init containers, so we explicitly specify the resources for the
// init container instead of reusing the resources of the Ray container.
Resources: v1.ResourceRequirements{
// The init container's resource consumption remains constant, as it solely sends requests to check the GCS status at a fixed frequency.
// Therefore, hard-coding the resources is acceptable.
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("200m"),
v1.ResourceMemory: resource.MustParse("256Mi"),
},
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("200m"),
v1.ResourceMemory: resource.MustParse("256Mi"),
},
},
}
podTemplate.Spec.InitContainers = append(podTemplate.Spec.InitContainers, initContainer)
}
Expand Down
2 changes: 0 additions & 2 deletions ray-operator/controllers/ray/common/pod_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -903,9 +903,7 @@ func TestDefaultInitContainer(t *testing.T) {
}
}

// The values of `Resources` should be the same in both Ray container and health-check container.
assert.NotEmpty(t, rayContainer.Resources, "The test only makes sense if the Ray container has resource limit/request.")
assert.Equal(t, rayContainer.Resources, healthCheckContainer.Resources)
}

func TestDefaultInitContainerImagePullPolicy(t *testing.T) {
Expand Down

0 comments on commit 0a56cd4

Please sign in to comment.