Skip to content

Commit 49e7520

Browse files
seanlaiiseanlaii
authored
[Feature] Provide multi-arch images for apiserver and security proxy (#4131)
* [Feature] Provide multi-arch image build and push for apiserver and security proxy Signed-off-by: wei-chenglai <[email protected]> * build on test and push only on master branch * fix lint --------- Signed-off-by: wei-chenglai <[email protected]>
1 parent 74860ff commit 49e7520

File tree

8 files changed

+178
-46
lines changed

8 files changed

+178
-46
lines changed

‎.github/workflows/image-release.yaml‎

Lines changed: 36 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -52,27 +52,50 @@ jobs:
5252
run: go test ./pkg/... ./cmd/... -race -parallel 4
5353
working-directory: ${{env.working-directory}}
5454

55-
- name: Set up Docker
56-
uses: docker/setup-docker-action@v4
57-
58-
- name: Build Docker Image - Apiserver
59-
run: |
60-
docker build -t kuberay/apiserver:${{ steps.vars.outputs.sha_short }} -f apiserver/Dockerfile .
61-
docker save -o /tmp/apiserver.tar kuberay/apiserver:${{ steps.vars.outputs.sha_short }}
62-
6355
- name: Log in to Quay.io
6456
uses: docker/login-action@v2
6557
with:
6658
registry: quay.io
6759
username: ${{ secrets.QUAY_USERNAME }}
6860
password: ${{ secrets.QUAY_ROBOT_TOKEN }}
6961

70-
- name: Push Apiserver to Quay.io
62+
# Build apiserver binaries inside the gh runner vm directly and then copy the go binaries to docker images using the Dockerfile.buildx
63+
- name: Build linux/amd64 apiserver go binary
64+
env:
65+
CGO_ENABLED: 0
66+
GOOS: linux
67+
GOARCH: amd64
68+
run: |
69+
CGO_ENABLED=$CGO_ENABLED GOOS=$GOOS GOARCH=$GOARCH go build -a -o kuberay-apiserver-$GOARCH cmd/main.go
70+
working-directory: ${{env.working-directory}}
71+
72+
- name: Build linux/arm64 apiserver binary
73+
env:
74+
CGO_ENABLED: 0
75+
GOOS: linux
76+
GOARCH: arm64
7177
run: |
72-
docker image tag kuberay/apiserver:${{ steps.vars.outputs.sha_short }} quay.io/kuberay/apiserver:${{ steps.vars.outputs.sha_short }};
73-
docker push quay.io/kuberay/apiserver:${{ steps.vars.outputs.sha_short }};
74-
docker image tag kuberay/apiserver:${{ steps.vars.outputs.sha_short }} quay.io/kuberay/apiserver:${{ env.tag }};
75-
docker push quay.io/kuberay/apiserver:${{ env.tag }}
78+
CGO_ENABLED=$CGO_ENABLED GOOS=$GOOS GOARCH=$GOARCH go build -a -o kuberay-apiserver-$GOARCH cmd/main.go
79+
working-directory: ${{env.working-directory}}
80+
81+
- name: Set up Docker Buildx
82+
uses: docker/setup-buildx-action@v3
83+
84+
- name: Build MultiArch Image
85+
uses: docker/build-push-action@v5
86+
env:
87+
PUSH: true
88+
REPO_ORG: kuberay
89+
REPO_NAME: apiserver
90+
with:
91+
platforms: linux/amd64,linux/arm64
92+
context: .
93+
file: ./apiserver/Dockerfile.buildx
94+
push: ${{env.PUSH}}
95+
provenance: false
96+
tags: |
97+
quay.io/${{env.REPO_ORG}}/${{env.REPO_NAME}}:${{ steps.vars.outputs.sha_short }}
98+
quay.io/${{env.REPO_ORG}}/${{env.REPO_NAME}}:${{ env.tag }}
7699
77100
release_operator_image:
78101
env:
@@ -122,14 +145,6 @@ jobs:
122145
run: make test
123146
working-directory: ${{env.working-directory}}
124147

125-
- name: Set up Docker
126-
uses: docker/setup-docker-action@v4
127-
128-
- name: Build Docker Image - Operator
129-
run: |
130-
IMG=kuberay/operator:${{ steps.vars.outputs.sha_short }} make docker-image
131-
working-directory: ${{env.working-directory}}
132-
133148
- name: Log in to Quay.io
134149
uses: docker/login-action@v2
135150
with:
@@ -215,9 +230,6 @@ jobs:
215230
id: vars
216231
run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
217232

218-
- name: Set up Docker
219-
uses: docker/setup-docker-action@v4
220-
221233
- name: Log in to Quay.io
222234
uses: docker/login-action@v2
223235
with:
@@ -273,14 +285,6 @@ jobs:
273285
run: go mod download
274286
working-directory: ${{env.working-directory}}
275287

276-
- name: Set up Docker
277-
uses: docker/setup-docker-action@v4
278-
279-
- name: Build Docker Image - Submitter
280-
run: |
281-
IMG=kuberay/submitter:${{ steps.vars.outputs.sha_short }} make docker-image-rayjob-submitter
282-
working-directory: ${{env.working-directory}}
283-
284288
- name: Log in to Quay.io
285289
uses: docker/login-action@v2
286290
with:

‎.github/workflows/test-job.yaml‎

Lines changed: 70 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -97,13 +97,42 @@ jobs:
9797
password: ${{ secrets.QUAY_ROBOT_TOKEN }}
9898
if: contains(fromJson('["refs/heads/master"]'), github.ref)
9999

100-
- name: Push Apiserver to Quay.io
100+
# Build apiserver binaries inside the gh runner vm directly and then copy the go binaries to docker images using the Dockerfile.buildx
101+
- name: Build linux/amd64 apiserver go binary
102+
env:
103+
CGO_ENABLED: 0
104+
GOOS: linux
105+
GOARCH: amd64
101106
run: |
102-
docker image tag kuberay/apiserver:${{ steps.vars.outputs.sha_short }} quay.io/kuberay/apiserver:${{ steps.vars.outputs.sha_short }};
103-
docker push quay.io/kuberay/apiserver:${{ steps.vars.outputs.sha_short }};
104-
docker image tag kuberay/apiserver:${{ steps.vars.outputs.sha_short }} quay.io/kuberay/apiserver:nightly;
105-
docker push quay.io/kuberay/apiserver:nightly
106-
if: contains(fromJson('["refs/heads/master"]'), github.ref)
107+
CGO_ENABLED=$CGO_ENABLED GOOS=$GOOS GOARCH=$GOARCH go build -a -o kuberay-apiserver-$GOARCH cmd/main.go
108+
working-directory: ${{env.working-directory}}
109+
110+
- name: Build linux/arm64 apiserver binary
111+
env:
112+
CGO_ENABLED: 0
113+
GOOS: linux
114+
GOARCH: arm64
115+
run: |
116+
CGO_ENABLED=$CGO_ENABLED GOOS=$GOOS GOARCH=$GOARCH go build -a -o kuberay-apiserver-$GOARCH cmd/main.go
117+
working-directory: ${{env.working-directory}}
118+
119+
- name: Set up Docker Buildx
120+
uses: docker/setup-buildx-action@v3
121+
122+
- name: Build MultiArch Docker Image and Push on Merge
123+
uses: docker/build-push-action@v5
124+
env:
125+
REPO_ORG: kuberay
126+
REPO_NAME: apiserver
127+
with:
128+
platforms: linux/amd64,linux/arm64
129+
context: .
130+
file: ./apiserver/Dockerfile.buildx
131+
push: ${{ contains(fromJson('["refs/heads/master"]'), github.ref) }}
132+
provenance: false
133+
tags: |
134+
quay.io/${{env.REPO_ORG}}/${{env.REPO_NAME}}:${{ steps.vars.outputs.sha_short }}
135+
quay.io/${{env.REPO_ORG}}/${{env.REPO_NAME}}:nightly
107136
108137
build_apiserversdk:
109138
env:
@@ -199,13 +228,42 @@ jobs:
199228
password: ${{ secrets.QUAY_ROBOT_TOKEN }}
200229
if: contains(fromJson('["refs/heads/master"]'), github.ref)
201230

202-
- name: Push security proxy to Quay.io
231+
# Build security proxy binaries inside the gh runner vm directly and then copy the go binaries to docker images using the Dockerfile.buildx
232+
- name: Build linux/amd64 security proxy go binary
233+
env:
234+
CGO_ENABLED: 0
235+
GOOS: linux
236+
GOARCH: amd64
203237
run: |
204-
docker image tag kuberay/security-proxy:${{ steps.vars.outputs.sha_short }} quay.io/kuberay/security-proxy:${{ steps.vars.outputs.sha_short }};
205-
docker push quay.io/kuberay/security-proxy:${{ steps.vars.outputs.sha_short }};
206-
docker image tag kuberay/security-proxy:${{ steps.vars.outputs.sha_short }} quay.io/kuberay/security-proxy:nightly;
207-
docker push quay.io/kuberay/security-proxy:nightly
208-
if: contains(fromJson('["refs/heads/master"]'), github.ref)
238+
CGO_ENABLED=$CGO_ENABLED GOOS=$GOOS GOARCH=$GOARCH go build -a -o security_proxy-$GOARCH cmd/main.go
239+
working-directory: ${{env.working-directory}}
240+
241+
- name: Build linux/arm64 security proxy binary
242+
env:
243+
CGO_ENABLED: 0
244+
GOOS: linux
245+
GOARCH: arm64
246+
run: |
247+
CGO_ENABLED=$CGO_ENABLED GOOS=$GOOS GOARCH=$GOARCH go build -a -o security_proxy-$GOARCH cmd/main.go
248+
working-directory: ${{env.working-directory}}
249+
250+
- name: Set up Docker Buildx
251+
uses: docker/setup-buildx-action@v3
252+
253+
- name: Build MultiArch Docker Image and Push on Merge
254+
uses: docker/build-push-action@v5
255+
env:
256+
REPO_ORG: kuberay
257+
REPO_NAME: security-proxy
258+
with:
259+
platforms: linux/amd64,linux/arm64
260+
context: .
261+
file: ./experimental/Dockerfile.buildx
262+
push: ${{ contains(fromJson('["refs/heads/master"]'), github.ref) }}
263+
provenance: false
264+
tags: |
265+
quay.io/${{env.REPO_ORG}}/${{env.REPO_NAME}}:${{ steps.vars.outputs.sha_short }}
266+
quay.io/${{env.REPO_ORG}}/${{env.REPO_NAME}}:nightly
209267
210268
build_operator:
211269
env:

‎apiserver/DEVELOPMENT.md‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,46 @@ Access the service at `localhost:8888` for http, and `localhost:8887` for the RP
156156
make docker-image
157157
```
158158

159+
#### Build Multi-Architecture Image
160+
161+
The API server supports building multi-architecture images for `linux/amd64` and `linux/arm64` platforms. This is useful for deploying on ARM-based systems like Apple Silicon.
162+
163+
**Prerequisites:**
164+
165+
* Docker Buildx installed and configured
166+
167+
**Build Process:**
168+
169+
The multi-arch build follows a similar pattern to the ray-operator:
170+
171+
**Build binaries for each architecture:**
172+
173+
```bash
174+
# Build for amd64
175+
cd apiserver
176+
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o kuberay-apiserver-amd64 cmd/main.go
177+
178+
# Build for arm64
179+
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -a -o kuberay-apiserver-arm64 cmd/main.go
180+
```
181+
182+
**Build and push multi-arch image using Dockerfile.buildx:**
183+
184+
```bash
185+
# From the project root directory
186+
docker buildx build --platform linux/amd64,linux/arm64 \
187+
-t your-registry/apiserver:your-tag \
188+
-f apiserver/Dockerfile.buildx \
189+
--push .
190+
```
191+
192+
**Note:**
193+
194+
* The multi-arch image uses `apiserver/Dockerfile.buildx` which is optimized for copying pre-built binaries.
195+
* Since the API server is built with `CGO_ENABLED=0`, no cross-compilation tools (like gcc-aarch64-linux-gnu) are needed, making the build process simpler than the operator.
196+
* The build context must be the project root directory (`.`) because the `proto/` directory needs to be copied for serving swagger files at runtime.
197+
* Multi-arch images are automatically built and pushed to `quay.io/kuberay/apiserver:nightly` on merges to the `master` branch via the GitHub Actions workflow.
198+
159199
#### Start Kubernetes Deployment
160200

161201
Note that you should make your KubeRay API server image available by either pushing it to an image registry, such as DockerHub or Quay, or by loading the image into the Kubernetes cluster. If you are using a Kind cluster for development, you can run `make load-image` to load the newly built API server image into the Kind cluster. The operator image will also be needed to be loaded on your cluster. If you want run secure API server, you can build security proxy using `make security-proxy-image` and load it to the cluster using `make load-security-proxy-image`

‎apiserver/Dockerfile.buildx‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
FROM scratch
2+
ARG TARGETARCH
3+
WORKDIR /workspace
4+
COPY ./apiserver/kuberay-apiserver-${TARGETARCH} apiserver/kuberay-apiserver
5+
COPY proto/ proto/
6+
USER 65532:65532
7+
8+
ENTRYPOINT ["/workspace/apiserver/kuberay-apiserver"]

‎apiserver/Makefile‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,12 @@ docker-image: test ## Build image for the api server.
188188
docker-push: ## Push image for the api server.
189189
$(ENGINE) push ${IMG}
190190

191+
docker-multi-arch-build: test ## Build multi arch image, amd64 and arm64 currently.
192+
$(ENGINE) buildx build --platform linux/amd64,linux/arm64 -t ${IMG} -f Dockerfile.buildx ..
193+
194+
docker-multi-arch-push: test ## Build and Push multi arch image, amd64 and arm64 currently.
195+
$(ENGINE) buildx build --push --platform linux/amd64,linux/arm64 -t ${IMG} -f Dockerfile.buildx ..
196+
191197
##@ Deployment
192198
.PHONY: install
193199
install: kustomize docker-image load-image ## Install the kuberay api server without security to the K8s cluster specified in ~/.kube/config.

‎experimental/Dockerfile.buildx‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
FROM scratch
2+
ARG TARGETARCH
3+
WORKDIR /workspace
4+
COPY ./experimental/security_proxy-${TARGETARCH} /usr/local/bin/security_proxy
5+
USER 65532:65532
6+
7+
ENTRYPOINT ["/usr/local/bin/security_proxy"]

‎experimental/Makefile‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,12 @@ docker-image: ## Build image for the security proxy.
7070
docker-push: ## Push image for the api server.
7171
$(ENGINE) push ${IMG}
7272

73+
docker-multi-arch-build: ## Build multi arch image, amd64 and arm64 currently.
74+
$(ENGINE) buildx build --platform linux/amd64,linux/arm64 -t ${IMG} -f Dockerfile.buildx ..
75+
76+
docker-multi-arch-push: ## Build and Push multi arch image, amd64 and arm64 currently.
77+
$(ENGINE) buildx build --push --platform linux/amd64,linux/arm64 -t ${IMG} -f Dockerfile.buildx ..
78+
7379
##@ Development Tools Setup
7480

7581
## Location to install dependencies to

‎ray-operator/Makefile‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -125,8 +125,11 @@ docker-build: build docker-image ## Build image with the manager.
125125
docker-push: ## Push image with the manager.
126126
${ENGINE} push ${IMG}
127127

128-
docker-multi-arch-image: # Build and push multi arch image, amd64 and arm64 currently
129-
${ENGINE} buildx build --push --platform linux/amd64,linux/arm64/v8 -t ${IMG} .
128+
docker-multi-arch-build: # Build multi arch image, amd64 and arm64 currently
129+
${ENGINE} buildx build --platform linux/amd64,linux/arm64 -t ${IMG} .
130+
131+
docker-multi-arch-push: ## Build and Push multi arch image, amd64 and arm64 currently
132+
${ENGINE} buildx build --push --platform linux/amd64,linux/arm64 -t ${IMG} .
130133

131134
docker-image-rayjob-submitter:
132135
${ENGINE} build -t ${IMG} --target submitter .

0 commit comments

Comments
 (0)