Skip to content

Commit 1c5a1b6

Browse files
nunnatsanunnatsa
authored
Produce multi-arch images (#3306)
When building images, either on release or the unstable images after PR merge, the images will be now a multi-arch manifests, for amd64, arm64 and s390x architectures. The relevant images are the operator, webhook, tests, artifacts-download and the index image. Note, the bundle image is data-only image, built from scratch, so we don't need it to be multi-arch. Signed-off-by: Nahshon Unna-Tsameret <[email protected]>
1 parent 8c84f40 commit 1c5a1b6

12 files changed

+140
-37
lines changed

‎.github/workflows/build-push-images.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -46,13 +46,13 @@ jobs:
4646
env:
4747
IMAGE_TAG: ${{ env.IMAGE_TAG }}
4848
run: |
49-
IMAGE_TAG=${IMAGE_TAG} make container-build
49+
IMAGE_TAG=${IMAGE_TAG} make build-multi-arch-images
5050
- name: Push Application Images
5151
env:
5252
IMAGE_TAG: ${{ env.IMAGE_TAG }}
5353
run: |
5454
make quay-login
55-
IMAGE_TAG=${IMAGE_TAG} make container-push
55+
IMAGE_TAG=${IMAGE_TAG} make push-multi-arch-images
5656
- name: Build Digester
5757
run: |
5858
(cd tools/digester && go build .)

‎.github/workflows/publish-community-operators.yaml

+4-4
Original file line numberDiff line numberDiff line change
@@ -48,13 +48,13 @@ jobs:
4848
env:
4949
IMAGE_TAG: ${{ env.CSV_VERSION }}
5050
run: |
51-
IMAGE_TAG=${CSV_VERSION} make container-build
51+
IMAGE_TAG=${CSV_VERSION} make build-multi-arch-images
5252
- name: Push Application Images
5353
env:
5454
IMAGE_TAG: ${{ env.CSV_VERSION }}
5555
run: |
5656
make quay-login
57-
IMAGE_TAG=${IMAGE_TAG} make container-push
57+
IMAGE_TAG=${IMAGE_TAG} make push-multi-arch-images
5858
- name: Build Digester
5959
run: |
6060
(cd tools/digester && go build .)
@@ -97,12 +97,12 @@ jobs:
9797
env:
9898
NEW_IMAGE_TAG: ${{ env.NEW_IMAGE_TAG }}
9999
run: |
100-
IMAGE_TAG=${NEW_IMAGE_TAG} make container-build
100+
IMAGE_TAG=${NEW_IMAGE_TAG} make build-multi-arch-images
101101
- name: Push next version Application Images
102102
env:
103103
NEW_IMAGE_TAG: ${{ env.NEW_IMAGE_TAG }}
104104
run: |
105-
IMAGE_TAG=${NEW_IMAGE_TAG} make container-push
105+
IMAGE_TAG=${NEW_IMAGE_TAG} make push-multi-arch-images
106106
- name: run manifest for next version
107107
env:
108108
PACKAGE_DIR: ${{ env.PACKAGE_DIR }}

‎Makefile

+52-5
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,16 @@ VIRT_ARTIFACTS_SERVER ?= $(REGISTRY_NAMESPACE)/virt-artifacts-server
1313
LDFLAGS ?= -w -s
1414
GOLANDCI_LINT_VERSION ?= v1.64.4
1515
HCO_BUMP_LEVEL ?= minor
16+
UNAME_ARCH := $(shell uname -m)
17+
ifeq ($(UNAME_ARCH),x86_64)
18+
TEMP_ARCH = amd64
19+
else ifeq ($(UNAME_ARCH),aarch64)
20+
TEMP_ARCH = arm64
21+
else
22+
TEMP_ARCH := $(UNAME_ARCH)
23+
endif
1624

25+
ARCH ?= $(TEMP_ARCH)
1726

1827
# Prow doesn't have docker command
1928
DO=./hack/in-docker.sh
@@ -87,11 +96,25 @@ hack-clean: ## Run ./hack/clean.sh
8796

8897
container-build: container-build-operator container-build-webhook container-build-operator-courier container-build-functest container-build-artifacts-server
8998

99+
build-multi-arch-images: build-multi-arch-operator-image build-multi-arch-webhook-image build-multi-arch-functest-image build-multi-arch-artifacts-server
100+
90101
container-build-operator:
91-
. "hack/cri-bin.sh" && $$CRI_BIN build -f build/Dockerfile -t $(IMAGE_REGISTRY)/$(OPERATOR_IMAGE):$(IMAGE_TAG) --build-arg git_sha=$(SHA) .
102+
. "hack/cri-bin.sh" && $$CRI_BIN build --platform=linux/$(ARCH) -f build/Dockerfile -t $(IMAGE_REGISTRY)/$(OPERATOR_IMAGE):$(IMAGE_TAG) --build-arg git_sha=$(SHA) .
103+
104+
build-multi-arch-operator-image:
105+
IMAGE_NAME=$(IMAGE_REGISTRY)/$(OPERATOR_IMAGE):$(IMAGE_TAG) SHA=SHA DOCKER_FILE=build/Dockerfile ./hack/build-multi-arch-images.sh
106+
107+
push-multi-arch-operator-image:
108+
. "hack/cri-bin.sh" && $$CRI_BIN manifest push $(IMAGE_REGISTRY)/$(OPERATOR_IMAGE):$(IMAGE_TAG)
92109

93110
container-build-webhook:
94-
. "hack/cri-bin.sh" && $$CRI_BIN build -f build/Dockerfile.webhook -t $(IMAGE_REGISTRY)/$(WEBHOOK_IMAGE):$(IMAGE_TAG) --build-arg git_sha=$(SHA) .
111+
. "hack/cri-bin.sh" && $$CRI_BIN build --platform=linux/$(ARCH) -f build/Dockerfile.webhook -t $(IMAGE_REGISTRY)/$(WEBHOOK_IMAGE):$(IMAGE_TAG) --build-arg git_sha=$(SHA) .
112+
113+
build-multi-arch-webhook-image:
114+
IMAGE_NAME=$(IMAGE_REGISTRY)/$(WEBHOOK_IMAGE):$(IMAGE_TAG) SHA=$(SHA) DOCKER_FILE="build/Dockerfile.webhook" ./hack/build-multi-arch-images.sh
115+
116+
push-multi-arch-webhook-image:
117+
. "hack/cri-bin.sh" && $$CRI_BIN manifest push $(IMAGE_REGISTRY)/$(WEBHOOK_IMAGE):$(IMAGE_TAG)
95118

96119
container-build-operator-courier:
97120
podman build -f tools/operator-courier/Dockerfile -t hco-courier .
@@ -100,13 +123,27 @@ container-build-validate-bundles:
100123
podman build -f tools/operator-sdk-validate/Dockerfile -t operator-sdk-validate-hco .
101124

102125
container-build-functest:
103-
. "hack/cri-bin.sh" && $$CRI_BIN build -f build/Dockerfile.functest -t $(IMAGE_REGISTRY)/$(FUNC_TEST_IMAGE):$(IMAGE_TAG) --build-arg git_sha=$(SHA) .
126+
. "hack/cri-bin.sh" && $$CRI_BIN build --platform=linux/$(ARCH) -f build/Dockerfile.functest -t $(IMAGE_REGISTRY)/$(FUNC_TEST_IMAGE):$(IMAGE_TAG) --build-arg git_sha=$(SHA) .
127+
128+
build-multi-arch-functest-image:
129+
IMAGE_NAME=$(IMAGE_REGISTRY)/$(FUNC_TEST_IMAGE):$(IMAGE_TAG) SHA=$(SHA) DOCKER_FILE="build/Dockerfile.functest" ./hack/build-multi-arch-images.sh
130+
131+
push-multi-arch-functest-image:
132+
. "hack/cri-bin.sh" && $$CRI_BIN manifest push $(IMAGE_REGISTRY)/$(FUNC_TEST_IMAGE):$(IMAGE_TAG)
104133

105134
container-build-artifacts-server:
106135
podman build -f build/Dockerfile.artifacts -t $(IMAGE_REGISTRY)/$(VIRT_ARTIFACTS_SERVER):$(IMAGE_TAG) --build-arg git_sha=$(SHA) .
107136

137+
build-multi-arch-artifacts-server:
138+
IMAGE_NAME=$(IMAGE_REGISTRY)/$(VIRT_ARTIFACTS_SERVER):$(IMAGE_TAG) SHA=$(SHA) DOCKER_FILE="build/Dockerfile.artifacts" ./hack/build-multi-arch-images.sh
139+
140+
push-multi-arch-artifacts-server:
141+
. "hack/cri-bin.sh" && $$CRI_BIN manifest push $(IMAGE_REGISTRY)/$(VIRT_ARTIFACTS_SERVER):$(IMAGE_TAG)
142+
108143
container-push: container-push-operator container-push-webhook container-push-functest container-push-artifacts-server
109144

145+
push-multi-arch-images: push-multi-arch-operator-image push-multi-arch-webhook-image push-multi-arch-functest-image push-multi-arch-artifacts-server
146+
110147
quay-login:
111148
podman login $(IMAGE_REGISTRY) -u $(QUAY_USERNAME) -p "$(QUAY_PASSWORD)"
112149

@@ -120,7 +157,7 @@ container-push-functest:
120157
. "hack/cri-bin.sh" && $$CRI_BIN push $$CRI_INSECURE $(IMAGE_REGISTRY)/$(FUNC_TEST_IMAGE):$(IMAGE_TAG)
121158

122159
container-push-artifacts-server:
123-
podman push $(IMAGE_REGISTRY)/$(VIRT_ARTIFACTS_SERVER):$(IMAGE_TAG)
160+
. "hack/cri-bin.sh" && $$CRI_BIN manifest push $(IMAGE_REGISTRY)/$(VIRT_ARTIFACTS_SERVER):$(IMAGE_TAG)
124161

125162
cluster-up:
126163
./cluster/up.sh
@@ -303,4 +340,14 @@ bump-hco:
303340
lint-monitoring \
304341
sanity \
305342
goimport \
306-
bump-hco
343+
bump-hco \
344+
build-multi-arch-operator-image \
345+
push-multi-arch-operator-image \
346+
build-multi-arch-webhook-image \
347+
push-multi-arch-webhook-image \
348+
build-multi-arch-functest-image \
349+
push-multi-arch-functest-image \
350+
build-multi-arch-artifacts-server \
351+
push-multi-arch-artifacts-server \
352+
build-multi-arch-images \
353+
push-multi-arch-images

‎build/Dockerfile

+7-3
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
1-
FROM docker.io/golang:1.23.4 AS builder
1+
FROM --platform=${BUILDPLATFORM} docker.io/golang:1.23.6 AS builder
22

33
WORKDIR /go/src/github.com/kubevirt/hyperconverged-cluster-operator/
44
COPY . .
5-
RUN make build-operator build-csv-merger
5+
ARG TARGETOS
6+
ARG TARGETARCH
7+
8+
RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} make build-operator build-csv-merger
69

710
FROM registry.access.redhat.com/ubi9/ubi-minimal
811
ENV OPERATOR=/usr/local/bin/hyperconverged-cluster-operator \
@@ -29,4 +32,5 @@ ARG git_sha=NONE
2932

3033
LABEL multi.GIT_URL=${git_url} \
3134
multi.GIT_SHA=${git_sha} \
32-
app=hyperconverged-cluster-operator
35+
app=hyperconverged-cluster-operator \
36+
arch=${ARCH}

‎build/Dockerfile.artifacts

+17-13
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,9 @@
1-
FROM registry.access.redhat.com/ubi9/nginx-124
2-
3-
WORKDIR /opt/app-root/src
4-
5-
COPY hack/config /tmp/config
6-
7-
USER 0
8-
RUN dnf -y install zip file && \
9-
sed -i -E '/^ +location.*$/a\ }\n\n location = /health {\n access_log off;\n return 200;' /etc/nginx/nginx.conf && \
10-
sed '/^\s*listen\s*\[::\]:8080/d' /etc/nginx/nginx.conf > /etc/nginx/nginx.conf.ipv4 && \
11-
sed '/^\s*listen\s*8080/d' /etc/nginx/nginx.conf > /etc/nginx/nginx.conf.ipv6
12-
USER 1001
1+
FROM --platform=${BUILDPLATFORM} quay.io/centos/centos:stream9 as downloader
132

143
ARG download_url=https://github.com/kubevirt/kubevirt/releases/download
154

5+
COPY hack/config /tmp/config
6+
RUN dnf -y install zip file
167
RUN eval $(cat /tmp/config |grep KUBEVIRT_VERSION=) && \
178
echo "KUBEVIRT_VERSION: $KUBEVIRT_VERSION" && \
189
for arch in amd64 arm64 s390x; do \
@@ -40,11 +31,24 @@ RUN eval $(cat /tmp/config |grep KUBEVIRT_VERSION=) && \
4031
printf "\n\n### Downloading ${url}\n"; \
4132
curl --fail -L -o virtctl${extension} "${url}" && \
4233
file virtctl${extension} && \
43-
mkdir -p ./${arch}/${l_os} && ${archive_command} ./${arch}/${l_os}/virtctl${archive_extension} virtctl${extension} && rm virtctl${extension}; \
34+
mkdir -p ./bin/${arch}/${l_os} && ${archive_command} ./bin/${arch}/${l_os}/virtctl${archive_extension} virtctl${extension} && rm virtctl${extension}; \
4435
fi; \
4536
done; \
4637
done
4738

39+
FROM --platform=${TARGETPLATFORM} registry.access.redhat.com/ubi9/nginx-124
40+
41+
WORKDIR /opt/app-root/src
42+
43+
COPY --from=downloader ./bin/ ./
44+
45+
USER 0
46+
RUN sed -i -E '/^ +location.*$/a\ }\n\n location = /health {\n access_log off;\n return 200;' /etc/nginx/nginx.conf && \
47+
sed '/^\s*listen\s*\[::\]:8080/d' /etc/nginx/nginx.conf > /etc/nginx/nginx.conf.ipv4 && \
48+
sed '/^\s*listen\s*8080/d' /etc/nginx/nginx.conf > /etc/nginx/nginx.conf.ipv6
49+
50+
USER 1001
51+
4852
ARG git_url=https://github.com/kubevirt/hyperconverged-cluster-operator.git
4953
ARG git_sha=NONE
5054

‎build/Dockerfile.functest

+7-3
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
1-
FROM docker.io/golang:1.23.4 AS builder
1+
FROM --platform=${BUILDPLATFORM} docker.io/golang:1.23.6 AS builder
22

33
WORKDIR /go/src/github.com/kubevirt/hyperconverged-cluster-operator/
44
COPY . .
5-
RUN make build-functest
5+
6+
ARG TARGETARCH
7+
8+
RUN ARCH=${TARGETARCH} make build-functest
69

710
FROM registry.access.redhat.com/ubi9/ubi-minimal
811

@@ -30,4 +33,5 @@ ARG git_sha=NONE
3033

3134
LABEL multi.GIT_URL=${git_url} \
3235
multi.GIT_SHA=${git_sha} \
33-
app=hyperconverged-cluster-functest
36+
app=hyperconverged-cluster-functest \
37+
arch=${ARCH}

‎build/Dockerfile.webhook

+8-3
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,12 @@
1-
FROM docker.io/golang:1.23.4 AS builder
1+
FROM --platform=${BUILDPLATFORM} docker.io/golang:1.23.6 AS builder
22

33
WORKDIR /go/src/github.com/kubevirt/hyperconverged-cluster-operator/
44
COPY . .
5-
RUN make build-webhook
5+
6+
ARG TARGETOS
7+
ARG TARGETARCH
8+
9+
RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} make build-webhook
610

711
FROM registry.access.redhat.com/ubi9/ubi-minimal
812
ENV WEBHOOK=/usr/local/bin/hyperconverged-cluster-webhook \
@@ -27,4 +31,5 @@ ARG git_sha=NONE
2731

2832
LABEL multi.GIT_URL=${git_url} \
2933
multi.GIT_SHA=${git_sha} \
30-
app=hyperconverged-cluster-webhook
34+
app=hyperconverged-cluster-webhook \
35+
arch=${TARGETARCH}

‎hack/build-index-image.sh

+10-3
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ BUNDLE_REGISTRY_IMAGE_NAME=${BUNDLE_REGISTRY_IMAGE_NAME:-hyperconverged-cluster-
2323
INDEX_REGISTRY_IMAGE_NAME=${INDEX_REGISTRY_IMAGE_NAME:-hyperconverged-cluster-index}
2424
OPM=${OPM:-opm}
2525
UNSTABLE=$2
26+
ARCHITECTURES="amd64 arm64 s390x"
2627

2728

2829
function create_index_image() {
@@ -48,8 +49,6 @@ function create_index_image() {
4849
BUNDLE_IMAGE_NAME=$("${PROJECT_ROOT}/tools/digester/digester" --image "${BUNDLE_IMAGE_NAME}")
4950

5051
create_file_based_catalog "${INITIAL_VERSION}" "${BUNDLE_IMAGE_NAME}" "${UNSTABLE}"
51-
52-
podman push "${INDEX_IMAGE_NAME}"
5352
}
5453

5554
function create_file_based_catalog() {
@@ -82,7 +81,15 @@ function create_file_based_catalog() {
8281
${OPM} validate fbc-catalog
8382
rm -f fbc-catalog.Dockerfile
8483
${OPM} generate dockerfile fbc-catalog
85-
podman build -t "${INDEX_IMAGE_NAME}" -f fbc-catalog.Dockerfile
84+
85+
IMAGES=
86+
for arch in ${ARCHITECTURES}; do
87+
podman build --platform="linux/${arch}" -t "${INDEX_IMAGE_NAME}-${arch}" -f fbc-catalog.Dockerfile
88+
IMAGES="${IMAGES} ${INDEX_IMAGE_NAME}-${arch}"
89+
done
90+
91+
podman manifest create "${INDEX_IMAGE_NAME}" ${IMAGES}
92+
podman manifest push "${INDEX_IMAGE_NAME}"
8693
}
8794

8895
function create_all_versions() {

‎hack/build-multi-arch-images.sh

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!/usr/bin/env bash
2+
3+
ARCHITECTURES="amd64 arm64 s390x"
4+
5+
if [[ -z ${IMAGE_NAME} ]]; then
6+
echo "IMAGE_NAME must be defined"
7+
exit 1
8+
fi
9+
10+
if [[ -z ${DOCKER_FILE} ]]; then
11+
echo "DOCKER_FILE must be defined"
12+
exit 1
13+
fi
14+
15+
SHA=$(git describe --no-match --always --abbrev=40 --dirty)
16+
17+
IMAGES=
18+
for arch in ${ARCHITECTURES}; do
19+
. "hack/cri-bin.sh" && ${CRI_BIN} build --platform=linux/${arch} -f ${DOCKER_FILE} -t "${IMAGE_NAME}-${arch}" --build-arg git_sha=${SHA} .
20+
IMAGES="${IMAGES} ${IMAGE_NAME}-${arch}"
21+
done
22+
23+
. "hack/cri-bin.sh" && ${CRI_BIN} manifest create "${IMAGE_NAME}" ${IMAGES}

‎hack/build-tests.sh

+5
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,16 @@ if [ "${JOB_TYPE}" == "travis" ]; then
1616
go test ${PKG_PACKAGE_PATH}webhooks/...
1717
go test -v -outputdir=./coverprofiles -coverprofile=cover.coverprofile ${PKG_PACKAGE_PATH}... ${CONTROLLERS_PACKAGE_PATH}...
1818
else
19+
set +u
1920
test_path="./tests/func-tests"
2021
GOFLAGS='' go install github.com/onsi/ginkgo/v2/ginkgo@$(grep github.com/onsi/ginkgo go.mod | cut -d " " -f2)
2122
go mod tidy
2223
go mod vendor
2324
test_out_path=${test_path}/_out
2425
mkdir -p ${test_out_path}
26+
27+
if [[ -n ${ARCH} ]]; then
28+
export GOARCH="${ARCH}"
29+
fi
2530
ginkgo build -o ${test_out_path} ${test_path}
2631
fi

‎hack/in-docker.sh

+3-1
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,9 @@ HCO_DIR="$(readlink -f $(dirname $0)/../)"
99
WORK_DIR="/go/src/github.com/kubevirt/hyperconverged-cluster-operator"
1010
REGISTRY=${REGISTRY:-quay.io/kubevirtci}
1111
REPOSITORY=${REPOSITORY:-hco-test-build}
12-
TAG=${TAG:-v20220510-9ff67b2}
12+
TAG=${TAG:-v20250102-3529abc}
1313
BUILD_TAG="${REGISTRY}/${REPOSITORY}:${TAG}"
14+
ARCH=${ARCH:amd64}
1415

1516
# Execute the build
1617
[ -t 1 ] && USE_TTY="-it"
@@ -20,5 +21,6 @@ $CRI_BIN run ${USE_TTY} \
2021
-e RUN_UID=$(id -u) \
2122
-e RUN_GID=$(id -g) \
2223
-e GOCACHE=/gocache \
24+
-e ARCH=${ARCH} \
2325
-w ${WORK_DIR} \
2426
${BUILD_TAG} "$1"

‎tests/func-tests/main_test.go

+2
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package tests_test
22

33
import (
44
"context"
5+
"runtime"
56
"testing"
67
"time"
78

@@ -15,6 +16,7 @@ import (
1516
)
1617

1718
func TestTests(t *testing.T) {
19+
GinkgoWriter.Printf("Start running the HCO functional tests; go version: %s; platform: %s/%s\n", runtime.Version(), runtime.GOOS, runtime.GOARCH)
1820
RegisterFailHandler(Fail)
1921
RunSpecs(t, "HyperConverged cluster E2E Test suite")
2022
}

0 commit comments

Comments
 (0)