Produce multi arch images (#3321)

nunnatsa · web-flow · commit 511d4eaf3cd9 · 2025-03-02T10:09:27.000+01:00
On release, produce a manifest image + images for 3 architectures:
* AMD64
* ARM64
* s390x

Signed-off-by: Nahshon Unna-Tsameret &lt;nunnatsa@redhat.com&gt;
diff --git a/.github/workflows/build-push-images.yaml b/.github/workflows/build-push-images.yaml
@@ -22,14 +22,16 @@ jobs:
       REGISTRY_NAMESPACE: kubevirt
       OPM_VERSION: v1.47.0
     steps:
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
       - name: Checkout the latest code
         uses: actions/checkout@v4
         with:
           ref: ${{ github.ref }}
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: "1.22"
+          go-version-file: go.mod
       - name: Get latest version
         run: |
           PACKAGE_DIR="./deploy/olm-catalog/community-kubevirt-hyperconverged"
@@ -42,17 +44,12 @@ jobs:
           echo "IMAGE_TAG=${CSV_VERSION}-unstable" >> $GITHUB_ENV
           echo "UNSTABLE=UNSTABLE" >> $GITHUB_ENV
 
-      - name: Build Applications Images
-        env:
-          IMAGE_TAG: ${{ env.IMAGE_TAG }}
-        run: |
-          IMAGE_TAG=${IMAGE_TAG} make container-build
-      - name: Push Application Images
+      - name: Build and Push Applications Images
         env:
           IMAGE_TAG: ${{ env.IMAGE_TAG }}
         run: |
           make quay-login
-          IMAGE_TAG=${IMAGE_TAG} make container-push
+          IMAGE_TAG=${IMAGE_TAG} make build-push-multi-arch-images
       - name: Build Digester
         run: |
           (cd tools/digester && go build .)
diff --git a/.github/workflows/pr-sanity.yaml b/.github/workflows/pr-sanity.yaml
@@ -30,7 +30,7 @@ jobs:
 
       - uses: actions/setup-go@v5
         with:
-          go-version: "1.22" # The Go version to download (if necessary) and use.
+          go-version-file: go.mod # The Go version to download (if necessary) and use.
 
       - name: Do sanity checks
         run: make sanity
diff --git a/.github/workflows/publish-community-operators.yaml b/.github/workflows/publish-community-operators.yaml
@@ -15,6 +15,8 @@ jobs:
       REGISTRY_NAMESPACE: kubevirt
       OPM_VERSION: v1.47.0
     steps:
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
       - name: resolve the correct branch of the tag
         run: |
           GIT_TAG=${{ github.ref }}
@@ -32,7 +34,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: '1.22'
+          go-version-file: go.mod
       - name: Checkout the latest code of ${{ env.TARGET_BRANCH }} branch
         uses: actions/checkout@v4
         with:
@@ -44,17 +46,12 @@ jobs:
           CSV_VERSION=$(ls -d ${PACKAGE_DIR}/*/ | sort -rV | awk "NR==1" | cut -d '/' -f 5)
           echo "CSV_VERSION=${CSV_VERSION}" >> $GITHUB_ENV
           echo "PACKAGE_DIR=${PACKAGE_DIR}" >> $GITHUB_ENV
-      - name: Build Applications Images
-        env:
-          IMAGE_TAG: ${{ env.CSV_VERSION }}
-        run: |
-          IMAGE_TAG=${CSV_VERSION} make container-build
-      - name: Push Application Images
+      - name: Build and Push Applications Images
         env:
           IMAGE_TAG: ${{ env.CSV_VERSION }}
         run: |
           make quay-login
-          IMAGE_TAG=${IMAGE_TAG} make container-push
+          IMAGE_TAG=${CSV_VERSION} make build-push-multi-arch-images
       - name: Build Digester
         run: |
           (cd tools/digester && go build .)
@@ -93,16 +90,11 @@ jobs:
           echo "NEW_VERSION=${NEW_VERSION}" >> $GITHUB_ENV
           echo "NEW_IMAGE_TAG=${NEW_IMAGE_TAG}" >> $GITHUB_ENV
 
-      - name: Build Applications next version Images
-        env:
-          NEW_IMAGE_TAG: ${{ env.NEW_IMAGE_TAG }}
-        run: |
-          IMAGE_TAG=${NEW_IMAGE_TAG} make container-build
-      - name: Push next version Application Images
+      - name: Build and Push Applications next version Images
         env:
           NEW_IMAGE_TAG: ${{ env.NEW_IMAGE_TAG }}
         run: |
-          IMAGE_TAG=${NEW_IMAGE_TAG} make container-push
+          IMAGE_TAG=${NEW_IMAGE_TAG} make build-push-multi-arch-images
       - name: run manifest for next version
         env:
           PACKAGE_DIR: ${{ env.PACKAGE_DIR }}
diff --git a/Makefile b/Makefile
@@ -13,7 +13,16 @@ VIRT_ARTIFACTS_SERVER ?= $(REGISTRY_NAMESPACE)/virt-artifacts-server
 LDFLAGS            ?= -w -s
 GOLANDCI_LINT_VERSION ?= v1.57.0
 HCO_BUMP_LEVEL ?= minor
+UNAME_ARCH     := $(shell uname -m)
+ifeq ($(UNAME_ARCH),x86_64)
+	TEMP_ARCH = amd64
+else ifeq ($(UNAME_ARCH),aarch64)
+	TEMP_ARCH = arm64
+else
+	TEMP_ARCH := $(UNAME_ARCH)
+endif
 
+ARCH ?= $(TEMP_ARCH)
 
 # Prow doesn't have docker command
 DO=./hack/in-docker.sh
@@ -88,11 +97,19 @@ hack-clean: ## Run ./hack/clean.sh
 
 container-build: container-build-operator container-build-webhook container-build-operator-courier container-build-functest container-build-artifacts-server
 
+build-push-multi-arch-images: build-push-multi-arch-operator-image build-push-multi-arch-webhook-image build-push-multi-arch-functest-image build-push-multi-arch-artifacts-server
+
 container-build-operator:
-	. "hack/cri-bin.sh" && $$CRI_BIN build -f build/Dockerfile -t $(IMAGE_REGISTRY)/$(OPERATOR_IMAGE):$(IMAGE_TAG) --build-arg git_sha=$(SHA) .
+	. "hack/cri-bin.sh" && $$CRI_BIN build --platform=linux/$(ARCH) -f build/Dockerfile -t $(IMAGE_REGISTRY)/$(OPERATOR_IMAGE):$(IMAGE_TAG) --build-arg git_sha=$(SHA) .
+
+build-push-multi-arch-operator-image:
+	IMAGE_NAME=$(IMAGE_REGISTRY)/$(OPERATOR_IMAGE):$(IMAGE_TAG) SHA=SHA DOCKER_FILE=build/Dockerfile ./hack/build-push-multi-arch-images.sh
 
 container-build-webhook:
-	. "hack/cri-bin.sh" && $$CRI_BIN build -f build/Dockerfile.webhook -t $(IMAGE_REGISTRY)/$(WEBHOOK_IMAGE):$(IMAGE_TAG) --build-arg git_sha=$(SHA) .
+	. "hack/cri-bin.sh" && $$CRI_BIN build --platform=linux/$(ARCH) -f build/Dockerfile.webhook -t $(IMAGE_REGISTRY)/$(WEBHOOK_IMAGE):$(IMAGE_TAG) --build-arg git_sha=$(SHA) .
+
+build-push-multi-arch-webhook-image:
+	IMAGE_NAME=$(IMAGE_REGISTRY)/$(WEBHOOK_IMAGE):$(IMAGE_TAG) SHA=$(SHA) DOCKER_FILE="build/Dockerfile.webhook" ./hack/build-push-multi-arch-images.sh
 
 container-build-operator-courier:
 	podman build -f tools/operator-courier/Dockerfile -t hco-courier .
@@ -101,11 +118,17 @@ container-build-validate-bundles:
 	podman build -f tools/operator-sdk-validate/Dockerfile -t operator-sdk-validate-hco .
 
 container-build-functest:
-	. "hack/cri-bin.sh" && $$CRI_BIN build -f build/Dockerfile.functest -t $(IMAGE_REGISTRY)/$(FUNC_TEST_IMAGE):$(IMAGE_TAG) --build-arg git_sha=$(SHA) .
+	. "hack/cri-bin.sh" && $$CRI_BIN build  --platform=linux/$(ARCH) -f build/Dockerfile.functest -t $(IMAGE_REGISTRY)/$(FUNC_TEST_IMAGE):$(IMAGE_TAG) --build-arg git_sha=$(SHA) .
+
+build-push-multi-arch-functest-image:
+	IMAGE_NAME=$(IMAGE_REGISTRY)/$(FUNC_TEST_IMAGE):$(IMAGE_TAG) SHA=$(SHA) DOCKER_FILE="build/Dockerfile.functest" ./hack/build-push-multi-arch-images.sh
 
 container-build-artifacts-server:
 	podman build -f build/Dockerfile.artifacts -t $(IMAGE_REGISTRY)/$(VIRT_ARTIFACTS_SERVER):$(IMAGE_TAG) --build-arg git_sha=$(SHA) .
 
+build-push-multi-arch-artifacts-server:
+	IMAGE_NAME=$(IMAGE_REGISTRY)/$(VIRT_ARTIFACTS_SERVER):$(IMAGE_TAG) SHA=$(SHA) DOCKER_FILE="build/Dockerfile.artifacts" ./hack/build-push-multi-arch-images.sh
+
 container-push: container-push-operator container-push-webhook container-push-functest container-push-artifacts-server
 
 quay-login:
@@ -121,7 +144,7 @@ container-push-functest:
 	. "hack/cri-bin.sh" && $$CRI_BIN push $$CRI_INSECURE $(IMAGE_REGISTRY)/$(FUNC_TEST_IMAGE):$(IMAGE_TAG)
 
 container-push-artifacts-server:
-	podman push $(IMAGE_REGISTRY)/$(VIRT_ARTIFACTS_SERVER):$(IMAGE_TAG)
+	. "hack/cri-bin.sh" && $$CRI_BIN manifest push $(IMAGE_REGISTRY)/$(VIRT_ARTIFACTS_SERVER):$(IMAGE_TAG)
 
 cluster-up:
 	./cluster/up.sh
@@ -292,4 +315,9 @@ bump-hco:
 		lint-monitoring \
 		sanity \
 		goimport \
-		bump-hco
+		bump-hco \
+		build-push-multi-arch-operator-image \
+		build-push-multi-arch-webhook-image \
+		build-push-multi-arch-functest-image \
+		build-push-multi-arch-artifacts-server \
+		build-push-multi-arch-images
diff --git a/automation/nightly/test-nightly-build.sh b/automation/nightly/test-nightly-build.sh
@@ -22,6 +22,9 @@ wget -q https://dl.google.com/go/go1.22.6.linux-amd64.tar.gz
 tar -C /usr/local -xf go*.tar.gz
 export PATH=/usr/local/go/bin:$PATH
 
+# add qemu-user-static
+$CRI_BIN run --rm --privileged docker.io/multiarch/qemu-user-static --reset -p yes
+
 # get latest KubeVirt commit
 latest_kubevirt=$(curl -sL https://storage.googleapis.com/kubevirt-prow/devel/nightly/release/kubevirt/kubevirt/latest)
 latest_kubevirt_image=$(curl -sL "https://storage.googleapis.com/kubevirt-prow/devel/nightly/release/kubevirt/kubevirt/${latest_kubevirt}/kubevirt-operator.yaml" | grep 'OPERATOR_IMAGE' -A1 | tail -n 1 | sed 's/.*value: //g')
@@ -60,11 +63,13 @@ export IMAGE_TAG="nb_${build_date}_$(git show -s --format=%h)"
 export IMAGE_PREFIX=kubevirtci
 TEMP_OPERATOR_IMAGE=${IMAGE_PREFIX}/hyperconverged-cluster-operator
 TEMP_WEBHOOK_IMAGE=${IMAGE_PREFIX}/hyperconverged-cluster-webhook
+TEMP_DOWNLOAD_IMAGE=${IMAGE_PREFIX}/virt-artifacts-server
 CSV_OPERATOR_IMAGE=${IMAGE_REGISTRY}/${TEMP_OPERATOR_IMAGE}
 CSV_WEBHOOK_IMAGE=${IMAGE_REGISTRY}/${TEMP_WEBHOOK_IMAGE}
+CSV_DOWNLOAD_IMAGE=${IMAGE_REGISTRY}/${TEMP_DOWNLOAD_IMAGE}
 
 # Build HCO & HCO Webhook
-OPERATOR_IMAGE=${TEMP_OPERATOR_IMAGE} WEBHOOK_IMAGE=${TEMP_WEBHOOK_IMAGE} make container-build-operator container-push-operator container-build-webhook container-push-webhook
+OPERATOR_IMAGE=${TEMP_OPERATOR_IMAGE} WEBHOOK_IMAGE=${TEMP_WEBHOOK_IMAGE} make build-push-multi-arch-operator-image build-push-multi-arch-webhook-image build-push-multi-arch-artifacts-server
 
 # Update image digests
 sed -i "s#quay.io/kubevirt/virt-#${kv_image/-*/-}#" deploy/images.csv
@@ -76,9 +81,10 @@ export HCO_VERSION="${IMAGE_TAG}"
 
 HCO_OPERATOR_IMAGE_DIGEST=$(tools/digester/digester --image ${CSV_OPERATOR_IMAGE}:${IMAGE_TAG})
 HCO_WEBHOOK_IMAGE_DIGEST=$(tools/digester/digester --image ${CSV_WEBHOOK_IMAGE}:${IMAGE_TAG})
+HCO_DOWNLOAD_IMAGE_DIGEST=$(tools/digester/digester --image ${CSV_DOWNLOAD_IMAGE}:${IMAGE_TAG})
 
 # Build the CSV
-HCO_OPERATOR_IMAGE=${HCO_OPERATOR_IMAGE_DIGEST} HCO_WEBHOOK_IMAGE=${HCO_WEBHOOK_IMAGE_DIGEST} ./hack/build-manifests.sh
+HCO_OPERATOR_IMAGE=${HCO_OPERATOR_IMAGE_DIGEST} HCO_WEBHOOK_IMAGE=${HCO_WEBHOOK_IMAGE_DIGEST} HCO_DOWNLOADS_IMAGE=${HCO_DOWNLOAD_IMAGE_DIGEST} ./hack/build-manifests.sh
 
 # Download OPM
 OPM_VERSION=v1.47.0
diff --git a/build/Dockerfile b/build/Dockerfile
@@ -1,10 +1,15 @@
-FROM docker.io/golang:1.22.11 AS builder
+FROM --platform=${BUILDPLATFORM} docker.io/golang:1.22.11 AS builder
 
 WORKDIR /go/src/github.com/kubevirt/hyperconverged-cluster-operator/
 COPY . .
-RUN make build-operator build-csv-merger
+ARG TARGETOS
+ARG TARGETARCH
+
+RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} make build-operator build-csv-merger
+
+FROM --platform=${TARGETPLATFORM} registry.access.redhat.com/ubi9/ubi-minimal
+ARG TARGETPLATFORM
 
-FROM registry.access.redhat.com/ubi9/ubi-minimal
 ENV OPERATOR=/usr/local/bin/hyperconverged-cluster-operator \
     CSV_MERGER=/usr/local/bin/csv-merger \
     USER_UID=1001 \
@@ -29,4 +34,5 @@ ARG git_sha=NONE
 
 LABEL multi.GIT_URL=${git_url} \
       multi.GIT_SHA=${git_sha} \
-      app=hyperconverged-cluster-operator
+      app=hyperconverged-cluster-operator \
+      golang.build.platform=${TARGETPLATFORM}
diff --git a/build/Dockerfile.artifacts b/build/Dockerfile.artifacts
@@ -1,17 +1,9 @@
-FROM registry.access.redhat.com/ubi9/nginx-120
-
-WORKDIR /opt/app-root/src
-
-COPY hack/config /tmp/config
-
-USER 0
-RUN dnf -y install zip file && \
-    sed '/^\s*listen\s*\[::\]:8080/d' /etc/nginx/nginx.conf > /etc/nginx/nginx.conf.ipv4 && \
-    sed '/^\s*listen\s*8080/d' /etc/nginx/nginx.conf > /etc/nginx/nginx.conf.ipv6
-USER 1001
+FROM --platform=${BUILDPLATFORM} quay.io/centos/centos:stream9 as downloader
 
 ARG download_url=https://github.com/kubevirt/kubevirt/releases/download
 
+COPY hack/config /tmp/config
+RUN dnf -y install zip file
 RUN eval $(cat /tmp/config  |grep KUBEVIRT_VERSION=) && \
     echo "KUBEVIRT_VERSION: $KUBEVIRT_VERSION" && \
     for arch in amd64 arm64 s390x; do \
@@ -39,11 +31,25 @@ RUN eval $(cat /tmp/config  |grep KUBEVIRT_VERSION=) && \
                 printf "\n\n### Downloading ${url}\n"; \
                 curl --fail -L -o virtctl${extension} "${url}" && \
                 file virtctl${extension} && \
-                mkdir -p ./${arch}/${l_os} && ${archive_command} ./${arch}/${l_os}/virtctl${archive_extension} virtctl${extension} && rm virtctl${extension}; \
+                mkdir -p ./bin/${arch}/${l_os} && ${archive_command} ./bin/${arch}/${l_os}/virtctl${archive_extension} virtctl${extension} && rm virtctl${extension}; \
             fi; \
         done; \
     done
 
+FROM --platform=${TARGETPLATFORM} registry.access.redhat.com/ubi9/nginx-124
+ARG TARGETPLATFORM
+
+WORKDIR /opt/app-root/src
+
+COPY --from=downloader ./bin/ ./
+
+USER 0
+RUN sed -i -E '/^ +location.*$/a\        }\n\n        location = /health {\n            access_log off;\n            return 200;' /etc/nginx/nginx.conf && \
+    sed '/^\s*listen\s*\[::\]:8080/d' /etc/nginx/nginx.conf > /etc/nginx/nginx.conf.ipv4 && \
+    sed '/^\s*listen\s*8080/d' /etc/nginx/nginx.conf > /etc/nginx/nginx.conf.ipv6
+
+USER 1001
+
 ARG git_url=https://github.com/kubevirt/hyperconverged-cluster-operator.git
 ARG git_sha=NONE
 
diff --git a/build/Dockerfile.functest b/build/Dockerfile.functest
@@ -1,10 +1,14 @@
-FROM docker.io/golang:1.22.11 AS builder
+FROM --platform=${BUILDPLATFORM} docker.io/golang:1.22.11 AS builder
 
 WORKDIR /go/src/github.com/kubevirt/hyperconverged-cluster-operator/
 COPY . .
-RUN make build-functest
 
-FROM registry.access.redhat.com/ubi9/ubi-minimal
+ARG TARGETARCH
+
+RUN ARCH=${TARGETARCH} make build-functest
+
+FROM --platform=${TARGETPLATFORM} registry.access.redhat.com/ubi9/ubi-minimal
+ARG TARGETPLATFORM
 
 ENV USER_UID=1001 \
     TEST_OUT_PATH=/test
@@ -30,4 +34,5 @@ ARG git_sha=NONE
 
 LABEL multi.GIT_URL=${git_url} \
       multi.GIT_SHA=${git_sha} \
-      app=hyperconverged-cluster-functest
+      app=hyperconverged-cluster-functest \
+      golang.build.platform=${TARGETPLATFORM}
diff --git a/build/Dockerfile.webhook b/build/Dockerfile.webhook
@@ -1,10 +1,16 @@
-FROM docker.io/golang:1.22.11 AS builder
+FROM --platform=${BUILDPLATFORM} docker.io/golang:1.22.11 AS builder
 
 WORKDIR /go/src/github.com/kubevirt/hyperconverged-cluster-operator/
 COPY . .
-RUN make build-webhook
 
-FROM registry.access.redhat.com/ubi9/ubi-minimal
+ARG TARGETOS
+ARG TARGETARCH
+
+RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} make build-webhook
+
+FROM --platform=${TARGETPLATFORM} registry.access.redhat.com/ubi9/ubi-minimal
+ARG TARGETPLATFORM
+
 ENV WEBHOOK=/usr/local/bin/hyperconverged-cluster-webhook \
     USER_UID=1001 \
     USER_NAME=hyperconverged-cluster-webhook \
@@ -27,4 +33,5 @@ ARG git_sha=NONE
 
 LABEL multi.GIT_URL=${git_url} \
       multi.GIT_SHA=${git_sha} \
-      app=hyperconverged-cluster-webhook
+      app=hyperconverged-cluster-webhook \
+      golang.build.platform=${TARGETPLATFORM}
diff --git a/hack/build-index-image.sh b/hack/build-index-image.sh
@@ -23,6 +23,7 @@ BUNDLE_REGISTRY_IMAGE_NAME=${BUNDLE_REGISTRY_IMAGE_NAME:-hyperconverged-cluster-
 INDEX_REGISTRY_IMAGE_NAME=${INDEX_REGISTRY_IMAGE_NAME:-hyperconverged-cluster-index}
 OPM=${OPM:-opm}
 UNSTABLE=$2
+ARCHITECTURES="amd64 arm64 s390x"
 
 
 function create_index_image() {
@@ -48,8 +49,6 @@ function create_index_image() {
   BUNDLE_IMAGE_NAME=$("${PROJECT_ROOT}/tools/digester/digester" --image "${BUNDLE_IMAGE_NAME}")
 
   create_file_based_catalog "${INITIAL_VERSION}" "${BUNDLE_IMAGE_NAME}" "${UNSTABLE}"
-
-  podman push "${INDEX_IMAGE_NAME}"
 }
 
 function create_file_based_catalog() {
@@ -82,7 +81,15 @@ function create_file_based_catalog() {
   ${OPM} validate fbc-catalog
   rm -f fbc-catalog.Dockerfile
   ${OPM} generate dockerfile fbc-catalog
-  podman build -t "${INDEX_IMAGE_NAME}" -f fbc-catalog.Dockerfile
+
+  IMAGES=
+  for arch in ${ARCHITECTURES}; do
+    podman build --platform="linux/${arch}" -t "${INDEX_IMAGE_NAME}-${arch}" -f fbc-catalog.Dockerfile
+    IMAGES="${IMAGES} ${INDEX_IMAGE_NAME}-${arch}"
+  done
+
+  podman manifest create "${INDEX_IMAGE_NAME}" ${IMAGES}
+  podman manifest push "${INDEX_IMAGE_NAME}"
 }
 
 function create_all_versions() {
diff --git a/hack/build-multi-arch-images.sh b/hack/build-multi-arch-images.sh
diff --git a/hack/build-tests.sh b/hack/build-tests.sh
diff --git a/hack/in-docker.sh b/hack/in-docker.sh
diff --git a/tests/build/Dockerfile b/tests/build/Dockerfile
