Skip to content

Commit aa76863

Browse files
nunnatsanunnatsa
authored
Fix issues with build multi-arch images (#3308)
When building image with different architecture than the build machine architecture, any binary that runs as part of the build will fail (even just creating a user. Another issue is that github action uses ubuntu, that its podman version is 4.x, and cannot build manifest from local images, as possible in podman 5.x. docker/podman needs qemu installed on the build machine, to allow runnig binaries in diffferent architecure as part of the image build process. This PR installs qemu in the relevant github actions, and pushes the images before building the manifest, to match the podman 4.x patter. Signed-off-by: Nahshon Unna-Tsameret <[email protected]>
1 parent de3d79a commit aa76863

9 files changed

+43
-62
lines changed

‎.github/workflows/build-push-images.yaml

+4-7
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ jobs:
2222
REGISTRY_NAMESPACE: kubevirt
2323
OPM_VERSION: v1.47.0
2424
steps:
25+
- name: Set up QEMU
26+
uses: docker/setup-qemu-action@v3
2527
- name: Checkout the latest code
2628
uses: actions/checkout@v4
2729
with:
@@ -42,17 +44,12 @@ jobs:
4244
echo "IMAGE_TAG=${CSV_VERSION}-unstable" >> $GITHUB_ENV
4345
echo "UNSTABLE=UNSTABLE" >> $GITHUB_ENV
4446
45-
- name: Build Applications Images
46-
env:
47-
IMAGE_TAG: ${{ env.IMAGE_TAG }}
48-
run: |
49-
IMAGE_TAG=${IMAGE_TAG} make build-multi-arch-images
50-
- name: Push Application Images
47+
- name: Build and Push Applications Images
5148
env:
5249
IMAGE_TAG: ${{ env.IMAGE_TAG }}
5350
run: |
5451
make quay-login
55-
IMAGE_TAG=${IMAGE_TAG} make push-multi-arch-images
52+
IMAGE_TAG=${IMAGE_TAG} make build-push-multi-arch-images
5653
- name: Build Digester
5754
run: |
5855
(cd tools/digester && go build .)

‎.github/workflows/publish-community-operators.yaml

+6-14
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ jobs:
1515
REGISTRY_NAMESPACE: kubevirt
1616
OPM_VERSION: v1.47.0
1717
steps:
18+
- name: Set up QEMU
19+
uses: docker/setup-qemu-action@v3
1820
- name: resolve the correct branch of the tag
1921
run: |
2022
GIT_TAG=${{ github.ref }}
@@ -44,17 +46,12 @@ jobs:
4446
CSV_VERSION=$(ls -d ${PACKAGE_DIR}/*/ | sort -rV | awk "NR==1" | cut -d '/' -f 5)
4547
echo "CSV_VERSION=${CSV_VERSION}" >> $GITHUB_ENV
4648
echo "PACKAGE_DIR=${PACKAGE_DIR}" >> $GITHUB_ENV
47-
- name: Build Applications Images
48-
env:
49-
IMAGE_TAG: ${{ env.CSV_VERSION }}
50-
run: |
51-
IMAGE_TAG=${CSV_VERSION} make build-multi-arch-images
52-
- name: Push Application Images
49+
- name: Build and Push Applications Images
5350
env:
5451
IMAGE_TAG: ${{ env.CSV_VERSION }}
5552
run: |
5653
make quay-login
57-
IMAGE_TAG=${IMAGE_TAG} make push-multi-arch-images
54+
IMAGE_TAG=${CSV_VERSION} make build-push-multi-arch-images
5855
- name: Build Digester
5956
run: |
6057
(cd tools/digester && go build .)
@@ -93,16 +90,11 @@ jobs:
9390
echo "NEW_VERSION=${NEW_VERSION}" >> $GITHUB_ENV
9491
echo "NEW_IMAGE_TAG=${NEW_IMAGE_TAG}" >> $GITHUB_ENV
9592
96-
- name: Build Applications next version Images
97-
env:
98-
NEW_IMAGE_TAG: ${{ env.NEW_IMAGE_TAG }}
99-
run: |
100-
IMAGE_TAG=${NEW_IMAGE_TAG} make build-multi-arch-images
101-
- name: Push next version Application Images
93+
- name: Build and Push Applications next version Images
10294
env:
10395
NEW_IMAGE_TAG: ${{ env.NEW_IMAGE_TAG }}
10496
run: |
105-
IMAGE_TAG=${NEW_IMAGE_TAG} make push-multi-arch-images
97+
IMAGE_TAG=${NEW_IMAGE_TAG} make build-push-multi-arch-images
10698
- name: run manifest for next version
10799
env:
108100
PACKAGE_DIR: ${{ env.PACKAGE_DIR }}

‎Makefile

+14-33
Original file line numberDiff line numberDiff line change
@@ -96,25 +96,19 @@ hack-clean: ## Run ./hack/clean.sh
9696

9797
container-build: container-build-operator container-build-webhook container-build-operator-courier container-build-functest container-build-artifacts-server
9898

99-
build-multi-arch-images: build-multi-arch-operator-image build-multi-arch-webhook-image build-multi-arch-functest-image build-multi-arch-artifacts-server
99+
build-push-multi-arch-images: build-push-multi-arch-operator-image build-push-multi-arch-webhook-image build-push-multi-arch-functest-image build-push-multi-arch-artifacts-server
100100

101101
container-build-operator:
102102
. "hack/cri-bin.sh" && $$CRI_BIN build --platform=linux/$(ARCH) -f build/Dockerfile -t $(IMAGE_REGISTRY)/$(OPERATOR_IMAGE):$(IMAGE_TAG) --build-arg git_sha=$(SHA) .
103103

104-
build-multi-arch-operator-image:
105-
IMAGE_NAME=$(IMAGE_REGISTRY)/$(OPERATOR_IMAGE):$(IMAGE_TAG) SHA=SHA DOCKER_FILE=build/Dockerfile ./hack/build-multi-arch-images.sh
106-
107-
push-multi-arch-operator-image:
108-
. "hack/cri-bin.sh" && $$CRI_BIN manifest push $(IMAGE_REGISTRY)/$(OPERATOR_IMAGE):$(IMAGE_TAG)
104+
build-push-multi-arch-operator-image:
105+
IMAGE_NAME=$(IMAGE_REGISTRY)/$(OPERATOR_IMAGE):$(IMAGE_TAG) SHA=SHA DOCKER_FILE=build/Dockerfile ./hack/build-push-multi-arch-images.sh
109106

110107
container-build-webhook:
111108
. "hack/cri-bin.sh" && $$CRI_BIN build --platform=linux/$(ARCH) -f build/Dockerfile.webhook -t $(IMAGE_REGISTRY)/$(WEBHOOK_IMAGE):$(IMAGE_TAG) --build-arg git_sha=$(SHA) .
112109

113-
build-multi-arch-webhook-image:
114-
IMAGE_NAME=$(IMAGE_REGISTRY)/$(WEBHOOK_IMAGE):$(IMAGE_TAG) SHA=$(SHA) DOCKER_FILE="build/Dockerfile.webhook" ./hack/build-multi-arch-images.sh
115-
116-
push-multi-arch-webhook-image:
117-
. "hack/cri-bin.sh" && $$CRI_BIN manifest push $(IMAGE_REGISTRY)/$(WEBHOOK_IMAGE):$(IMAGE_TAG)
110+
build-push-multi-arch-webhook-image:
111+
IMAGE_NAME=$(IMAGE_REGISTRY)/$(WEBHOOK_IMAGE):$(IMAGE_TAG) SHA=$(SHA) DOCKER_FILE="build/Dockerfile.webhook" ./hack/build-push-multi-arch-images.sh
118112

119113
container-build-operator-courier:
120114
podman build -f tools/operator-courier/Dockerfile -t hco-courier .
@@ -125,25 +119,17 @@ container-build-validate-bundles:
125119
container-build-functest:
126120
. "hack/cri-bin.sh" && $$CRI_BIN build --platform=linux/$(ARCH) -f build/Dockerfile.functest -t $(IMAGE_REGISTRY)/$(FUNC_TEST_IMAGE):$(IMAGE_TAG) --build-arg git_sha=$(SHA) .
127121

128-
build-multi-arch-functest-image:
129-
IMAGE_NAME=$(IMAGE_REGISTRY)/$(FUNC_TEST_IMAGE):$(IMAGE_TAG) SHA=$(SHA) DOCKER_FILE="build/Dockerfile.functest" ./hack/build-multi-arch-images.sh
130-
131-
push-multi-arch-functest-image:
132-
. "hack/cri-bin.sh" && $$CRI_BIN manifest push $(IMAGE_REGISTRY)/$(FUNC_TEST_IMAGE):$(IMAGE_TAG)
122+
build-push-multi-arch-functest-image:
123+
IMAGE_NAME=$(IMAGE_REGISTRY)/$(FUNC_TEST_IMAGE):$(IMAGE_TAG) SHA=$(SHA) DOCKER_FILE="build/Dockerfile.functest" ./hack/build-push-multi-arch-images.sh
133124

134125
container-build-artifacts-server:
135126
podman build -f build/Dockerfile.artifacts -t $(IMAGE_REGISTRY)/$(VIRT_ARTIFACTS_SERVER):$(IMAGE_TAG) --build-arg git_sha=$(SHA) .
136127

137-
build-multi-arch-artifacts-server:
138-
IMAGE_NAME=$(IMAGE_REGISTRY)/$(VIRT_ARTIFACTS_SERVER):$(IMAGE_TAG) SHA=$(SHA) DOCKER_FILE="build/Dockerfile.artifacts" ./hack/build-multi-arch-images.sh
139-
140-
push-multi-arch-artifacts-server:
141-
. "hack/cri-bin.sh" && $$CRI_BIN manifest push $(IMAGE_REGISTRY)/$(VIRT_ARTIFACTS_SERVER):$(IMAGE_TAG)
128+
build-push-multi-arch-artifacts-server:
129+
IMAGE_NAME=$(IMAGE_REGISTRY)/$(VIRT_ARTIFACTS_SERVER):$(IMAGE_TAG) SHA=$(SHA) DOCKER_FILE="build/Dockerfile.artifacts" ./hack/build-push-multi-arch-images.sh
142130

143131
container-push: container-push-operator container-push-webhook container-push-functest container-push-artifacts-server
144132

145-
push-multi-arch-images: push-multi-arch-operator-image push-multi-arch-webhook-image push-multi-arch-functest-image push-multi-arch-artifacts-server
146-
147133
quay-login:
148134
podman login $(IMAGE_REGISTRY) -u $(QUAY_USERNAME) -p "$(QUAY_PASSWORD)"
149135

@@ -341,13 +327,8 @@ bump-hco:
341327
sanity \
342328
goimport \
343329
bump-hco \
344-
build-multi-arch-operator-image \
345-
push-multi-arch-operator-image \
346-
build-multi-arch-webhook-image \
347-
push-multi-arch-webhook-image \
348-
build-multi-arch-functest-image \
349-
push-multi-arch-functest-image \
350-
build-multi-arch-artifacts-server \
351-
push-multi-arch-artifacts-server \
352-
build-multi-arch-images \
353-
push-multi-arch-images
330+
build-push-multi-arch-operator-image \
331+
build-push-multi-arch-webhook-image \
332+
build-push-multi-arch-functest-image \
333+
build-push-multi-arch-artifacts-server \
334+
build-push-multi-arch-images

‎automation/nightly/test-nightly-build.sh

+5-2
Original file line numberDiff line numberDiff line change
@@ -60,11 +60,13 @@ export IMAGE_TAG="nb_${build_date}_$(git show -s --format=%h)"
6060
export IMAGE_PREFIX=kubevirtci
6161
TEMP_OPERATOR_IMAGE=${IMAGE_PREFIX}/hyperconverged-cluster-operator
6262
TEMP_WEBHOOK_IMAGE=${IMAGE_PREFIX}/hyperconverged-cluster-webhook
63+
TEMP_DOWNLOAD_IMAGE=${IMAGE_PREFIX}/virt-artifacts-server
6364
CSV_OPERATOR_IMAGE=${IMAGE_REGISTRY}/${TEMP_OPERATOR_IMAGE}
6465
CSV_WEBHOOK_IMAGE=${IMAGE_REGISTRY}/${TEMP_WEBHOOK_IMAGE}
66+
CSV_DOWNLOAD_IMAGE=${IMAGE_REGISTRY}/${TEMP_DOWNLOAD_IMAGE}
6567

6668
# Build HCO & HCO Webhook
67-
OPERATOR_IMAGE=${TEMP_OPERATOR_IMAGE} WEBHOOK_IMAGE=${TEMP_WEBHOOK_IMAGE} make container-build-operator container-push-operator container-build-webhook container-push-webhook
69+
OPERATOR_IMAGE=${TEMP_OPERATOR_IMAGE} WEBHOOK_IMAGE=${TEMP_WEBHOOK_IMAGE} make build-push-multi-arch-operator-image build-push-multi-arch-webhook-image build-push-multi-arch-artifacts-server
6870

6971
# Update image digests
7072
sed -i "s#quay.io/kubevirt/virt-#${kv_image/-*/-}#" deploy/images.csv
@@ -76,9 +78,10 @@ export HCO_VERSION="${IMAGE_TAG}"
7678

7779
HCO_OPERATOR_IMAGE_DIGEST=$(tools/digester/digester --image ${CSV_OPERATOR_IMAGE}:${IMAGE_TAG})
7880
HCO_WEBHOOK_IMAGE_DIGEST=$(tools/digester/digester --image ${CSV_WEBHOOK_IMAGE}:${IMAGE_TAG})
81+
HCO_DOWNLOAD_IMAGE_DIGEST=$(tools/digester/digester --image ${CSV_DOWNLOAD_IMAGE}:${IMAGE_TAG})
7982

8083
# Build the CSV
81-
HCO_OPERATOR_IMAGE=${HCO_OPERATOR_IMAGE_DIGEST} HCO_WEBHOOK_IMAGE=${HCO_WEBHOOK_IMAGE_DIGEST} ./hack/build-manifests.sh
84+
HCO_OPERATOR_IMAGE=${HCO_OPERATOR_IMAGE_DIGEST} HCO_WEBHOOK_IMAGE=${HCO_WEBHOOK_IMAGE_DIGEST} HCO_DOWNLOADS_IMAGE=${HCO_DOWNLOAD_IMAGE_DIGEST} ./hack/build-manifests.sh
8285

8386
# Download OPM
8487
OPM_VERSION=v1.47.0

‎build/Dockerfile

+4-2
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,9 @@ ARG TARGETARCH
77

88
RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} make build-operator build-csv-merger
99

10-
FROM registry.access.redhat.com/ubi9/ubi-minimal
10+
FROM --platform=${TARGETPLATFORM} registry.access.redhat.com/ubi9/ubi-minimal
11+
ARG TARGETPLATFORM
12+
1113
ENV OPERATOR=/usr/local/bin/hyperconverged-cluster-operator \
1214
CSV_MERGER=/usr/local/bin/csv-merger \
1315
USER_UID=1001 \
@@ -33,4 +35,4 @@ ARG git_sha=NONE
3335
LABEL multi.GIT_URL=${git_url} \
3436
multi.GIT_SHA=${git_sha} \
3537
app=hyperconverged-cluster-operator \
36-
arch=${ARCH}
38+
golang.build.platform=${TARGETPLATFORM}

‎build/Dockerfile.artifacts

+1
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ RUN eval $(cat /tmp/config |grep KUBEVIRT_VERSION=) && \
3737
done
3838

3939
FROM --platform=${TARGETPLATFORM} registry.access.redhat.com/ubi9/nginx-124
40+
ARG TARGETPLATFORM
4041

4142
WORKDIR /opt/app-root/src
4243

‎build/Dockerfile.functest

+3-2
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@ ARG TARGETARCH
77

88
RUN ARCH=${TARGETARCH} make build-functest
99

10-
FROM registry.access.redhat.com/ubi9/ubi-minimal
10+
FROM --platform=${TARGETPLATFORM} registry.access.redhat.com/ubi9/ubi-minimal
11+
ARG TARGETPLATFORM
1112

1213
ENV USER_UID=1001 \
1314
TEST_OUT_PATH=/test
@@ -34,4 +35,4 @@ ARG git_sha=NONE
3435
LABEL multi.GIT_URL=${git_url} \
3536
multi.GIT_SHA=${git_sha} \
3637
app=hyperconverged-cluster-functest \
37-
arch=${ARCH}
38+
golang.build.platform=${TARGETPLATFORM}

‎build/Dockerfile.webhook

+4-2
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,9 @@ ARG TARGETARCH
88

99
RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} make build-webhook
1010

11-
FROM registry.access.redhat.com/ubi9/ubi-minimal
11+
FROM --platform=${TARGETPLATFORM} registry.access.redhat.com/ubi9/ubi-minimal
12+
ARG TARGETPLATFORM
13+
1214
ENV WEBHOOK=/usr/local/bin/hyperconverged-cluster-webhook \
1315
USER_UID=1001 \
1416
USER_NAME=hyperconverged-cluster-webhook \
@@ -32,4 +34,4 @@ ARG git_sha=NONE
3234
LABEL multi.GIT_URL=${git_url} \
3335
multi.GIT_SHA=${git_sha} \
3436
app=hyperconverged-cluster-webhook \
35-
arch=${TARGETARCH}
37+
golang.build.platform=${TARGETPLATFORM}

‎hack/build-multi-arch-images.sh ‎hack/build-push-multi-arch-images.sh

+2
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@ SHA=$(git describe --no-match --always --abbrev=40 --dirty)
1717
IMAGES=
1818
for arch in ${ARCHITECTURES}; do
1919
. "hack/cri-bin.sh" && ${CRI_BIN} build --platform=linux/${arch} -f ${DOCKER_FILE} -t "${IMAGE_NAME}-${arch}" --build-arg git_sha=${SHA} .
20+
. "hack/cri-bin.sh" && ${CRI_BIN} push "${IMAGE_NAME}-${arch}"
2021
IMAGES="${IMAGES} ${IMAGE_NAME}-${arch}"
2122
done
2223

2324
. "hack/cri-bin.sh" && ${CRI_BIN} manifest create "${IMAGE_NAME}" ${IMAGES}
25+
. "hack/cri-bin.sh" && ${CRI_BIN} manifest push "${IMAGE_NAME}"

0 commit comments

Comments
 (0)