Native multiarch image builds

.github/workflows/collector-builder.yml

@@ -11,7 +11,7 @@ on:
     outputs:
       collector-builder-tag:
         description: The builder tag used by the build
-        value: ${{ jobs.build-builder-image.outputs.collector-builder-tag || 'master' }}
+        value: ${{ jobs.builder-needs-rebuilding.outputs.collector-builder-tag }}
 
 env:
   COLLECTOR_TAG: ${{ inputs.collector-tag }}
@@ -24,6 +24,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       build-image: ${{ steps.changed.outputs.builder-changed }}
+      collector-builder-tag: ${{ steps.set-builder-tag.outputs.collector-builder-tag || 'master' }}
 
     steps:
       - uses: actions/checkout@v4
@@ -38,6 +39,23 @@ jobs:
               - builder/Dockerfile
               - .github/workflows/collector-builder.yml
 
+      - name: Set builder tag
+        id: set-builder-tag
+        if: |
+          steps.changed.outputs.builder-changed == 'true' ||
+          contains(github.event.pull_request.labels.*.name, 'build-builder-image') || (
+            github.event_name == 'push' && (
+              github.ref_type == 'tag' || startsWith(github.ref_name, 'release-')
+          ))
+        run: |
+          COLLECTOR_BUILDER_TAG="${DEFAULT_BUILDER_TAG}"
+          if [[ "${{ github.event_name }}" == 'pull_request' || \
+                "${{ github.ref_type }}" == 'tag' || \
+                "${{ github.ref_name }}" =~ ^release- ]]; then
+            COLLECTOR_BUILDER_TAG="${{ inputs.collector-tag }}"
+          fi
+          echo "collector-builder-tag=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_OUTPUT"
+
   build-builder-image:
     name: Build the builder image
     runs-on: ubuntu-latest
@@ -51,16 +69,15 @@ jobs:
         github.ref_type == 'tag' || startsWith(github.ref_name, 'release-')
       )) ||
       contains(github.event.pull_request.labels.*.name, 'build-builder-image')
-    outputs:
-      collector-builder-tag: ${{ steps.builder-tag.outputs.collector-builder-tag }}
     strategy:
       fail-fast: false
       matrix:
-        arch: [amd64, ppc64le, s390x, arm64]
+        arch: [amd64]
 
     env:
       PLATFORM: linux/${{ matrix.arch }}
       BUILD_TYPE: ci
+      COLLECTOR_BUILDER_TAG: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }}
 
     steps:
       - uses: actions/checkout@v4
@@ -73,6 +90,60 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
+      - name: Create ansible vars
+        run: |
+          cat << EOF > ${{ github.workspace }}/ansible/secrets.yml
+          ---
+          stackrox_io_username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }}
+          stackrox_io_password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }}
+          rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }}
+          rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }}
+          collector_git_ref: ${{ github.ref }}
+          collector_git_sha: ${{ github.sha }}
+          collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }}
+          clone_repository: false
+          EOF
+
+      - name: Build images
+        timeout-minutes: 480
+        run: |
+          ansible-galaxy install -r ansible/requirements.yml
+          ansible-playbook \
+            --connection local \
+            -i localhost, \
+            --limit localhost \
+            -e arch='${{ matrix.arch }}' \
+            -e @'${{ github.workspace }}/ansible/secrets.yml' \
+            ansible/ci-build-builder.yml
+
+  build-builder-image-remote-vm:
+    name: Build the builder image
+    runs-on: ubuntu-latest
+    # Multiarch builds sometimes take for eeeeeeeeeever
+    timeout-minutes: 480
+    needs:
+    - builder-needs-rebuilding
+    if: |
+      needs.builder-needs-rebuilding.outputs.build-image == 'true' || (
+        github.event_name == 'push' && (
+        github.ref_type == 'tag' || startsWith(github.ref_name, 'release-')
+      )) || (
+        contains(github.event.pull_request.labels.*.name, 'build-builder-image') &&
+        contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds')
+      )
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [ppc64le, s390x, arm64]
+
+    env:
+      PLATFORM: linux/${{ matrix.arch }}
+      BUILD_TYPE: ci
+      COLLECTOR_BUILDER_TAG: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }}
+
+    steps:
+      - uses: actions/checkout@v4
+
       - uses: actions/setup-python@v5
         with:
           python-version: "3.10"
@@ -94,62 +165,28 @@ jobs:
           ppc64le-key: ${{ secrets.IBM_CLOUD_POWER_API_KEY }}
           redhat-username: ${{ secrets.REDHAT_USERNAME }}
           redhat-password: ${{ secrets.REDHAT_PASSWORD }}
-          vm-type: all
+          vm-type: rhel-${{ matrix.arch }}
           job-tag: builder
 
       - name: Create Build VMs
-        if: |
-          matrix.arch == 's390x' &&
-          (github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds'))
         run: |
           make -C "${{ github.workspace }}/ansible" create-build-vms
 
-      - name: Define builder tag
-        id: builder-tag
-        run: |
-          COLLECTOR_BUILDER_TAG="${DEFAULT_BUILDER_TAG}"
-          if [[ "${{ github.event_name }}" == 'pull_request' || \
-                "${{ github.ref_type }}" == 'tag' || \
-                "${{ github.ref_name }}" =~ ^release- ]]; then
-            COLLECTOR_BUILDER_TAG="${{ inputs.collector-tag }}"
-          fi
-
-          echo "COLLECTOR_BUILDER_TAG=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_ENV"
-          echo "collector-builder-tag=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_OUTPUT"
-
       - name: Create ansible vars
         run: |
-          {
-            echo "---"
-            echo "stackrox_io_username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }}"
-            echo "stackrox_io_password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }}"
-            echo "rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }}"
-            echo "rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }}"
-            echo "collector_git_ref: ${{ github.ref }}"
-            echo "collector_git_sha: ${{ github.sha }}"
-            echo "collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }}"
-          } > ${{ github.workspace }}/ansible/secrets.yml
+          cat << EOF > ${{ github.workspace }}/ansible/secrets.yml
+          ---
+          stackrox_io_username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }}
+          stackrox_io_password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }}
+          rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }}
+          rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }}
+          collector_git_ref: ${{ github.ref }}
+          collector_git_sha: ${{ github.sha }}
+          collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }}
+          clone_repository: true
+          EOF
 
       - name: Build images
-        if: |
-          (github.event_name == 'push' && matrix.arch != 's390x') ||
-          matrix.arch == 'amd64' ||
-          (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && matrix.arch != 's390x')
-        timeout-minutes: 480
-        run: |
-          ansible-galaxy install -r ansible/requirements.yml
-          ansible-playbook \
-            --connection local \
-            -i localhost, \
-            --limit localhost \
-            -e arch='${{ matrix.arch }}' \
-            -e @'${{ github.workspace }}/ansible/secrets.yml' \
-            ansible/ci-build-builder.yml
-
-      - name: Build s390x images
-        if: |
-          (github.event_name == 'push' && matrix.arch == 's390x') ||
-          (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && matrix.arch == 's390x')
         timeout-minutes: 480
         run: |
           ansible-playbook \
@@ -160,21 +197,24 @@ jobs:
             ansible/ci-build-builder.yml
 
       - name: Destroy VMs
-        if: always() && matrix.arch == 's390x'
+        if: always()
         run: |
           make -C ansible destroy-vms
 
   create-multiarch-manifest:
     needs:
+    - builder-needs-rebuilding
     - build-builder-image
+    - build-builder-image-remote-vm
     name: Create Multiarch manifest
     runs-on: ubuntu-latest
     if: |
-      github.event_name == 'push' ||
-      (needs.build-builder-image.outputs.collector-builder-tag != 'cache' &&
-       contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds'))
+      github.event_name == 'push' || (
+        needs.builder-needs-rebuilding.outputs.collector-builder-tag != 'master' &&
+        contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds')
+      )
     env:
-      COLLECTOR_BUILDER_TAG: ${{ needs.build-builder-image.outputs.collector-builder-tag }}
+      COLLECTOR_BUILDER_TAG: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }}
       ARCHS: amd64 ppc64le s390x arm64
 
     steps:
@@ -208,15 +248,16 @@ jobs:
 
   retag-x86-image:
     needs:
+    - builder-needs-rebuilding
     - build-builder-image
     name: Retag x86 builder image
     runs-on: ubuntu-latest
     if: |
       github.event_name == 'pull_request' &&
-      needs.build-builder-image.outputs.collector-builder-tag != 'cache' &&
+      needs.builder-needs-rebuilding.outputs.collector-builder-tag != 'master' &&
       !contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds')
     env:
-      COLLECTOR_BUILDER_TAG: ${{ needs.build-builder-image.outputs.collector-builder-tag }}
+      COLLECTOR_BUILDER_TAG: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }}
     steps:
       - name: Pull image to retag
         run: |

.github/workflows/collector-slim.yml

@@ -35,7 +35,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        arch: [amd64, ppc64le, arm64]
+        arch: [amd64, ppc64le]
 
     env:
       PLATFORM: linux/${{ matrix.arch }}
@@ -62,18 +62,19 @@ jobs:
           collector_git_ref: ${{ github.ref }}
           collector_git_sha: ${{ github.sha }}
           collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }}
-          disable_profiling: ${{ matrix.arch != 'amd64' && matrix.arch != 'arm64' }}
+          disable_profiling: ${{ matrix.arch == 'amd64' }}
           rhacs_eng_image: ${{ env.RHACS_ENG_IMAGE }}
           collector_image: ${{ inputs.collector-image }}
           collector_tag: ${{ inputs.collector-tag }}
+          clone_repository: false
           EOF
 
       - name: Build images
+        timeout-minutes: 480
         if: |
           github.event_name == 'push' ||
           matrix.arch == 'amd64' ||
           contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds')
-        timeout-minutes: 480
         run: |
           ansible-playbook \
             --connection local \
@@ -90,7 +91,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        arch: [s390x]
+        arch: [s390x, arm64]
 
     env:
       PLATFORM: linux/${{ matrix.arch }}
@@ -137,10 +138,11 @@ jobs:
           collector_git_ref: ${{ github.ref }}
           collector_git_sha: ${{ github.sha }}
           collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }}
-          disable_profiling: ${{ matrix.arch != 'amd64' && matrix.arch != 'arm64' }}
+          disable_profiling: ${{ matrix.arch == 'arm64' }}
           rhacs_eng_image: ${{ env.RHACS_ENG_IMAGE }}
           collector_image: ${{ inputs.collector-image }}
           collector_tag: ${{ inputs.collector-tag }}
+          clone_repository: true
           EOF
 
       - name: Build ${{ matrix.arch }} image

ansible/ci-build-builder.yml

@@ -16,9 +16,9 @@
         repo: https://github.com/stackrox/collector
         dest: "{{ collector_root }}"
         version: "{{ collector_git_sha }}"
-        refspec: "+{{ collector_git_ref | replace('refs/', '') }}"
+        refspec: "+{{ collector_git_ref }}"
         recursive: true
-      when: arch == "s390x"
+      when: clone_repository
 
     - name: Build the collector builder image
       community.general.make:

ansible/ci-build-collector.yml

@@ -18,9 +18,9 @@
         repo: https://github.com/stackrox/collector
         dest: "{{ collector_root }}"
         version: "{{ collector_git_sha }}"
-        refspec: "+{{ collector_git_ref | replace('refs/', '') }}"
+        refspec: "+{{ collector_git_ref }}"
         recursive: true
-      when: arch == "s390x"
+      when: clone_repository
 
     - name: Run the builder image
       community.general.make:

ansible/ci-create-build-vms.yml

@@ -8,9 +8,11 @@
       include_role:
         name: create-all-vms
       vars:
-        vm_list:
-          # s390x
-          rhel-s390x: "{{ virtual_machines['rhel-s390x'] }}"
+        vm_list: "{{ virtual_machines }}"
+  post_tasks:
+    # We have all the VMs created now, so refresh the inventory - this allows
+    # us to use the provisioning role on the VMs we just created
+    - meta: refresh_inventory
 
 - name: Provision Build VMs
   hosts: "job_id_{{ job_id }}"