add cross-arch aarch64 container image builds

.github/workflows/stackhpc-container-image-build.yml

@@ -82,18 +82,20 @@ jobs:
       # Dynamically define job matrix.
       # We need a separate matrix entry for each distribution, when the relevant input is true.
       # https://stackoverflow.com/questions/65384420/how-do-i-make-a-github-action-matrix-element-conditional
+      # NOTE(bbezak): Both amd64 and aarch64 need to be built in a single workflow to create a multi-architecture manifest.
+      # For now include only RL9 in aarch64
       - name: Generate build matrix
         id: set-matrix
         run: |
+          echo -n "matrix={\"include\": [" >> $GITHUB_OUTPUT
           comma=""
-          echo -n "matrix={\"distro\": [" >> $GITHUB_OUTPUT
-          if [[ ${{ inputs.rocky-linux-9 }} == 'true' ]]; then
-            echo -n "$comma\"rocky\"" >> $GITHUB_OUTPUT
+          if [[ '${{ inputs.rocky-linux-9 }}' == 'true' ]]; then
+            echo -n "$comma{\"distro\": \"rocky\", \"arch\": \"amd64\"}" >> $GITHUB_OUTPUT
             comma=", "
+            echo -n "$comma{\"distro\": \"rocky\", \"arch\": \"aarch64\"}" >> $GITHUB_OUTPUT
           fi
-          if [[ ${{ inputs.ubuntu-jammy }} == 'true' ]]; then
-            echo -n "$comma\"ubuntu\"" >> $GITHUB_OUTPUT
-            comma=", "
+          if [[ '${{ inputs.ubuntu-jammy }}' == 'true' ]]; then
+            echo -n "$comma{\"distro\": \"ubuntu\", \"arch\": \"amd64\"}" >> $GITHUB_OUTPUT
           fi
           echo "]}" >> $GITHUB_OUTPUT
 
@@ -190,8 +192,15 @@ jobs:
         continue-on-error: true
         run: |
           args="${{ inputs.regexes }}"
+          if [[ "${{ matrix.arch }}" = 'aarch64' ]]; then
+            args="$args -e kolla_base_arch=${{ matrix.arch }}"
+          fi
           args="$args -e kolla_base_distro=${{ matrix.distro }}"
-          args="$args -e kolla_tag=${{ steps.write-kolla-tag.outputs.kolla-tag }}"
+          if [[ "${{ matrix.distro }}" = 'rocky' ]]; then
+            args="$args -e kolla_tag=${{ steps.write-kolla-tag.outputs.kolla-tag }}-${{ matrix.arch }}"
+          else
+            args="$args -e kolla_tag=${{ steps.write-kolla-tag.outputs.kolla-tag }}"
+          fi
           args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
@@ -204,6 +213,10 @@ jobs:
         run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-overcloud.log
         if: inputs.overcloud
 
+      - name: Copy build configs to output directory
+        run: sudo cp -rnL /opt/kayobe/etc/kolla/* image-build-logs/
+        if: inputs.overcloud
+
       - name: Build kolla seed images
         id: build_seed_images
         continue-on-error: true
@@ -216,14 +229,14 @@ jobs:
           kayobe seed container image build $args
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        if: inputs.seed
+        if: inputs.seed && matrix.arch == 'amd64'
 
       - name: Copy seed container image build logs to output directory
         run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-seed.log
-        if: inputs.seed
+        if: inputs.seed && matrix.arch == 'amd64'
 
       - name: Get built container images
-        run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}" > ${{ matrix.distro }}-container-images
+        run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}*" > ${{ matrix.distro }}-container-images
 
       - name: Fail if no images have been built
         run: if [ $(wc -l < ${{ matrix.distro }}-container-images) -le 1 ]; then exit 1; fi
@@ -284,7 +297,7 @@ jobs:
       - name: Upload output artifact
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.distro }}-logs
+          name: ${{ matrix.distro }}-${{ matrix.arch }}-logs
           path: image-build-logs
           retention-days: 7
         if: ${{ !cancelled() }}
@@ -308,6 +321,120 @@ jobs:
         run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/critical-images.txt && exit 1; fi
         if: ${{ !inputs.push-dirty && !cancelled() }}
 
+  create-manifests:
+    # Only for Rocky Linux for now
+    name: Create Multiarch Docker Manifests
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push
+    runs-on: arc-skc-container-image-builder-runner
+    permissions: {}
+    needs:
+      - container-image-build
+    steps:
+      - name: Install package dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y git unzip python3-wheel python3-pip python3-venv curl jq wget openssh-server openssh-client
+
+      - name: Install gh
+        run: |
+          sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null
+          sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
+          sudo apt update
+          sudo apt install gh -y
+
+      - name: Checkout Kayobe Config
+        uses: actions/checkout@v4
+        with:
+          path: src/kayobe-config
+
+      - name: Install Kayobe
+        run: |
+          mkdir -p venvs &&
+          pushd venvs &&
+          python3 -m venv kayobe &&
+          source kayobe/bin/activate &&
+          pip install -U pip &&
+          pip install -r ../src/kayobe-config/requirements.txt
+
+      # Required for Pulp auth proxy deployment and Docker registry login.
+      # Normally installed during host configure.
+      - name: Install Docker Python SDK
+        run: |
+          sudo pip install docker
+
+      - name: Configure localhost as a seed
+        run: |
+          cat > src/kayobe-config/etc/kayobe/environments/ci-builder/inventory/hosts << EOF
+          # A 'seed' host used for building images.
+          # Use localhost for container image builds.
+          [seed]
+          localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python3
+          EOF
+
+      # See etc/kayobe/ansible/roles/pulp_auth_proxy/README.md for details.
+      # NOTE: We override pulp_auth_proxy_conf_path to a path shared by the
+      # runner and dind containers.
+      - name: Deploy an authenticating package repository mirror proxy
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-auth-proxy.yml -e pulp_auth_proxy_conf_path=/home/runner/_work/pulp_proxy
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+
+      - name: Combine pushed images lists
+        run: |
+          find . -name 'push-attempt-images.txt' -exec cat {} + > all-pushed-images.txt
+
+      - name: Log in to Docker registry
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/docker-registry-login.yml
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+
+      - name: Create and Push Docker Manifests
+        run: |
+          set -ex
+          mkdir -p logs
+          images=$(cat all-pushed-images.txt | sort | uniq)
+          # Filter out Ubuntu and Rocky Bifrost images
+          manifest_images=$(echo "$images" | grep -E '.*-(amd64|aarch64)$' | sed -E 's/-(amd64|aarch64)$//' | sort | uniq)
+          if [ -z "$manifest_images" ]; then
+            echo "No Rocky overcloud images found. Skipping manifest creation." | tee -a logs/manifest-creation.log
+            exit 0
+          fi
+          for base_image in $manifest_images; do
+            arch_images=""
+            for arch in amd64 aarch64; do
+              arch_image="${base_image}-${arch}"
+              # Check if the image exists in the registry
+              if docker manifest inspect "$arch_image" > /dev/null 2>&1; then
+                arch_images="$arch_images $arch_image"
+              fi
+            done
+            if [ -n "$arch_images" ]; then
+              echo "Creating manifest for $base_image with images:$arch_images" | tee -a logs/manifest-creation.log
+              docker manifest create "$base_image" $arch_images | tee -a logs/manifest-creation.log
+              docker manifest push "$base_image" | tee -a logs/manifest-creation.log
+            else
+              echo "No images found for $base_image, skipping." | tee -a logs/manifest-creation.log
+            fi
+          done
+
+      - name: Upload manifest logs
+        uses: actions/upload-artifact@v4
+        with:
+          name: manifest-logs
+          path: |
+            all-pushed-images.txt
+            logs/manifest-creation.log
+
       # NOTE(mgoddard): Trigger another CI workflow in the
       # stackhpc-release-train repository.
       - name: Trigger container image repository sync

etc/kayobe/environments/ci-aio/stackhpc-ci.yml

@@ -23,23 +23,23 @@ stackhpc_repo_mirror_password: !vault |
 
 # Build against released Pulp repository versions.
 stackhpc_repo_grafana_version: "{{ stackhpc_pulp_repo_grafana_version }}"
-stackhpc_repo_rhel9_rabbitmq_erlang_version: "{{ stackhpc_pulp_repo_rhel9_rabbitmq_erlang_version }}"
+stackhpc_repo_rhel9_rabbitmq_erlang_version: "{{ stackhpc_pulp_repo_multiarch_rhel9_rabbitmq_erlang_version }}"
 stackhpc_repo_rhel9_rabbitmq_server_version: "{{ stackhpc_pulp_repo_rhel9_rabbitmq_server_version }}"
 stackhpc_repo_ubuntu_jammy_version: "{{ stackhpc_pulp_repo_ubuntu_jammy_version }}"
 stackhpc_repo_ubuntu_jammy_security_version: "{{ stackhpc_pulp_repo_ubuntu_jammy_security_version }}"
 stackhpc_repo_ubuntu_jammy_cve_2024_6387_version: ""
 stackhpc_repo_ubuntu_cloud_archive_version: "{{ stackhpc_pulp_repo_ubuntu_cloud_archive_version }}"
 stackhpc_repo_docker_ce_ubuntu_jammy_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_jammy_version }}"
 stackhpc_repo_ceph_reef_debian_version: "{{ stackhpc_pulp_repo_ceph_reef_debian_version }}"
-stackhpc_repo_centos_stream_9_nfv_openvswitch_version: "{{ stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch_version }}"
-stackhpc_repo_centos_stream_9_openstack_caracal_version: "{{ stackhpc_pulp_repo_centos_stream_9_openstack_caracal_version }}"
-stackhpc_repo_centos_stream_9_opstools_version: "{{ stackhpc_pulp_repo_centos_stream_9_opstools_version }}"
-stackhpc_repo_centos_stream_9_storage_ceph_reef_version: "{{ stackhpc_pulp_repo_centos_stream_9_storage_ceph_reef_version }}"
-stackhpc_repo_centos_stream_9_docker_version: "{{ stackhpc_pulp_repo_centos_stream_9_docker_version }}"
-stackhpc_repo_rhel_9_treasuredata_5_version: "{{ stackhpc_pulp_repo_rhel_9_treasuredata_5_version }}"
-stackhpc_repo_rhel_9_mariadb_10_11_version: "{{ stackhpc_pulp_repo_rhel_9_mariadb_10_11_version }}"
-stackhpc_repo_rhel_9_influxdb_version: "{{ stackhpc_pulp_repo_rhel_9_influxdb_version }}"
-stackhpc_repo_epel_9_version: "{{ stackhpc_pulp_repo_epel_9_version }}"
+stackhpc_repo_centos_stream_9_nfv_openvswitch_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_nfv_openvswitch_version }}"
+stackhpc_repo_centos_stream_9_openstack_caracal_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_openstack_caracal_version }}"
+stackhpc_repo_centos_stream_9_opstools_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_opstools_version }}"
+stackhpc_repo_centos_stream_9_storage_ceph_reef_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_storage_ceph_reef_version }}"
+stackhpc_repo_centos_stream_9_docker_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_docker_version }}"
+stackhpc_repo_rhel_9_treasuredata_5_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_treasuredata_5_version }}"
+stackhpc_repo_rhel_9_mariadb_10_11_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_mariadb_10_11_version }}"
+stackhpc_repo_rhel_9_influxdb_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_influxdb_version }}"
+stackhpc_repo_epel_9_version: "{{ stackhpc_pulp_repo_multiarch_epel_9_version }}"
 stackhpc_repo_opensearch_2_x_version: "{{ stackhpc_pulp_repo_opensearch_2_x_version }}"
 stackhpc_repo_opensearch_dashboards_2_x_version: "{{ stackhpc_pulp_repo_opensearch_dashboards_2_x_version }}"
 ## Use derived vars from etc/kayobe/pulp.yml to switch between
@@ -49,7 +49,7 @@ stackhpc_repo_rocky_9_appstream_version: "{{ stackhpc_pulp_repo_rocky_9_appstrea
 stackhpc_repo_rocky_9_extras_version: "{{ stackhpc_pulp_repo_rocky_9_extras_version }}"
 stackhpc_repo_rocky_9_crb_version: "{{ stackhpc_pulp_repo_rocky_9_crb_version }}"
 stackhpc_repo_rocky_9_highavailability_version: "{{ stackhpc_pulp_repo_rocky_9_highavailability_version }}"
-stackhpc_repo_rocky_9_sig_security_common_version: "{{ stackhpc_pulp_repo_rocky_9_sig_security_common_version }}"
+stackhpc_repo_rocky_9_sig_security_common_version: "{{ stackhpc_pulp_repo_multiarch_rocky_9_sig_security_common_version }}"
 
 # Rocky-and-CI-specific Pulp urls
 stackhpc_include_os_minor_version_in_repo_url: true

etc/kayobe/environments/ci-builder/stackhpc-ci.yml

@@ -48,33 +48,34 @@ stackhpc_repo_mirror_password: !vault |
 
 # Build against released Pulp repository versions.
 stackhpc_repo_grafana_version: "{{ stackhpc_pulp_repo_grafana_version }}"
-stackhpc_repo_rhel9_rabbitmq_erlang_version: "{{ stackhpc_pulp_repo_rhel9_rabbitmq_erlang_version }}"
+stackhpc_repo_rhel9_rabbitmq_erlang_version: "{{ stackhpc_pulp_repo_multiarch_rhel9_rabbitmq_erlang_version }}"
 stackhpc_repo_rhel9_rabbitmq_server_version: "{{ stackhpc_pulp_repo_rhel9_rabbitmq_server_version }}"
 stackhpc_repo_ubuntu_jammy_version: "{{ stackhpc_pulp_repo_ubuntu_jammy_version }}"
 stackhpc_repo_ubuntu_jammy_security_version: "{{ stackhpc_pulp_repo_ubuntu_jammy_security_version }}"
 stackhpc_repo_ubuntu_jammy_cve_2024_6387_version: ""
 stackhpc_repo_ubuntu_cloud_archive_version: "{{ stackhpc_pulp_repo_ubuntu_cloud_archive_version }}"
 stackhpc_repo_docker_ce_ubuntu_jammy_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_jammy_version }}"
 stackhpc_repo_ceph_reef_debian_version: "{{ stackhpc_pulp_repo_ceph_reef_debian_version }}"
-stackhpc_repo_centos_stream_9_nfv_openvswitch_version: "{{ stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch_version }}"
-stackhpc_repo_centos_stream_9_openstack_caracal_version: "{{ stackhpc_pulp_repo_centos_stream_9_openstack_caracal_version }}"
-stackhpc_repo_centos_stream_9_opstools_version: "{{ stackhpc_pulp_repo_centos_stream_9_opstools_version }}"
-stackhpc_repo_centos_stream_9_storage_ceph_reef_version: "{{ stackhpc_pulp_repo_centos_stream_9_storage_ceph_reef_version }}"
-stackhpc_repo_centos_stream_9_docker_version: "{{ stackhpc_pulp_repo_centos_stream_9_docker_version }}"
-stackhpc_repo_rhel_9_treasuredata_5_version: "{{ stackhpc_pulp_repo_rhel_9_treasuredata_5_version }}"
-stackhpc_repo_rhel_9_mariadb_10_11_version: "{{ stackhpc_pulp_repo_rhel_9_mariadb_10_11_version }}"
-stackhpc_repo_rhel_9_influxdb_version: "{{ stackhpc_pulp_repo_rhel_9_influxdb_version }}"
-stackhpc_repo_epel_9_version: "{{ stackhpc_pulp_repo_epel_9_version }}"
+stackhpc_repo_centos_stream_9_nfv_openvswitch_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_nfv_openvswitch_version }}"
+stackhpc_repo_centos_stream_9_openstack_caracal_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_openstack_caracal_version }}"
+stackhpc_repo_centos_stream_9_opstools_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_opstools_version }}"
+stackhpc_repo_centos_stream_9_storage_ceph_reef_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_storage_ceph_reef_version }}"
+stackhpc_repo_centos_stream_9_docker_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_docker_version }}"
+stackhpc_repo_rhel_9_treasuredata_5_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_treasuredata_5_version }}"
+stackhpc_repo_rhel_9_mariadb_10_11_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_mariadb_10_11_version }}"
+stackhpc_repo_rhel_9_influxdb_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_influxdb_version }}"
+stackhpc_repo_epel_9_version: "{{ stackhpc_pulp_repo_multiarch_epel_9_version }}"
 stackhpc_repo_opensearch_2_x_version: "{{ stackhpc_pulp_repo_opensearch_2_x_version }}"
 stackhpc_repo_opensearch_dashboards_2_x_version: "{{ stackhpc_pulp_repo_opensearch_dashboards_2_x_version }}"
+
 ## Use derived vars from etc/kayobe/pulp.yml to switch between
 ## minor Rocky versions using stackhpc_pulp_repo_rocky_x_minor_version
 stackhpc_repo_rocky_9_baseos_version: "{{ stackhpc_pulp_repo_rocky_9_baseos_version }}"
 stackhpc_repo_rocky_9_appstream_version: "{{ stackhpc_pulp_repo_rocky_9_appstream_version }}"
 stackhpc_repo_rocky_9_extras_version: "{{ stackhpc_pulp_repo_rocky_9_extras_version }}"
 stackhpc_repo_rocky_9_crb_version: "{{ stackhpc_pulp_repo_rocky_9_crb_version }}"
 stackhpc_repo_rocky_9_highavailability_version: "{{ stackhpc_pulp_repo_rocky_9_highavailability_version }}"
-stackhpc_repo_rocky_9_sig_security_common_version: "{{ stackhpc_pulp_repo_rocky_9_sig_security_common_version }}"
+stackhpc_repo_rocky_9_sig_security_common_version: "{{ stackhpc_pulp_repo_multiarch_rocky_9_sig_security_common_version }}"
 
 # Rocky-and-CI-specific Pulp urls
 stackhpc_include_os_minor_version_in_repo_url: true