Merge branch 'stackhpc/2024.1' into cross-arch-builds-2024.1

.automation.conf/config.sh

@@ -3,7 +3,14 @@
 
 # See: https://github.com/stackhpc/docker-rally/blob/master/bin/rally-verify-wrapper.sh for a full list of tempest parameters that can be overriden.
 # You can override tempest parameters like so:
-export TEMPEST_CONCURRENCY=2
+
+# The Tempest concurrency determines how many tests can be running at once.
+# Higher values run tests faster but risk running out of resources and failing tests
+# On production systems, Tempest concurrency can usually be set to a high number e.g. 16-64. It is often limited by the number of available floating IPs.
+# On virtualised test environments, compute and networking speeds often limit the concurrency to 1-16 before tests begin to fail due to timeouts.
+export TEMPEST_CONCURRENCY=16
+
+
 # Specify single test whilst experimenting
 #export TEMPEST_PATTERN="${TEMPEST_PATTERN:-tempest.api.compute.servers.test_create_server.ServersTestJSON.test_host_name_is_same_as_server_name}"
 
@@ -21,9 +28,7 @@ if [ ! -z ${KAYOBE_ENVIRONMENT:+x} ]; then
   fi
 
   if [[ "$KAYOBE_ENVIRONMENT" =~ "ci-multinode" ]]; then
-    # SMSLab is currently running with 1G switches. This causes tests using volumes and images to fail if
-    # the concurrency is set too high.
-    export TEMPEST_CONCURRENCY=1
+    export TEMPEST_CONCURRENCY=4
     # Uncomment this to perform a full tempest test
     # export KAYOBE_AUTOMATION_TEMPEST_LOADLIST=tempest-full
     # export KAYOBE_AUTOMATION_TEMPEST_SKIPLIST=ci-multinode-tempest-full

.github/workflows/stackhpc-all-in-one.yml

@@ -88,7 +88,7 @@ jobs:
       - name: Install Package
         uses: ConorMacBride/install-package@main
         with:
-          apt: git unzip nodejs
+          apt: git unzip nodejs openssh-client
 
       # If testing upgrade, checkout previous release, otherwise checkout current branch
       - name: Checkout ${{ inputs.upgrade && 'previous release' || 'current' }} config
@@ -223,6 +223,7 @@ jobs:
           admin_bootproto: dhcp
           admin_ips:
             controller0: "{{ access_ip_v4.value }}"
+          admin_zone: admin
           EOF
 
       - name: Write Terraform network interface config
@@ -443,7 +444,7 @@ jobs:
             -v $(pwd)/tempest-artifacts:/stack/tempest-artifacts \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
             $KAYOBE_IMAGE \
-            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/tempest.sh -e ansible_user=stack -e rally_no_sensitive_log=false
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/tempest.sh -e ansible_user=stack
         env:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
 

.github/workflows/stackhpc-build-kayobe-image.yml

@@ -98,3 +98,25 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+
+      - name: Send message to Slack via Workflow Builder
+        uses: slackapi/[email protected]
+        with:
+          payload: |
+            {
+              "channel-id": "${{ env.SLACK_CHANNEL_ID }}",
+              "inputs": "${{ env.INPUTS }}",
+              "message": "${{ env.MESSAGE }}",
+              "results-url": "${{ env.RESULTS_URL }}",
+              "workflow-url": "${{ env.WORKFLOW_URL }}"
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+          # #release-train-alerts
+          SLACK_CHANNEL_ID: C03B28HRP53
+          INPUTS: >-
+            branch: ${{ github.ref_name }}
+          MESSAGE: "SKC Build Kayobe Image workflow failed :sob:"
+          RESULTS_URL: "N/A"
+          WORKFLOW_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+        if: failure() && github.event_name == 'push'

.github/workflows/stackhpc-ci-cleanup.yml

@@ -75,3 +75,24 @@ jobs:
           OS_CLOUD: openstack
           OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
+
+      - name: Send message to Slack via Workflow Builder
+        uses: slackapi/[email protected]
+        with:
+          payload: |
+            {
+              "channel-id": "${{ env.SLACK_CHANNEL_ID }}",
+              "inputs": "${{ env.INPUTS }}",
+              "message": "${{ env.MESSAGE }}",
+              "results-url": "${{ env.RESULTS_URL }}",
+              "workflow-url": "${{ env.WORKFLOW_URL }}"
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+          # #release-train-alerts
+          SLACK_CHANNEL_ID: C03B28HRP53
+          INPUTS: "N/A"
+          MESSAGE: "SKC CI Cleanup workflow failed :sob:"
+          RESULTS_URL: "N/A"
+          WORKFLOW_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+        if: failure()

.github/workflows/stackhpc-container-image-build.yml

@@ -71,20 +71,21 @@ jobs:
       # Dynamically define job matrix.
       # We need a separate matrix entry for each distribution, when the relevant input is true.
       # https://stackoverflow.com/questions/65384420/how-do-i-make-a-github-action-matrix-element-conditional
+      # For now include only RL9 in aarch64
       - name: Generate build matrix
         id: set-matrix
         run: |
-          echo -n "matrix={\"distro\": [" >> $GITHUB_OUTPUT
+          echo -n "matrix={\"include\": [" >> $GITHUB_OUTPUT
           comma=""
-          if [[ ${{ inputs.rocky-linux-9 }} == 'true' ]]; then
-            echo -n "$comma\"rocky\"" >> $GITHUB_OUTPUT
+          if [[ '${{ inputs.rocky-linux-9 }}' == 'true' ]]; then
+            echo -n "$comma{\"distro\": \"rocky\", \"arch\": \"amd64\"}" >> $GITHUB_OUTPUT
             comma=", "
+            echo -n "$comma{\"distro\": \"rocky\", \"arch\": \"aarch64\"}" >> $GITHUB_OUTPUT
           fi
-          if [[ ${{ inputs.ubuntu-jammy }} == 'true' ]]; then
-            echo -n "$comma\"ubuntu\"" >> $GITHUB_OUTPUT
-            comma=", "
+          if [[ '${{ inputs.ubuntu-jammy }}' == 'true' ]]; then
+            echo -n "$comma{\"distro\": \"ubuntu\", \"arch\": \"amd64\"}" >> $GITHUB_OUTPUT
           fi
-          echo "], \"arch\": [\"amd64\", \"aarch64\"]}" >> $GITHUB_OUTPUT
+          echo "]}" >> $GITHUB_OUTPUT
 
       - name: Display container datetime tag
         run: |
@@ -99,10 +100,6 @@ jobs:
     strategy:
       fail-fast: false
       matrix: ${{ fromJson(needs.generate-tag.outputs.matrix) }}
-      # Exclude ubuntu aarch64 builds for now
-      exclude:
-        - distro: ubuntu
-          arch: aarch64
     needs:
       - generate-tag
     steps:
@@ -192,7 +189,7 @@ jobs:
             args="$args -e kolla_base_arch=${{ matrix.arch }}"
           fi
           args="$args -e kolla_base_distro=${{ matrix.distro }}"
-          args="$args -e kolla_tag=${{ steps.write-kolla-tag.outputs.kolla-tag }}
+          args="$args -e kolla_tag=${{ steps.write-kolla-tag.outputs.kolla-tag }}"
           args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
@@ -232,7 +229,7 @@ jobs:
 
       - name: Copy seed container image build logs to output directory
         run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-seed.log
-        if: inputs.seed
+        if: inputs.seed && matrix.arch == 'amd64'
 
       - name: Get built container images
         run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}*" > ${{ matrix.distro }}-container-images
@@ -320,6 +317,116 @@ jobs:
         run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/critical-images.txt && exit 1; fi
         if: ${{ !inputs.push-dirty && !cancelled() }}
 
+  create-manifests:
+    # Only for Rocky Linux for now
+    name: Create Docker Manifests
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push
+    runs-on: arc-skc-container-image-builder-runner
+    permissions: {}
+    needs:
+      - container-image-build
+    steps:
+      - name: Install package dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y git unzip python3-wheel python3-pip python3-venv curl jq wget openssh-server openssh-client
+
+      - name: Install gh
+        run: |
+          sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null
+          sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
+          sudo apt update
+          sudo apt install gh -y
+
+      - name: Checkout Kayobe Config
+        uses: actions/checkout@v4
+        with:
+          path: src/kayobe-config
+
+      - name: Install Kayobe
+        run: |
+          mkdir -p venvs &&
+          pushd venvs &&
+          python3 -m venv kayobe &&
+          source kayobe/bin/activate &&
+          pip install -U pip &&
+          pip install -r ../src/kayobe-config/requirements.txt
+
+      # Required for Pulp auth proxy deployment and Docker registry login.
+      # Normally installed during host configure.
+      - name: Install Docker Python SDK
+        run: |
+          sudo pip install docker
+
+      - name: Configure localhost as a seed
+        run: |
+          cat > src/kayobe-config/etc/kayobe/environments/ci-builder/inventory/hosts << EOF
+          # A 'seed' host used for building images.
+          # Use localhost for container image builds.
+          [seed]
+          localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python3
+          EOF
+
+      # See etc/kayobe/ansible/roles/pulp_auth_proxy/README.md for details.
+      # NOTE: We override pulp_auth_proxy_conf_path to a path shared by the
+      # runner and dind containers.
+      - name: Deploy an authenticating package repository mirror proxy
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-auth-proxy.yml -e pulp_auth_proxy_conf_path=/home/runner/_work/pulp_proxy
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+
+      - name: Combine pushed images lists
+        run: |
+          find . -name 'push-attempt-images.txt' -exec cat {} + > all-pushed-images.txt
+
+      - name: Log in to Docker registry
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/docker-registry-login.yml
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+
+      - name: Create and Push Docker Manifests
+        run: |
+          set -ex
+          mkdir -p logs
+          images=$(cat all-pushed-images.txt | sort | uniq)
+          # Filter out Ubuntu images
+          manifest_images=$(echo "$images" | grep 'rocky' | sed -E 's/-(amd64|aarch64)$//' | sort | uniq)
+          for base_image in $manifest_images; do
+            arch_images=""
+            for arch in amd64 aarch64; do
+              arch_image="${base_image}-${arch}"
+              # Check if the image exists in the registry
+              if docker manifest inspect "$arch_image" > /dev/null 2>&1; then
+                arch_images="$arch_images $arch_image"
+              fi
+            done
+            if [ -n "$arch_images" ]; then
+              echo "Creating manifest for $base_image with images:$arch_images" | tee -a logs/manifest-creation.log
+              docker manifest create "$base_image" $arch_images | tee -a logs/manifest-creation.log
+              docker manifest push "$base_image" | tee -a logs/manifest-creation.log
+            else
+              echo "No images found for $base_image, skipping." | tee -a logs/manifest-creation.log
+            fi
+          done
+
+      - name: Upload manifest logs
+        uses: actions/upload-artifact@v4
+        with:
+          name: manifest-logs
+          path: |
+            all-pushed-images.txt
+            logs/manifest-creation.log
+
       # NOTE(mgoddard): Trigger another CI workflow in the
       # stackhpc-release-train repository.
       - name: Trigger container image repository sync

.github/workflows/stackhpc-multinode-periodic.yml

@@ -35,7 +35,7 @@ jobs:
     name: Multinode periodic
     needs:
       - generate-inputs
-    uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/multinode.yml@1.1.0
+    uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/multinode.yml@1.2.0
     with:
       multinode_name: mn-prdc-${{ github.run_id }}
       os_distribution: ${{ needs.generate-inputs.outputs.os_distribution }}

.github/workflows/stackhpc-multinode.yml

@@ -52,7 +52,7 @@ name: Multinode
 jobs:
   multinode:
     name: Multinode
-    uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/multinode.yml@1.1.0
+    uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/multinode.yml@1.2.0
     with:
       multinode_name: ${{ inputs.multinode_name }}
       os_distribution: ${{ inputs.os_distribution }}

.github/workflows/stackhpc-promote.yml

@@ -43,3 +43,25 @@ jobs:
       - name: Display link to container image promotion workflows
         run: |
           echo "::notice Container image promote workflow: https://github.com/stackhpc/stackhpc-release-train/actions/workflows/container-promote.yml"
+
+      - name: Send message to Slack via Workflow Builder
+        uses: slackapi/[email protected]
+        with:
+          payload: |
+            {
+              "channel-id": "${{ env.SLACK_CHANNEL_ID }}",
+              "inputs": "${{ env.INPUTS }}",
+              "message": "${{ env.MESSAGE }}",
+              "results-url": "${{ env.RESULTS_URL }}",
+              "workflow-url": "${{ env.WORKFLOW_URL }}"
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+          # #release-train-alerts
+          SLACK_CHANNEL_ID: C03B28HRP53
+          INPUTS: >-
+            branch: ${{ github.ref_name }}
+          MESSAGE: "SKC promote workflow failed :sob:"
+          RESULTS_URL: "N/A"
+          WORKFLOW_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+        if: failure()

doc/source/configuration/cephadm.rst

@@ -342,6 +342,8 @@ should be used in the Kolla Manila configuration e.g.:
 
   manila_cephfs_filesystem_name: manila-cephfs
 
+.. _RGWs-With-Ceph:
+
 RADOS Gateways
 --------------