Make Repro-CI Artifacts Reflect Configs Repo Structure

.github/workflows/config-pr-1-ci.yml

@@ -17,19 +17,33 @@ on:
       additional-artifact-content-paths:
         type: string
         required: false
-        # For example, the value of 'checksum' will expand to something like
-        # '/scratch/tm70/repro-ci/experiments/MODEL-configs/dev-CONFIG/checksum'
+        # For example, the value of 'testing/checksum' will expand to something like
+        # '/scratch/tm70/repro-ci/experiments/MODEL-configs/dev-CONFIG/testing/checksum'
         # on the remote.
         description: |
-          Newline-separated paths for inclusion in the release artifact.
+          Newline-separated paths for inclusion in an 'additional-*' artifact.
           Note that all paths given have 'env.EXPERIMENT_LOCATION/' prepended.
+      additional-artifact-content-paths-root:
+        type: string
+        required: false
+        # default: technically '.', but we don't want a broken rsync recieve path like 'some/dir/.'
+        description: |
+          If content paths are being included in an additional artifact.
+          A path relative to the artifact root to store the given additional artifact content paths.
+          Defaults to the root (`.`) of the artifact.
     outputs:
       repro-ci-artifact-name:
         value: ${{ jobs.repro-ci.outputs.artifact-name }}
         description: Name of the repro-ci artifact
       repro-ci-artifact-url:
         value: ${{ jobs.repro-ci.outputs.artifact-url }}
         description: URL to the repro-ci artifact
+      repro-ci-additional-artifact-name:
+        value: ${{ jobs.repro-ci.outputs.additional-artifact-name }}
+        description: Name of the additional content repro-ci artifact
+      repro-ci-additional-artifact-url:
+        value: ${{ jobs.repro-ci.outputs.additional-artifact-url }}
+        description: URL to the additional content repro-ci artifact
       repro-ci-experiment-location:
         value: ${{ jobs.repro-ci.outputs.experiment-location }}
         description: Location of the experiment on the target environment
@@ -227,6 +241,7 @@ jobs:
       model-config-tests-version: ${{ needs.config.outputs.repro-model-config-tests-version }}
       payu-version: ${{ needs.config.outputs.repro-payu-version }}
       additional-artifact-content-paths: ${{ inputs.additional-artifact-content-paths }}
+      additional-artifact-content-paths-root: ${{ inputs.additional-artifact-content-paths-root }}
     secrets: inherit
     permissions:
       contents: write
@@ -257,7 +272,7 @@ jobs:
         id: tests
         uses: EnricoMi/publish-unit-test-result-action/composite@e780361cd1fc1b1a170624547b3ffda64787d365  #v2.12.0
         with:
-          files: ${{ env.TESTING_LOCAL_LOCATION }}/checksum/test_report.xml
+          files: ${{ env.TESTING_LOCAL_LOCATION }}/testing/checksum/test_report.xml
           comment_mode: off
           check_run: true
           check_name: Repro Test Results

.github/workflows/config-pr-2-confirm.yml

@@ -6,6 +6,14 @@
 name: Confirm
 on:
   workflow_call:
+    inputs:
+      additional-artifact-content-paths-commit:
+        type: boolean
+        required: false
+        default: false
+        description: |
+          If additional content paths have been included in an artifact.
+          Whether to commit the additional content.
   # Workflows that call this workflow use the following triggers:
   # issue_comment:
   #   types:
@@ -128,10 +136,19 @@ jobs:
             --jq='[.[] | select(.headBranch == "${{ steps.pr.outputs.branch }}" and .status == "completed")] | sort_by(.updatedAt) | last | .databaseId')
           gh run download $associated_run -D ${{ env.ARTIFACT_LOCAL_LOCATION }}
 
-      - name: Update metadata.yaml and Checksum files
+      - name: Update metadata.yaml and add checksums
+        # We have the strange double asterix in the `cp` because
+        # `gh run download` may download multiple artifacts if they exist.
         run: |
           yq -i '.version = "${{ needs.bump-version.outputs.after }}"' metadata.yaml
-          cp --recursive --verbose ${{ env.ARTIFACT_LOCAL_LOCATION }}/*/* testing
+          rsync --recursive --verbose --exclude 'additional-*' ${{ env.ARTIFACT_LOCAL_LOCATION }} .
+
+      - name: Add additional files
+        if: inputs.additional-artifact-content-paths-commit
+        # Note: This copies over (and commits) files that are
+        # specified by config-pr-1-ci.ymls inputs.additional-artifact-content-paths
+        run: |
+          rsync --recursive --verbose ${{ env.ARTIFACT_LOCAL_LOCATION }}/additional-*/* .
 
       - name: Import Commit-Signing Key
         uses: crazy-max/ghaction-import-gpg@01dd5d3ca463c7f10f7f4f7b4f177225ac661ee4  # v6.1.0
@@ -146,12 +163,19 @@ jobs:
           git_tag_gpgsign: true
 
       - name: Commit and Push Updates
+        # In this, we add various sections to the commit message depending if
+        # checksums or additional content  are added, along with the version bump.
+        env:
+          # We interleave the optional bits between the 'Bumped version...' bit
+          # because stripped block scalars (>-) put in a newline if there are two newlines.
+          COMMIT_MSG: >-
+            ${{ needs.bump-version.outputs.type == 'major' && 'Updated checksums, ' || '' }}
+            Bumped version to ${{ needs.bump-version.outputs.after }}
+            ${{ inputs.additional-artifact-content-paths-commit && ', and updated files' || '' }}
+            as part of ${{ env.RUN_URL }}
         run: |
-          if [[ "${{ needs.bump-version.outputs.type }}" == "minor" ]]; then
-            git commit -am "Bumped version to ${{ needs.bump-version.outputs.after }} as part of ${{ env.RUN_URL }}"
-          elif [[ "${{ needs.bump-version.outputs.type }}" == "major" ]]; then
-            git commit -am "Updated checksums and bumped version to ${{ needs.bump-version.outputs.after }} as part of ${{ env.RUN_URL }}"
-          fi
+          git add .
+          git commit -m "${{ env.COMMIT_MSG }}"
           git push
 
       - name: Comment Success

.github/workflows/config-schedule-2-start.yml

@@ -80,7 +80,7 @@ jobs:
         id: tests
         uses: EnricoMi/publish-unit-test-result-action/composite@e780361cd1fc1b1a170624547b3ffda64787d365  #v2.12.0
         with:
-          files: ${{ env.TESTING_LOCAL_LOCATION }}/checksum/test_report.xml
+          files: ${{ env.TESTING_LOCAL_LOCATION }}/testing/checksum/test_report.xml
           comment_mode: off
           check_run: true
           compare_to_earlier_commit: false

.github/workflows/generate-checksums.yml

@@ -10,11 +10,6 @@ on:
         type: boolean
         required: true
         description: Whether to commit the checksums to the config branch once generated.
-      committed-checksum-location:
-        type: string
-        required: false
-        default: ./testing/checksum
-        description: "If checksums are committed: Where in the repository the generated checksums should be committed to."
       committed-checksum-tag:
         type: string
         required: false
@@ -90,6 +85,7 @@ jobs:
         id: run
         env:
           BASE_EXPERIMENT_LOCATION: ${{ env.EXPERIMENT_LOCATION }}/base-experiment
+          CHECKSUM_LOCATION: ${{ env.EXPERIMENT_LOCATION }}/testing
           TEST_VENV_LOCATION: ${{ env.EXPERIMENT_LOCATION }}/test-venv
         run: |
           ssh ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash<<EOT
@@ -131,7 +127,7 @@ jobs:
           # Run pytests - this also generates checksums files
           model-config-tests -s \
             -m "checksum" \
-            --output-path ${{ env.EXPERIMENT_LOCATION }}
+            --output-path "${{ env.CHECKSUM_LOCATION }}"
 
           # Deactivate and remove the test virtual environment
           deactivate
@@ -143,12 +139,13 @@ jobs:
           EOT
 
           echo "experiment-location=${{ env.EXPERIMENT_LOCATION }}" >> $GITHUB_OUTPUT
+          echo "checksum-location=${{ env.CHECKSUM_LOCATION }}" >> $GITHUB_OUTPUT
           echo "::notice::Checksums generated on ${{ vars.DEPLOYMENT_TARGET }} at ${{ env.EXPERIMENT_LOCATION }}"
 
       - name: Copy Back Checksums
         run: |
           rsync --recursive -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \
-              '${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST_DATA }}:${{ env.EXPERIMENT_LOCATION }}/checksum' \
+              '${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST_DATA }}:${{ steps.run.outputs.checksum-location }}' \
               ${{ env.OUTPUT_LOCAL_LOCATION }}
 
       - name: Generate Output Artifact Name
@@ -181,10 +178,8 @@ jobs:
           name: ${{ needs.generate-checksum.outputs.artifact-name }}
           path: ${{ env.OUTPUT_LOCAL_LOCATION }}
 
-      - name: Move Checksums to Repo
-        run: |
-          mkdir -p ${{ inputs.committed-checksum-location }}
-          mv ${{ env.OUTPUT_LOCAL_LOCATION }}/checksum/* ${{ inputs.committed-checksum-location }}
+      - name: Move Outputs to Repo
+        run: mv ${{ env.OUTPUT_LOCAL_LOCATION }} .
 
       - name: Update version in metadata.yaml
         if: inputs.committed-checksum-tag != ''

.github/workflows/test-repro.yml

@@ -35,13 +35,27 @@ on:
         description: |
           Newline-separated paths for inclusion in the release artifact.
           Note that all paths given have 'env.EXPERIMENT_LOCATION/' prepended.
+      additional-artifact-content-paths-root:
+        type: string
+        required: false
+        # default: technically '.', but we don't want a broken rsync recieve path like 'some/dir/.'
+        description: |
+          If additional content paths are being included in the artifact.
+          A path relative to the artifact root to store the given additional artifact content paths.
+          Defaults to the root (`.`) of the artifact.
     outputs:
       artifact-name:
         value: ${{ jobs.repro.outputs.artifact-name }}
         description: Name of the artifact containing the checksums and test report for this repro run
       artifact-url:
         value: ${{ jobs.repro.outputs.artifact-url }}
         description: URL to the artifact containing the checksums and test report for this repro run
+      additional-artifact-name:
+        value: ${{ jobs.additional-artifact.outputs.artifact-name }}
+        description: If inputs.additional-artifact-content was specified - Name of the artifact containing additional content for this repro run
+      additional-artifact-url:
+        value: ${{ jobs.additional-artifact.outputs.artifact-url }}
+        description: If inputs.additional-artifact-content was specified - URL to the artifact containing additional content for this repro run
       experiment-location:
         value: ${{ jobs.repro.outputs.experiment-location }}
         description: Location of the experiment on the target environment
@@ -96,6 +110,7 @@ jobs:
         id: run
         env:
           BASE_EXPERIMENT_LOCATION: ${{ env.EXPERIMENT_LOCATION }}/base-experiment
+          CHECKSUM_LOCATION: ${{ env.EXPERIMENT_LOCATION }}/testing
           COMPARED_CHECKSUM_LOCATION: ${{ env.EXPERIMENT_LOCATION }}/compared
           TEST_VENV_LOCATION: ${{ env.EXPERIMENT_LOCATION }}/test-venv
         run: |
@@ -152,9 +167,9 @@ jobs:
           # Run model-config-tests pytests - this also generates checksums files.
           # If there is a checksum to compare against, add --checksum-path arg.
           model-config-tests -s -m "${{ inputs.test-markers }}" \
-            --output-path "${{ env.EXPERIMENT_LOCATION }}" \
+            --output-path "${{ env.CHECKSUM_LOCATION }}" \
             ${{ inputs.compared-config-ref != '' && '--checksum-path $COMPARED_CHECKSUM_FILE \' || '\' }}
-            --junitxml="${{ env.EXPERIMENT_LOCATION }}/checksum/test_report.xml"
+            --junitxml="${{ env.CHECKSUM_LOCATION }}/checksum/test_report.xml"
 
           # Deactivate and remove the test virtual environment
           deactivate
@@ -165,31 +180,63 @@ jobs:
           exit 0
           EOT
           echo "experiment-location=${{ env.EXPERIMENT_LOCATION }}" >> $GITHUB_OUTPUT
+          echo "checksum-location=${{ env.CHECKSUM_LOCATION }}" >> $GITHUB_OUTPUT
 
       - name: Copy Back Checksums and Test Report
         run: |
           rsync --recursive --verbose -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \
-            '${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST_DATA }}:${{ env.EXPERIMENT_LOCATION }}/checksum' \
+            '${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST_DATA }}:${{ steps.run.outputs.checksum-location }}' \
             ${{ env.ARTIFACT_LOCAL_LOCATION }}
 
+      - name: Generate Test Output Artifact Name
+        id: artifact
+        run: echo "name=${{ github.event.repository.name }}-${{ inputs.config-ref }}" >> $GITHUB_OUTPUT
+
+      - name: Upload Test Output
+        id: upload
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ steps.artifact.outputs.name }}
+          if-no-files-found: error
+          path: ${{ env.ARTIFACT_LOCAL_LOCATION }}
+
+  additional-artifact:
+    name: Additional Content Artifact Upload
+    needs:
+      - repro
+    if: inputs.additional-artifact-content-paths != ''
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment-name }}
+    outputs:
+      artifact-name: ${{ steps.artifact.outputs.name }}
+      artifact-url: ${{ steps.upload.outputs.artifact-url }}
+    steps:
+      - name: Setup SSH
+        id: ssh
+        uses: access-nri/actions/.github/actions/setup-ssh@main
+        with:
+          hosts: |
+            ${{ secrets.SSH_HOST }}
+            ${{ secrets.SSH_HOST_DATA }}
+          private-key: ${{ secrets.SSH_KEY }}
+
       - name: Copy Back Additional Artifact Content
-        if: inputs.additional-artifact-content-paths != ''
         # For each of the additional artifact content paths, rsync the data
         # from the target into our runner for upload as an artifact
         run: |
           while IFS= read -r content_path; do
             if [[ -n "$content_path" ]]; then
-              rsync --recursive --relative --verbose -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \
-                "${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST_DATA }}:${{ env.EXPERIMENT_LOCATION }}/./$content_path" \
-                ${{ env.ARTIFACT_LOCAL_LOCATION }}
+              rsync --recursive --relative --mkpath --verbose -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \
+                "${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST_DATA }}:${{ needs.repro.outputs.experiment-location }}/./$content_path" \
+                ${{ env.ARTIFACT_LOCAL_LOCATION }}/${{ inputs.additional-artifact-content-paths-root }}
             fi
           done <<< "${{ inputs.additional-artifact-content-paths }}"
 
-      - name: Generate Test Output Artifact Name
+      - name: Generate Additional Content Artifact Name
         id: artifact
-        run: echo "name=${{ github.event.repository.name }}-${{ inputs.config-ref }}" >> $GITHUB_OUTPUT
+        run: echo "name=additional-${{ github.event.repository.name }}-${{ inputs.config-ref }}" >> $GITHUB_OUTPUT
 
-      - name: Upload Test Output
+      - name: Additional Content Artifact
         id: upload
         uses: actions/upload-artifact@v4
         with: