diff --git a/.github/workflows/config-pr-1-ci.yml b/.github/workflows/config-pr-1-ci.yml index b1adf88..ebca543 100644 --- a/.github/workflows/config-pr-1-ci.yml +++ b/.github/workflows/config-pr-1-ci.yml @@ -17,12 +17,20 @@ on: additional-artifact-content-paths: type: string required: false - # For example, the value of 'checksum' will expand to something like - # '/scratch/tm70/repro-ci/experiments/MODEL-configs/dev-CONFIG/checksum' + # For example, the value of 'testing/checksum' will expand to something like + # '/scratch/tm70/repro-ci/experiments/MODEL-configs/dev-CONFIG/testing/checksum' # on the remote. description: | - Newline-separated paths for inclusion in the release artifact. + Newline-separated paths for inclusion in an 'additional-*' artifact. Note that all paths given have 'env.EXPERIMENT_LOCATION/' prepended. + additional-artifact-content-paths-root: + type: string + required: false + # default: technically '.', but we don't want a broken rsync recieve path like 'some/dir/.' + description: | + If content paths are being included in an additional artifact. + A path relative to the artifact root to store the given additional artifact content paths. + Defaults to the root (`.`) of the artifact. outputs: repro-ci-artifact-name: value: ${{ jobs.repro-ci.outputs.artifact-name }} @@ -30,6 +38,12 @@ on: repro-ci-artifact-url: value: ${{ jobs.repro-ci.outputs.artifact-url }} description: URL to the repro-ci artifact + repro-ci-additional-artifact-name: + value: ${{ jobs.repro-ci.outputs.additional-artifact-name }} + description: Name of the additional content repro-ci artifact + repro-ci-additional-artifact-url: + value: ${{ jobs.repro-ci.outputs.additional-artifact-url }} + description: URL to the additional content repro-ci artifact repro-ci-experiment-location: value: ${{ jobs.repro-ci.outputs.experiment-location }} description: Location of the experiment on the target environment @@ -227,6 +241,7 @@ jobs: model-config-tests-version: ${{ needs.config.outputs.repro-model-config-tests-version }} payu-version: ${{ needs.config.outputs.repro-payu-version }} additional-artifact-content-paths: ${{ inputs.additional-artifact-content-paths }} + additional-artifact-content-paths-root: ${{ inputs.additional-artifact-content-paths-root }} secrets: inherit permissions: contents: write @@ -257,7 +272,7 @@ jobs: id: tests uses: EnricoMi/publish-unit-test-result-action/composite@e780361cd1fc1b1a170624547b3ffda64787d365 #v2.12.0 with: - files: ${{ env.TESTING_LOCAL_LOCATION }}/checksum/test_report.xml + files: ${{ env.TESTING_LOCAL_LOCATION }}/testing/checksum/test_report.xml comment_mode: off check_run: true check_name: Repro Test Results diff --git a/.github/workflows/config-pr-2-confirm.yml b/.github/workflows/config-pr-2-confirm.yml index 6b1c86a..4ac56e6 100644 --- a/.github/workflows/config-pr-2-confirm.yml +++ b/.github/workflows/config-pr-2-confirm.yml @@ -6,6 +6,14 @@ name: Confirm on: workflow_call: + inputs: + additional-artifact-content-paths-commit: + type: boolean + required: false + default: false + description: | + If additional content paths have been included in an artifact. + Whether to commit the additional content. # Workflows that call this workflow use the following triggers: # issue_comment: # types: @@ -128,10 +136,19 @@ jobs: --jq='[.[] | select(.headBranch == "${{ steps.pr.outputs.branch }}" and .status == "completed")] | sort_by(.updatedAt) | last | .databaseId') gh run download $associated_run -D ${{ env.ARTIFACT_LOCAL_LOCATION }} - - name: Update metadata.yaml and Checksum files + - name: Update metadata.yaml and add checksums + # We have the strange double asterix in the `cp` because + # `gh run download` may download multiple artifacts if they exist. run: | yq -i '.version = "${{ needs.bump-version.outputs.after }}"' metadata.yaml - cp --recursive --verbose ${{ env.ARTIFACT_LOCAL_LOCATION }}/*/* testing + rsync --recursive --verbose --exclude 'additional-*' ${{ env.ARTIFACT_LOCAL_LOCATION }} . + + - name: Add additional files + if: inputs.additional-artifact-content-paths-commit + # Note: This copies over (and commits) files that are + # specified by config-pr-1-ci.ymls inputs.additional-artifact-content-paths + run: | + rsync --recursive --verbose ${{ env.ARTIFACT_LOCAL_LOCATION }}/additional-*/* . - name: Import Commit-Signing Key uses: crazy-max/ghaction-import-gpg@01dd5d3ca463c7f10f7f4f7b4f177225ac661ee4 # v6.1.0 @@ -146,12 +163,19 @@ jobs: git_tag_gpgsign: true - name: Commit and Push Updates + # In this, we add various sections to the commit message depending if + # checksums or additional content are added, along with the version bump. + env: + # We interleave the optional bits between the 'Bumped version...' bit + # because stripped block scalars (>-) put in a newline if there are two newlines. + COMMIT_MSG: >- + ${{ needs.bump-version.outputs.type == 'major' && 'Updated checksums, ' || '' }} + Bumped version to ${{ needs.bump-version.outputs.after }} + ${{ inputs.additional-artifact-content-paths-commit && ', and updated files' || '' }} + as part of ${{ env.RUN_URL }} run: | - if [[ "${{ needs.bump-version.outputs.type }}" == "minor" ]]; then - git commit -am "Bumped version to ${{ needs.bump-version.outputs.after }} as part of ${{ env.RUN_URL }}" - elif [[ "${{ needs.bump-version.outputs.type }}" == "major" ]]; then - git commit -am "Updated checksums and bumped version to ${{ needs.bump-version.outputs.after }} as part of ${{ env.RUN_URL }}" - fi + git add . + git commit -m "${{ env.COMMIT_MSG }}" git push - name: Comment Success diff --git a/.github/workflows/config-schedule-2-start.yml b/.github/workflows/config-schedule-2-start.yml index f146bd0..7bbe32e 100644 --- a/.github/workflows/config-schedule-2-start.yml +++ b/.github/workflows/config-schedule-2-start.yml @@ -80,7 +80,7 @@ jobs: id: tests uses: EnricoMi/publish-unit-test-result-action/composite@e780361cd1fc1b1a170624547b3ffda64787d365 #v2.12.0 with: - files: ${{ env.TESTING_LOCAL_LOCATION }}/checksum/test_report.xml + files: ${{ env.TESTING_LOCAL_LOCATION }}/testing/checksum/test_report.xml comment_mode: off check_run: true compare_to_earlier_commit: false diff --git a/.github/workflows/generate-checksums.yml b/.github/workflows/generate-checksums.yml index fb61b2f..0891306 100644 --- a/.github/workflows/generate-checksums.yml +++ b/.github/workflows/generate-checksums.yml @@ -10,11 +10,6 @@ on: type: boolean required: true description: Whether to commit the checksums to the config branch once generated. - committed-checksum-location: - type: string - required: false - default: ./testing/checksum - description: "If checksums are committed: Where in the repository the generated checksums should be committed to." committed-checksum-tag: type: string required: false @@ -90,6 +85,7 @@ jobs: id: run env: BASE_EXPERIMENT_LOCATION: ${{ env.EXPERIMENT_LOCATION }}/base-experiment + CHECKSUM_LOCATION: ${{ env.EXPERIMENT_LOCATION }}/testing TEST_VENV_LOCATION: ${{ env.EXPERIMENT_LOCATION }}/test-venv run: | ssh ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash<> $GITHUB_OUTPUT + echo "checksum-location=${{ env.CHECKSUM_LOCATION }}" >> $GITHUB_OUTPUT echo "::notice::Checksums generated on ${{ vars.DEPLOYMENT_TARGET }} at ${{ env.EXPERIMENT_LOCATION }}" - name: Copy Back Checksums run: | rsync --recursive -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \ - '${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST_DATA }}:${{ env.EXPERIMENT_LOCATION }}/checksum' \ + '${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST_DATA }}:${{ steps.run.outputs.checksum-location }}' \ ${{ env.OUTPUT_LOCAL_LOCATION }} - name: Generate Output Artifact Name @@ -181,10 +178,8 @@ jobs: name: ${{ needs.generate-checksum.outputs.artifact-name }} path: ${{ env.OUTPUT_LOCAL_LOCATION }} - - name: Move Checksums to Repo - run: | - mkdir -p ${{ inputs.committed-checksum-location }} - mv ${{ env.OUTPUT_LOCAL_LOCATION }}/checksum/* ${{ inputs.committed-checksum-location }} + - name: Move Outputs to Repo + run: mv ${{ env.OUTPUT_LOCAL_LOCATION }} . - name: Update version in metadata.yaml if: inputs.committed-checksum-tag != '' diff --git a/.github/workflows/test-repro.yml b/.github/workflows/test-repro.yml index aa83e41..e33c49e 100644 --- a/.github/workflows/test-repro.yml +++ b/.github/workflows/test-repro.yml @@ -35,6 +35,14 @@ on: description: | Newline-separated paths for inclusion in the release artifact. Note that all paths given have 'env.EXPERIMENT_LOCATION/' prepended. + additional-artifact-content-paths-root: + type: string + required: false + # default: technically '.', but we don't want a broken rsync recieve path like 'some/dir/.' + description: | + If additional content paths are being included in the artifact. + A path relative to the artifact root to store the given additional artifact content paths. + Defaults to the root (`.`) of the artifact. outputs: artifact-name: value: ${{ jobs.repro.outputs.artifact-name }} @@ -42,6 +50,12 @@ on: artifact-url: value: ${{ jobs.repro.outputs.artifact-url }} description: URL to the artifact containing the checksums and test report for this repro run + additional-artifact-name: + value: ${{ jobs.additional-artifact.outputs.artifact-name }} + description: If inputs.additional-artifact-content was specified - Name of the artifact containing additional content for this repro run + additional-artifact-url: + value: ${{ jobs.additional-artifact.outputs.artifact-url }} + description: If inputs.additional-artifact-content was specified - URL to the artifact containing additional content for this repro run experiment-location: value: ${{ jobs.repro.outputs.experiment-location }} description: Location of the experiment on the target environment @@ -96,6 +110,7 @@ jobs: id: run env: BASE_EXPERIMENT_LOCATION: ${{ env.EXPERIMENT_LOCATION }}/base-experiment + CHECKSUM_LOCATION: ${{ env.EXPERIMENT_LOCATION }}/testing COMPARED_CHECKSUM_LOCATION: ${{ env.EXPERIMENT_LOCATION }}/compared TEST_VENV_LOCATION: ${{ env.EXPERIMENT_LOCATION }}/test-venv run: | @@ -152,9 +167,9 @@ jobs: # Run model-config-tests pytests - this also generates checksums files. # If there is a checksum to compare against, add --checksum-path arg. model-config-tests -s -m "${{ inputs.test-markers }}" \ - --output-path "${{ env.EXPERIMENT_LOCATION }}" \ + --output-path "${{ env.CHECKSUM_LOCATION }}" \ ${{ inputs.compared-config-ref != '' && '--checksum-path $COMPARED_CHECKSUM_FILE \' || '\' }} - --junitxml="${{ env.EXPERIMENT_LOCATION }}/checksum/test_report.xml" + --junitxml="${{ env.CHECKSUM_LOCATION }}/checksum/test_report.xml" # Deactivate and remove the test virtual environment deactivate @@ -165,31 +180,63 @@ jobs: exit 0 EOT echo "experiment-location=${{ env.EXPERIMENT_LOCATION }}" >> $GITHUB_OUTPUT + echo "checksum-location=${{ env.CHECKSUM_LOCATION }}" >> $GITHUB_OUTPUT - name: Copy Back Checksums and Test Report run: | rsync --recursive --verbose -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \ - '${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST_DATA }}:${{ env.EXPERIMENT_LOCATION }}/checksum' \ + '${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST_DATA }}:${{ steps.run.outputs.checksum-location }}' \ ${{ env.ARTIFACT_LOCAL_LOCATION }} + - name: Generate Test Output Artifact Name + id: artifact + run: echo "name=${{ github.event.repository.name }}-${{ inputs.config-ref }}" >> $GITHUB_OUTPUT + + - name: Upload Test Output + id: upload + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.artifact.outputs.name }} + if-no-files-found: error + path: ${{ env.ARTIFACT_LOCAL_LOCATION }} + + additional-artifact: + name: Additional Content Artifact Upload + needs: + - repro + if: inputs.additional-artifact-content-paths != '' + runs-on: ubuntu-latest + environment: ${{ inputs.environment-name }} + outputs: + artifact-name: ${{ steps.artifact.outputs.name }} + artifact-url: ${{ steps.upload.outputs.artifact-url }} + steps: + - name: Setup SSH + id: ssh + uses: access-nri/actions/.github/actions/setup-ssh@main + with: + hosts: | + ${{ secrets.SSH_HOST }} + ${{ secrets.SSH_HOST_DATA }} + private-key: ${{ secrets.SSH_KEY }} + - name: Copy Back Additional Artifact Content - if: inputs.additional-artifact-content-paths != '' # For each of the additional artifact content paths, rsync the data # from the target into our runner for upload as an artifact run: | while IFS= read -r content_path; do if [[ -n "$content_path" ]]; then - rsync --recursive --relative --verbose -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \ - "${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST_DATA }}:${{ env.EXPERIMENT_LOCATION }}/./$content_path" \ - ${{ env.ARTIFACT_LOCAL_LOCATION }} + rsync --recursive --relative --mkpath --verbose -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \ + "${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST_DATA }}:${{ needs.repro.outputs.experiment-location }}/./$content_path" \ + ${{ env.ARTIFACT_LOCAL_LOCATION }}/${{ inputs.additional-artifact-content-paths-root }} fi done <<< "${{ inputs.additional-artifact-content-paths }}" - - name: Generate Test Output Artifact Name + - name: Generate Additional Content Artifact Name id: artifact - run: echo "name=${{ github.event.repository.name }}-${{ inputs.config-ref }}" >> $GITHUB_OUTPUT + run: echo "name=additional-${{ github.event.repository.name }}-${{ inputs.config-ref }}" >> $GITHUB_OUTPUT - - name: Upload Test Output + - name: Additional Content Artifact id: upload uses: actions/upload-artifact@v4 with: