From 0c70c150d8dc08d625907659f73ee7260d2a2d6b Mon Sep 17 00:00:00 2001 From: rcrutherford Date: Tue, 10 Oct 2023 13:15:41 -0400 Subject: [PATCH 01/14] Add Crusher push mirror. --- .github/workflows/ornl_crusher_mirror.yaml | 37 ++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/ornl_crusher_mirror.yaml diff --git a/.github/workflows/ornl_crusher_mirror.yaml b/.github/workflows/ornl_crusher_mirror.yaml new file mode 100644 index 00000000..66ca1fa0 --- /dev/null +++ b/.github/workflows/ornl_crusher_mirror.yaml @@ -0,0 +1,37 @@ +name: ORNL Crusher Mirror + +# triggers a github action everytime there is a push or mr +on: + pull_request: + push: + branches: + - develop + - main + +jobs: + # To test on HPC resources we must first mirror the repo and then trigger a pipeline + push: + # Latest Ubuntu as of Feb 2023 is 20.04 + runs-on: ubuntu-22.04 + steps: + # Action checks-out your repository under $GITHUB_WORKSPACE, so your workflow can access it. + - uses: actions/checkout@v1 + + # Necessary to get branch name independent of PR/push + - name: Extract branch name + shell: bash + run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT + id: extract_branch + + # Action for mirroring your commits to a different remote repository + - uses: yesolutions/mirror-action@master + # Give actions access to some secrets + with: + GIT_PUSH_ARGS: '--tags --force --push-option=ci.skip' + REMOTE: 'https://code.olcf.ornl.gov/ci/csc359/dev/exago' + GIT_USERNAME: ${{ secrets.GIT_USER }} + GIT_PASSWORD: ${{ secrets.ORNL_CRUSHER_GIT_PASSWORD }} + + # Trigger CI pipeline since it was skipped in the above push + - name: Trigger Pipeline + run: curl -X POST -F token=${{ secrets.ORNL_ASCENT_PIPELINE_TRIGGER }} -F ref=${{ steps.extract_branch.outputs.branch }} https://code.olcf.ornl.gov/api/v4/projects/48/trigger/pipeline From 1ea6356b2864730aeaa036e3e060841cd9fd38b2 Mon Sep 17 00:00:00 2001 From: rcrutherford Date: Tue, 10 Oct 2023 13:21:08 -0400 Subject: [PATCH 02/14] Add reporting from Crusher pipeline. --- .gitlab/ornl/crusher.gitlab-ci.yml | 143 ++++++++++++++++++++++++----- 1 file changed, 121 insertions(+), 22 deletions(-) diff --git a/.gitlab/ornl/crusher.gitlab-ci.yml b/.gitlab/ornl/crusher.gitlab-ci.yml index 65b18570..506562a3 100644 --- a/.gitlab/ornl/crusher.gitlab-ci.yml +++ b/.gitlab/ornl/crusher.gitlab-ci.yml @@ -1,9 +1,62 @@ -# Crusher Variables -.crusher_rules: +# https://stackoverflow.com/questions/67009007 +# We have to use YAML anchors for rules here +# Rules are also evaluated sequentially, so you need to be careful about +# the order in which these rules are included in child jobs - list "never" rules before other rules +.rules-template: rules: - - if: $CI_PIPELINE_SOURCE == "web" - - if: $CI_PINELINE_SOURCE == "schedule" - - when: never + # -- + # Rule to fall back on... + - &default + when: always + # -- + # Only run when the commit **DOES NOT** contains "[newell-rebuild]" + - &rule_no_newell_rebuild + if: '$CI_COMMIT_TITLE =~ /\[newell-rebuild\]/' + when: never + # Only run when the commit **DOES NOT** contains "[deception-rebuild]" + - &rule_no_deception_rebuild + if: '$CI_COMMIT_TITLE =~ /\[deception-rebuild\]/' + when: never + # Only run when the commit **DOES NOT** contains "[incline-rebuild]" + - &rule_no_incline_rebuild + if: '$CI_COMMIT_TITLE =~ /\[incline-rebuild\]/' + when: never + # Only run when the commit **DOES NOT** contains "[ascent-rebuild]" + - &rule_no_ascent_rebuild + if: '$CI_COMMIT_TITLE =~ /\[ascent-rebuild\]/' + when: never + # Only run when the commit **DOES NOT** contains "[newell-test]" + - &rule_no_newell_test + if: '$CI_COMMIT_TITLE =~ /\[newell-test\]/' + when: never + # Only run when the commit **DOES** contains "[newell-test]" + - &rule_yes_newell_test + if: '$CI_COMMIT_TITLE =~ /\[newell-test\]/' + when: always + # Only run when the commit **DOES NOT** contains "[deception-test]" + - &rule_no_deception_test + if: '$CI_COMMIT_TITLE =~ /\[deception-test\]/' + when: never + # Only run when the commit **DOES** contains "[deception-test]" + - &rule_yes_deception_test + if: '$CI_COMMIT_TITLE =~ /\[deception-test\]/' + when: always + # Only run when the commit **DOES NOT** contains "[incline-test]" + - &rule_no_incline_test + if: '$CI_COMMIT_TITLE =~ /\[incline-test\]/' + when: never + # Only run when the commit **DOES** contains "[incline-test]" + - &rule_yes_incline_test + if: '$CI_COMMIT_TITLE =~ /\[incline-test\]/' + when: always + # Only run when the commit **DOES NOT** contains "[ascent-test]" + - &rule_no_ascent_test + if: '$CI_COMMIT_TITLE =~ /\[ascent-test\]/' + when: never + # Only run when the commit **DOES** contains "[ascent-test]" + - &rule_yes_ascent_test + if: '$CI_COMMIT_TITLE =~ /\[ascent-test\]/' + when: always .crusher_variables: # Only for slurm tagged jobs... @@ -11,6 +64,18 @@ SCHEDULER_PARAMETERS: "-N 1 -A CSC359 --time=60" WORKDIR: /gpfs/alpine/csc359/proj-shared/ci/${CI_PIPELINE_ID} +.crusher_rules: + rules: + - *rule_no_deception_test + - *rule_no_deception_rebuild + - *rule_no_newell_test + - *rule_no_newell_rebuild + - *rule_no_incline_test + - *rule_no_incline_rebuild + - *rule_no_ascent_test + - *rule_no_ascent_rebuild + - *default + # Crusher Jobs Crusher Build: stage: build @@ -38,7 +103,7 @@ Crusher Test: script: - cd "$WORKDIR" - export srcdir=$WORKDIR builddir=$WORKDIR/build installdir=$WORKDIR/install - - export CTESTARGS="--output-on-failure -E Python" + - export CTESTARGS="--output-on-failure" - MY_CLUSTER=crusher ./buildsystem/build.sh --test-only --job=clang-hip - res=$? - exit $res @@ -49,22 +114,56 @@ Crusher Test: - .crusher_rules - .crusher_variables -Crusher Python Test: - stage: test - dependencies: - - "Crusher Build" +# -- Reporting Ascent Status to PNNL GitHub +.report-job: + when: always variables: - # Don't clone for test jobs + MY_CLUSTER: Crusher GIT_STRATEGY: none - allow_failure: true - tags: [crusher, slurm] + STATUS_PROJECT: pnnl/ExaGO + STATUS_NAME: amd64/clang@14.0/rocm@5.2.0/MI100@gfx908 + tags: + - nobatch + extends: .ascent_environment_template + environment: + name: reporting-gitlab + resource_group: status + retry: 1 + +.report-status: + extends: .report-job script: - - cd "$WORKDIR" - - export srcdir=$WORKDIR builddir=$WORKDIR/build installdir=$WORKDIR/install - - export CTESTARGS="--output-on-failure -R Python" - - MY_CLUSTER=crusher ./buildsystem/build.sh --test-only --job=clang-hip - - res=$? - - exit $res - extends: - - .crusher_rules - - .crusher_variables + - | + curl -L \ + -X POST \ + -H @${GITHUB_CURL_HEADER}\ + https://api.github.com/repos/${STATUS_PROJECT}/statuses/${CI_COMMIT_SHA} \ + -d "{\"state\":\"${CI_JOB_NAME}\",\"target_url\":\"${CI_PIPELINE_URL}\",\"description\":\"${STATUS_NAME}\",\"context\":\"${MY_CLUSTER}\"}" + environment: + name: reporting-gitlab + +.report-pending: + extends: .report-job + script: + - | + set -x + curl -L \ + -X POST \ + -H @${GITHUB_CURL_HEADER}\ + https://api.github.com/repos/${STATUS_PROJECT}/statuses/${CI_COMMIT_SHA} \ + -d "{\"state\":\"${CI_JOB_NAME}\",\"target_url\":\"${CI_PIPELINE_URL}\",\"context\":\"${MY_CLUSTER}\"}" + +pending: + extends: .report-pending + stage: .pre + +success: + extends: .report-status + stage: .post + +failure: + stage: .post + extends: .report-status + rules: + - when: on_failure +# --- From 84f2a88c25b7a9b977655ad6e5b95ed471790421 Mon Sep 17 00:00:00 2001 From: rcrutherford Date: Tue, 10 Oct 2023 13:27:22 -0400 Subject: [PATCH 03/14] Fix pipeline trigger token for Crusher. --- .github/workflows/ornl_crusher_mirror.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ornl_crusher_mirror.yaml b/.github/workflows/ornl_crusher_mirror.yaml index 66ca1fa0..728fead8 100644 --- a/.github/workflows/ornl_crusher_mirror.yaml +++ b/.github/workflows/ornl_crusher_mirror.yaml @@ -34,4 +34,4 @@ jobs: # Trigger CI pipeline since it was skipped in the above push - name: Trigger Pipeline - run: curl -X POST -F token=${{ secrets.ORNL_ASCENT_PIPELINE_TRIGGER }} -F ref=${{ steps.extract_branch.outputs.branch }} https://code.olcf.ornl.gov/api/v4/projects/48/trigger/pipeline + run: curl -X POST -F token=${{ secrets.ORNL_CRUSHER_PIPELINE_TRIGGER }} -F ref=${{ steps.extract_branch.outputs.branch }} https://code.olcf.ornl.gov/api/v4/projects/48/trigger/pipeline From 74db5607cf29283d9a530af19cd1b363d627799b Mon Sep 17 00:00:00 2001 From: rcrutherford Date: Tue, 10 Oct 2023 13:30:30 -0400 Subject: [PATCH 04/14] Fix last ascent reference in Crusher pipelines... --- .gitlab/ornl/crusher.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/ornl/crusher.gitlab-ci.yml b/.gitlab/ornl/crusher.gitlab-ci.yml index 506562a3..b0987081 100644 --- a/.gitlab/ornl/crusher.gitlab-ci.yml +++ b/.gitlab/ornl/crusher.gitlab-ci.yml @@ -124,7 +124,7 @@ Crusher Test: STATUS_NAME: amd64/clang@14.0/rocm@5.2.0/MI100@gfx908 tags: - nobatch - extends: .ascent_environment_template + extends: .crusher_variables environment: name: reporting-gitlab resource_group: status From 9460affb5318ab937326817eabad2a494935aafd Mon Sep 17 00:00:00 2001 From: rcrutherford Date: Tue, 10 Oct 2023 13:42:51 -0400 Subject: [PATCH 05/14] Use correct tag for crusher. --- .gitlab/ornl/crusher.gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab/ornl/crusher.gitlab-ci.yml b/.gitlab/ornl/crusher.gitlab-ci.yml index b0987081..2ed37908 100644 --- a/.gitlab/ornl/crusher.gitlab-ci.yml +++ b/.gitlab/ornl/crusher.gitlab-ci.yml @@ -122,8 +122,7 @@ Crusher Test: GIT_STRATEGY: none STATUS_PROJECT: pnnl/ExaGO STATUS_NAME: amd64/clang@14.0/rocm@5.2.0/MI100@gfx908 - tags: - - nobatch + tags: [crusher, shell] extends: .crusher_variables environment: name: reporting-gitlab From b8a407985744c877806fa51a860b8f21e41cdd1a Mon Sep 17 00:00:00 2001 From: rcrutherford Date: Tue, 10 Oct 2023 14:02:38 -0400 Subject: [PATCH 06/14] Add rule for default workflow to Crusher CI. --- .gitlab/ornl/crusher.gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab/ornl/crusher.gitlab-ci.yml b/.gitlab/ornl/crusher.gitlab-ci.yml index 2ed37908..fd9c5e5d 100644 --- a/.gitlab/ornl/crusher.gitlab-ci.yml +++ b/.gitlab/ornl/crusher.gitlab-ci.yml @@ -1,3 +1,7 @@ +workflow: + rules: + - when: always + # https://stackoverflow.com/questions/67009007 # We have to use YAML anchors for rules here # Rules are also evaluated sequentially, so you need to be careful about From 56c35402594809555240ec45f21061fd7a6cb7fc Mon Sep 17 00:00:00 2001 From: rcrutherford Date: Tue, 10 Oct 2023 14:12:05 -0400 Subject: [PATCH 07/14] Allow pipelines to run on trigger. --- .gitlab/ornl/crusher.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/ornl/crusher.gitlab-ci.yml b/.gitlab/ornl/crusher.gitlab-ci.yml index fd9c5e5d..83fcb6e1 100644 --- a/.gitlab/ornl/crusher.gitlab-ci.yml +++ b/.gitlab/ornl/crusher.gitlab-ci.yml @@ -1,6 +1,6 @@ workflow: rules: - - when: always + - if: '$CI_PIPELINE_SOURCE == "trigger"' # https://stackoverflow.com/questions/67009007 # We have to use YAML anchors for rules here From 6b1feded0025b2324b09a1d91f73ce5dfd9cf9ce Mon Sep 17 00:00:00 2001 From: rcrutherford Date: Tue, 10 Oct 2023 18:17:50 -0400 Subject: [PATCH 08/14] Run on web or schedule instead of trigger. --- .gitlab/ornl/crusher.gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab/ornl/crusher.gitlab-ci.yml b/.gitlab/ornl/crusher.gitlab-ci.yml index 83fcb6e1..c93cacf3 100644 --- a/.gitlab/ornl/crusher.gitlab-ci.yml +++ b/.gitlab/ornl/crusher.gitlab-ci.yml @@ -1,6 +1,7 @@ workflow: rules: - - if: '$CI_PIPELINE_SOURCE == "trigger"' + - if: $CI_PIPELINE_SOURCE == "web" + - if: $CI_PINELINE_SOURCE == "schedule" # https://stackoverflow.com/questions/67009007 # We have to use YAML anchors for rules here From e0745db32fa402a43fba0c4e3d7e55a6b31c1766 Mon Sep 17 00:00:00 2001 From: rcrutherford Date: Tue, 10 Oct 2023 18:19:20 -0400 Subject: [PATCH 09/14] Remove crusher pipeline trigger. --- .github/workflows/ornl_crusher_mirror.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ornl_crusher_mirror.yaml b/.github/workflows/ornl_crusher_mirror.yaml index 728fead8..c9da4804 100644 --- a/.github/workflows/ornl_crusher_mirror.yaml +++ b/.github/workflows/ornl_crusher_mirror.yaml @@ -32,6 +32,8 @@ jobs: GIT_USERNAME: ${{ secrets.GIT_USER }} GIT_PASSWORD: ${{ secrets.ORNL_CRUSHER_GIT_PASSWORD }} - # Trigger CI pipeline since it was skipped in the above push - - name: Trigger Pipeline - run: curl -X POST -F token=${{ secrets.ORNL_CRUSHER_PIPELINE_TRIGGER }} -F ref=${{ steps.extract_branch.outputs.branch }} https://code.olcf.ornl.gov/api/v4/projects/48/trigger/pipeline + # Can only run on Crusher with web or schedule pipeline, so no way to trigger + # # Trigger CI pipeline since it was skipped in the above push + # - name: Trigger Pipeline + # run: curl -X POST -F token=${{ secrets.ORNL_CRUSHER_PIPELINE_TRIGGER }} -F ref=${{ steps.extract_branch.outputs.branch }} https://code.olcf.ornl.gov/api/v4/projects/48/trigger/pipeline + From 36c7d84bb2891ede6bf6c182557c01d4d43b9eb5 Mon Sep 17 00:00:00 2001 From: rcrutherford Date: Tue, 10 Oct 2023 19:31:42 -0400 Subject: [PATCH 10/14] Use orion instead of alpine for job on crusher. --- .gitlab/ornl/crusher.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/ornl/crusher.gitlab-ci.yml b/.gitlab/ornl/crusher.gitlab-ci.yml index c93cacf3..03e4fc82 100644 --- a/.gitlab/ornl/crusher.gitlab-ci.yml +++ b/.gitlab/ornl/crusher.gitlab-ci.yml @@ -67,7 +67,7 @@ workflow: # Only for slurm tagged jobs... variables: SCHEDULER_PARAMETERS: "-N 1 -A CSC359 --time=60" - WORKDIR: /gpfs/alpine/csc359/proj-shared/ci/${CI_PIPELINE_ID} + WORKDIR: /lustre/orion/csc359/proj-shared/ci/${CI_PIPELINE_ID} .crusher_rules: rules: From 4a41dc6e31acf5bba793d61083161b4a883ba405 Mon Sep 17 00:00:00 2001 From: rcrutherford Date: Tue, 10 Oct 2023 19:33:01 -0400 Subject: [PATCH 11/14] Fix end of file. --- .github/workflows/ornl_crusher_mirror.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ornl_crusher_mirror.yaml b/.github/workflows/ornl_crusher_mirror.yaml index c9da4804..cf9ec4e7 100644 --- a/.github/workflows/ornl_crusher_mirror.yaml +++ b/.github/workflows/ornl_crusher_mirror.yaml @@ -36,4 +36,3 @@ jobs: # # Trigger CI pipeline since it was skipped in the above push # - name: Trigger Pipeline # run: curl -X POST -F token=${{ secrets.ORNL_CRUSHER_PIPELINE_TRIGGER }} -F ref=${{ steps.extract_branch.outputs.branch }} https://code.olcf.ornl.gov/api/v4/projects/48/trigger/pipeline - From 2fbcc9937bdb110b982e0c01238eec832c0faced Mon Sep 17 00:00:00 2001 From: rcrutherford Date: Wed, 11 Oct 2023 11:36:03 -0400 Subject: [PATCH 12/14] Fix crusher partition. --- .gitlab/ornl/crusher.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/ornl/crusher.gitlab-ci.yml b/.gitlab/ornl/crusher.gitlab-ci.yml index 03e4fc82..fcd64823 100644 --- a/.gitlab/ornl/crusher.gitlab-ci.yml +++ b/.gitlab/ornl/crusher.gitlab-ci.yml @@ -66,7 +66,7 @@ workflow: .crusher_variables: # Only for slurm tagged jobs... variables: - SCHEDULER_PARAMETERS: "-N 1 -A CSC359 --time=60" + SCHEDULER_PARAMETERS: "-N 1 -A CSC359_crusher --time=60" WORKDIR: /lustre/orion/csc359/proj-shared/ci/${CI_PIPELINE_ID} .crusher_rules: From 0039914993256945961be3bd426e669463f62029 Mon Sep 17 00:00:00 2001 From: rcrutherford Date: Wed, 11 Oct 2023 11:54:49 -0400 Subject: [PATCH 13/14] Start build without pending and only run success on success? --- .gitlab/ornl/crusher.gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab/ornl/crusher.gitlab-ci.yml b/.gitlab/ornl/crusher.gitlab-ci.yml index fcd64823..20801367 100644 --- a/.gitlab/ornl/crusher.gitlab-ci.yml +++ b/.gitlab/ornl/crusher.gitlab-ci.yml @@ -83,6 +83,7 @@ workflow: # Crusher Jobs Crusher Build: + needs: [] stage: build tags: [crusher, shell] script: @@ -162,6 +163,7 @@ pending: stage: .pre success: + needs: ['Crusher Test'] extends: .report-status stage: .post From 4224870365e77ecdff27be34725792ca3575cb8c Mon Sep 17 00:00:00 2001 From: rcrutherford Date: Wed, 11 Oct 2023 12:06:34 -0400 Subject: [PATCH 14/14] Fix Ascent needs relationship. --- .gitlab/ornl/ascent.gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/ornl/ascent.gitlab-ci.yml b/.gitlab/ornl/ascent.gitlab-ci.yml index 86a99952..ee5c8e05 100644 --- a/.gitlab/ornl/ascent.gitlab-ci.yml +++ b/.gitlab/ornl/ascent.gitlab-ci.yml @@ -214,6 +214,7 @@ pending: stage: .pre success: + needs: ['Ascent Test'] extends: .report-status stage: .post