From 969aaddaa1ff0279b25b87359769d2b279aa9c79 Mon Sep 17 00:00:00 2001
From: Iain Russell <Iain.Russell@ecmwf.int>
Date: Wed, 24 Apr 2024 10:47:30 +0100
Subject: [PATCH 1/5] Add to downstream-ci

---
 .github/ci-hpc-config.yml            |   3 +
 .github/workflows/ci.yml             | 205 ++++-----------------------
 .github/workflows/legacy-ci.yml      | 192 +++++++++++++++++++++++++
 tests/downstream-ci-requirements.txt |   1 +
 4 files changed, 224 insertions(+), 177 deletions(-)
 create mode 100644 .github/ci-hpc-config.yml
 create mode 100644 .github/workflows/legacy-ci.yml
 create mode 100644 tests/downstream-ci-requirements.txt

diff --git a/.github/ci-hpc-config.yml b/.github/ci-hpc-config.yml
new file mode 100644
index 0000000..67a79c3
--- /dev/null
+++ b/.github/ci-hpc-config.yml
@@ -0,0 +1,3 @@
+build:
+  python: 3.10
+  parallel: 1
\ No newline at end of file
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4ef239e..3204613 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1,192 +1,43 @@
 name: ci
 
 on:
+  # Trigger the workflow on push to master or develop, except tag creation
   push:
     branches:
-    - main
-    - develop
-    tags:
-    - "*"
-  pull_request:
-    branches:
-    - main
-    - develop
-  pull_request_target:
-    types: [labeled]
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-defaults:
-  run:
-    shell: bash -l {0}
-
-jobs:
-  pre-commit:
-    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        ref: ${{ github.event.pull_request.head.sha || github.ref }}
-    - uses: actions/setup-python@v4
-      with:
-        python-version: 3.x
-    - uses: pre-commit/action@v3.0.0
-
-  unit-tests:
-    name: unit-tests (3.10)
-    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        ref: ${{ github.event.pull_request.head.sha || github.ref }}
-    - name: Install Conda environment with Micromamba
-      uses: mamba-org/provision-with-micromamba@v14
-      with:
-        environment-file: tests/environment-unit-tests.yml
-        environment-name: DEVELOP
-        channels: conda-forge
-        cache-env: true
-        extra-specs: |
-          python=3.10
-    - name: Install package
-      run: |
-        python -m pip install --no-deps -e .
-    - name: Run tests
-      run: |
-        make unit-tests
+      - 'main'
+      - 'develop'
+    tags-ignore:
+      - '**'
 
-  type-check:
-    needs: [unit-tests]
-    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash -l {0}
+  # Trigger the workflow on pull request
+  pull_request: ~
 
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        ref: ${{ github.event.pull_request.head.sha || github.ref }}
-    - name: Install Conda environment with Micromamba
-      uses: mamba-org/provision-with-micromamba@v12
-      with:
-        environment-file: environment.yml
-        environment-name: DEVELOP
-        channels: conda-forge
-        cache-env: true
-        cache-env-key: ubuntu-latest-3.10
-        extra-specs: |
-          python=3.10
-    - name: Install package
-      run: |
-        python -m pip install --no-deps -e .
-    - name: Run code quality checks
-      run: |
-        echo type-check not used
-
-  documentation:
-    needs: [unit-tests]
-    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash -l {0}
+  # Trigger the workflow manually
+  workflow_dispatch: ~
 
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        ref: ${{ github.event.pull_request.head.sha || github.ref }}
-    - name: Install Conda environment with Micromamba
-      uses: mamba-org/provision-with-micromamba@v12
-      with:
-        environment-file: environment.yml
-        environment-name: DEVELOP
-        channels: conda-forge
-        cache-env: true
-        cache-env-key: ubuntu-latest-3.10
-        extra-specs: |
-          python=3.10
-    - name: Install package
-      run: |
-        python -m pip install --no-deps -e .
-    - name: Build documentation
-      run: |
-        make docs-build
+  # Trigger after public PR approved for CI
+  pull_request_target:
+    types: [labeled]
 
-  integration-tests:
-    needs: [unit-tests]
+jobs:
+  # Run CI including downstream packages on self-hosted runners
+  downstream-ci:
+    name: downstream-ci
     if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash -l {0}
-
-    strategy:
-      matrix:
-        include:
-        - python-version: "3.10"
-              # extra: -minver  # This will need to be uncommented and environment-minver.yml updated if we want to publish on conda
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@add-earthkit-meteo
+    with:
+      earthkit-meteo: ecmwf/earthkit-meteo@${{ github.event.pull_request.head.sha || github.sha }}
+      codecov_upload: true
+    secrets: inherit
 
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        ref: ${{ github.event.pull_request.head.sha || github.ref }}
-    - name: Install Conda environment with Micromamba
-      uses: mamba-org/provision-with-micromamba@v12
-      with:
-        environment-file: tests/environment-unit-tests${{ matrix.extra }}.yml
-        environment-name: DEVELOP${{ matrix.extra }}
-        channels: conda-forge
-        cache-env: true
-        cache-env-key: ubuntu-latest-${{ matrix.python-version }}${{ matrix.extra }}.
-        extra-specs: |
-          python=${{matrix.python-version }}
-    - name: Install package
-      run: |
-        python -m pip install --no-deps -e .
-    - name: Run tests
-      run: |
-        make unit-tests
 
-  distribution:
-    needs: [integration-tests, type-check, documentation]
+  # Build downstream packages on HPC
+  downstream-ci-hpc:
+    name: downstream-ci-hpc
     if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-    runs-on: ubuntu-latest
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@add-earthkit-meteo
+    with:
+      earthkit-meteo: ecmwf/earthkit-meteo@${{ github.event.pull_request.head.sha || github.sha }}
+    secrets: inherit
 
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        ref: ${{ github.event.pull_request.head.sha || github.ref }}
-    - name: Build distributions
-      run: |
-        $CONDA/bin/python -m pip install build
-        $CONDA/bin/python -m build
-    - name: Publish a Python distribution to PyPI
-      if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
-      uses: pypa/gh-action-pypi-publish@release/v1
-      with:
-        user: __token__
-        password: ${{ secrets.PYPI_API_TOKEN }}
 
-  notify:
-    if: always() && ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-    needs:
-    - pre-commit
-    - unit-tests
-    - type-check
-    - documentation
-    - integration-tests
-    - distribution
-    runs-on: ubuntu-latest
-    steps:
-    - name: Trigger Teams notification
-      uses: ecmwf-actions/notify-teams@v1
-      with:
-        incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }}
-        needs_context: ${{ toJSON(needs) }}
diff --git a/.github/workflows/legacy-ci.yml b/.github/workflows/legacy-ci.yml
new file mode 100644
index 0000000..4ef239e
--- /dev/null
+++ b/.github/workflows/legacy-ci.yml
@@ -0,0 +1,192 @@
+name: ci
+
+on:
+  push:
+    branches:
+    - main
+    - develop
+    tags:
+    - "*"
+  pull_request:
+    branches:
+    - main
+    - develop
+  pull_request_target:
+    types: [labeled]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash -l {0}
+
+jobs:
+  pre-commit:
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        ref: ${{ github.event.pull_request.head.sha || github.ref }}
+    - uses: actions/setup-python@v4
+      with:
+        python-version: 3.x
+    - uses: pre-commit/action@v3.0.0
+
+  unit-tests:
+    name: unit-tests (3.10)
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        ref: ${{ github.event.pull_request.head.sha || github.ref }}
+    - name: Install Conda environment with Micromamba
+      uses: mamba-org/provision-with-micromamba@v14
+      with:
+        environment-file: tests/environment-unit-tests.yml
+        environment-name: DEVELOP
+        channels: conda-forge
+        cache-env: true
+        extra-specs: |
+          python=3.10
+    - name: Install package
+      run: |
+        python -m pip install --no-deps -e .
+    - name: Run tests
+      run: |
+        make unit-tests
+
+  type-check:
+    needs: [unit-tests]
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        ref: ${{ github.event.pull_request.head.sha || github.ref }}
+    - name: Install Conda environment with Micromamba
+      uses: mamba-org/provision-with-micromamba@v12
+      with:
+        environment-file: environment.yml
+        environment-name: DEVELOP
+        channels: conda-forge
+        cache-env: true
+        cache-env-key: ubuntu-latest-3.10
+        extra-specs: |
+          python=3.10
+    - name: Install package
+      run: |
+        python -m pip install --no-deps -e .
+    - name: Run code quality checks
+      run: |
+        echo type-check not used
+
+  documentation:
+    needs: [unit-tests]
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        ref: ${{ github.event.pull_request.head.sha || github.ref }}
+    - name: Install Conda environment with Micromamba
+      uses: mamba-org/provision-with-micromamba@v12
+      with:
+        environment-file: environment.yml
+        environment-name: DEVELOP
+        channels: conda-forge
+        cache-env: true
+        cache-env-key: ubuntu-latest-3.10
+        extra-specs: |
+          python=3.10
+    - name: Install package
+      run: |
+        python -m pip install --no-deps -e .
+    - name: Build documentation
+      run: |
+        make docs-build
+
+  integration-tests:
+    needs: [unit-tests]
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    strategy:
+      matrix:
+        include:
+        - python-version: "3.10"
+              # extra: -minver  # This will need to be uncommented and environment-minver.yml updated if we want to publish on conda
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        ref: ${{ github.event.pull_request.head.sha || github.ref }}
+    - name: Install Conda environment with Micromamba
+      uses: mamba-org/provision-with-micromamba@v12
+      with:
+        environment-file: tests/environment-unit-tests${{ matrix.extra }}.yml
+        environment-name: DEVELOP${{ matrix.extra }}
+        channels: conda-forge
+        cache-env: true
+        cache-env-key: ubuntu-latest-${{ matrix.python-version }}${{ matrix.extra }}.
+        extra-specs: |
+          python=${{matrix.python-version }}
+    - name: Install package
+      run: |
+        python -m pip install --no-deps -e .
+    - name: Run tests
+      run: |
+        make unit-tests
+
+  distribution:
+    needs: [integration-tests, type-check, documentation]
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        ref: ${{ github.event.pull_request.head.sha || github.ref }}
+    - name: Build distributions
+      run: |
+        $CONDA/bin/python -m pip install build
+        $CONDA/bin/python -m build
+    - name: Publish a Python distribution to PyPI
+      if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        user: __token__
+        password: ${{ secrets.PYPI_API_TOKEN }}
+
+  notify:
+    if: always() && ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    needs:
+    - pre-commit
+    - unit-tests
+    - type-check
+    - documentation
+    - integration-tests
+    - distribution
+    runs-on: ubuntu-latest
+    steps:
+    - name: Trigger Teams notification
+      uses: ecmwf-actions/notify-teams@v1
+      with:
+        incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }}
+        needs_context: ${{ toJSON(needs) }}
diff --git a/tests/downstream-ci-requirements.txt b/tests/downstream-ci-requirements.txt
new file mode 100644
index 0000000..24ce15a
--- /dev/null
+++ b/tests/downstream-ci-requirements.txt
@@ -0,0 +1 @@
+numpy

From c7f3e0ea33620d8a88a04f2135ea5007bac812be Mon Sep 17 00:00:00 2001
From: Iain Russell <Iain.Russell@ecmwf.int>
Date: Wed, 24 Apr 2024 10:56:23 +0100
Subject: [PATCH 2/5] Add to downstream-ci - add qa

---
 .github/workflows/ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3204613..c62884b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -28,6 +28,7 @@ jobs:
     with:
       earthkit-meteo: ecmwf/earthkit-meteo@${{ github.event.pull_request.head.sha || github.sha }}
       codecov_upload: true
+      python_qa: true
     secrets: inherit
 
 

From 7e3b674e1eb61d166f0506d76c2509436c27b103 Mon Sep 17 00:00:00 2001
From: Iain Russell <Iain.Russell@ecmwf.int>
Date: Wed, 24 Apr 2024 11:14:46 +0100
Subject: [PATCH 3/5] Add to downstream-ci - finish

---
 .github/workflows/ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c62884b..4804a2d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -24,7 +24,7 @@ jobs:
   downstream-ci:
     name: downstream-ci
     if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@add-earthkit-meteo
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
     with:
       earthkit-meteo: ecmwf/earthkit-meteo@${{ github.event.pull_request.head.sha || github.sha }}
       codecov_upload: true
@@ -36,7 +36,7 @@ jobs:
   downstream-ci-hpc:
     name: downstream-ci-hpc
     if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@add-earthkit-meteo
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main
     with:
       earthkit-meteo: ecmwf/earthkit-meteo@${{ github.event.pull_request.head.sha || github.sha }}
     secrets: inherit

From 2052911c55c62ae163c438773cf8db6fb714e4dc Mon Sep 17 00:00:00 2001
From: Iain Russell <Iain.Russell@ecmwf.int>
Date: Wed, 24 Apr 2024 11:16:13 +0100
Subject: [PATCH 4/5] Add to downstream-ci - rename legacy-ci

---
 .github/workflows/legacy-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/legacy-ci.yml b/.github/workflows/legacy-ci.yml
index 4ef239e..921fb15 100644
--- a/.github/workflows/legacy-ci.yml
+++ b/.github/workflows/legacy-ci.yml
@@ -1,4 +1,4 @@
-name: ci
+name: legacy-ci
 
 on:
   push:

From 9ee97039246680d733b5439a0e41a1cb4b938c06 Mon Sep 17 00:00:00 2001
From: Iain Russell <Iain.Russell@ecmwf.int>
Date: Wed, 24 Apr 2024 13:30:44 +0100
Subject: [PATCH 5/5] Add to downstream-ci - format yaml

---
 .github/ci-hpc-config.yml |  2 +-
 .github/workflows/ci.yml  | 12 +++++-------
 2 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/.github/ci-hpc-config.yml b/.github/ci-hpc-config.yml
index 67a79c3..2e20c20 100644
--- a/.github/ci-hpc-config.yml
+++ b/.github/ci-hpc-config.yml
@@ -1,3 +1,3 @@
 build:
   python: 3.10
-  parallel: 1
\ No newline at end of file
+  parallel: 1
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4804a2d..7783738 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -4,16 +4,16 @@ on:
   # Trigger the workflow on push to master or develop, except tag creation
   push:
     branches:
-      - 'main'
-      - 'develop'
+    - 'main'
+    - 'develop'
     tags-ignore:
-      - '**'
+    - '**'
 
   # Trigger the workflow on pull request
-  pull_request: ~
+  pull_request:
 
   # Trigger the workflow manually
-  workflow_dispatch: ~
+  workflow_dispatch:
 
   # Trigger after public PR approved for CI
   pull_request_target:
@@ -40,5 +40,3 @@ jobs:
     with:
       earthkit-meteo: ecmwf/earthkit-meteo@${{ github.event.pull_request.head.sha || github.sha }}
     secrets: inherit
-
-