Merge branch 'main' into canada_trends

ESMValGroup · Oct 16, 2024 · 6914561 · 6914561
2 parents de335cf + 1de5bf6
commit 6914561
Show file tree

Hide file tree

Showing 444 changed files with 16,817 additions and 5,712 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -27,6 +27,7 @@ commands:
           name: Log versions
           command: |
             . /opt/conda/etc/profile.d/conda.sh
+            conda env export --name base > /logs/base_environment.yml
             conda activate esmvaltool
             esmvaltool version
             dpkg -l > /logs/versions.txt
@@ -40,6 +41,7 @@ commands:
           command: |
             . /opt/conda/etc/profile.d/conda.sh
             conda activate esmvaltool
+            mamba --version
             pytest -n 4 --junitxml=test-reports/report.xml
             esmvaltool version
             esmvaltool -- --help
@@ -127,6 +129,7 @@ jobs:
       - run:
           name: Install dependencies
           command: |
+            set -x
             . /opt/conda/etc/profile.d/conda.sh
             conda activate esmvaltool
             mkdir /logs
@@ -213,9 +216,8 @@ jobs:
             conda activate esmvaltool
             mkdir -p ~/climate_data
             esmvaltool config get_config_user
-            echo "offline: false" >> ~/.esmvaltool/config-user.yml
+            echo "search_esgf: when_missing" >> ~/.esmvaltool/config-user.yml
             cat ~/.esmvaltool/config-user.yml
-            esmvaltool run examples/recipe_python.yml
             for recipe in esmvaltool/recipes/testing/recipe_*.yml; do
               esmvaltool run "$recipe"
             done
@@ -232,7 +234,7 @@ jobs:
     # Test building documentation
     docker:
       - image: condaforge/mambaforge:latest
-    resource_class: small
+    resource_class: medium
     steps:
       - checkout
       - run:
@@ -256,7 +258,7 @@ jobs:
     # Test conda package installation
     docker:
       - image: condaforge/mambaforge:latest
-    resource_class: medium
+    resource_class: large
     steps:
       - run:
           command: |
@@ -265,7 +267,7 @@ jobs:
             # Install prerequisites
             mkdir /logs
             # Install ESMValTool in a new conda environment
-            mamba create -y --name esmvaltool -c conda-forge esmvaltool julia 'python=3.10' >> /logs/conda.txt 2>&1
+            mamba create -y --name esmvaltool -c conda-forge esmvaltool julia 'python=3.11' >> /logs/conda.txt 2>&1
             # Activate the environment
             set +x; conda activate esmvaltool; set -x
             # install the Julia dependencies

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1,2 +1,2 @@
-esmvaltool/cmorizers @remi-kazeroni
+esmvaltool/cmorizers @ESMValGroup/obs-maintainers
 .github/workflows @valeriupredoi
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -62,6 +62,7 @@ It is the responsibility of the author to make sure the pull request is ready to
 
 - [ ] [🛠][1] [Documentation](https://docs.esmvaltool.org/en/latest/community/dataset.html#dataset-documentation) is available
 - [ ] [🛠][1] The dataset has been [added to the CMOR check recipe](https://docs.esmvaltool.org/en/latest/community/dataset.html#testing)
+- [ ] [🛠][1] The dataset has been added to the shared [data pools](https://docs.esmvaltool.org/en/latest/community/dataset.html#cmorized-data) of DKRZ and Jasmin by the @ESMValGroup/OBS-maintainers team
 - [ ] [🧪][2] Numbers and units of the data look [physically meaningful](https://docs.esmvaltool.org/en/latest/community/dataset.html#scientific-sanity-check)
 
 ***

diff --git a/.github/workflows/citation_file_validator.yml b/.github/workflows/citation_file_validator.yml
@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out a copy of the repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Check whether the citation metadata from CITATION.cff is valid
         uses: citation-file-format/[email protected]

diff --git a/.github/workflows/create-condalock-file.yml b/.github/workflows/create-condalock-file.yml
@@ -6,66 +6,75 @@ on:
   # and hence lots of automated PRs
   # push:
   #   branches:
-  #     - condalock-update
+  #     - main
   schedule:
     - cron: '0 4 */10 * *'
 
+# Required shell entrypoint to have properly configured bash shell
+defaults:
+  run:
+    shell: bash -l {0}
+
 jobs:
   create-lock-file:
     name: Create conda lock file for latest Python
     runs-on: 'ubuntu-latest'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: conda-incubator/setup-miniconda@v3
         with:
           auto-update-conda: true
           activate-environment: esmvaltool-fromlock
-          python-version: "3.10"
-          miniconda-version: "latest"
-          channels: conda-forge
+          python-version: "3.11"
+          miniforge-version: "latest"
+          miniforge-variant: Mambaforge
+          use-mamba: true
       - name: Show conda config
-        shell: bash -l {0}
         run: |
-          conda info
-          conda list
+          conda update -n base -c conda-forge conda
+          conda --version
+          # setup-miniconda@v3 installs an old conda and mamba
+          # forcing a modern mamba updates both mamba and conda
+          conda install -c conda-forge "mamba>=1.4.8"
           conda config --show-sources
           conda config --show
+          conda --version
+          mamba --version
       - name: Python info
-        shell: bash -l {0}
         run: |
           which python
           python --version
       - name: Install conda-lock
-        shell: bash -l {0}
-        run: conda install -y conda-lock
+        run: mamba install -y -c conda-forge conda-lock
+      - name: Check version of conda-lock
+        run: conda-lock --version
       - name: Create conda lock file for linux-64
-        shell: bash -l {0}
         run: conda-lock lock --platform linux-64 -f environment.yml --mamba --kind explicit
-      - shell: bash -l {0}
+      - name: Show conda version again
         run: conda --version
-      - shell: bash -l {0}
-        run: which python
-      - shell: bash -l {0}
-        run: python -V
-      - shell: bash -l {0}
+      - name: Show Python exec and version again
+        run: |
+          which python
+          python -V
+      - name: Create conda-lock environment
         run: conda create --name esmvaltool-fromlock --file conda-linux-64.lock
-      - shell: bash -l {0}
-        run: conda install -y -c conda-forge pip
-      - shell: bash -l {0}
-        run: which python
-      - shell: bash -l {0}
-        run: pip --version
-      - shell: bash -l {0}
+      - name: Install pip
+        run: mamba install -y pip
+      - name: Check Python and pip versions post pip-install
+        run: |
+          which python
+          pip --version
+      - name: Install ESMValTool
         run: pip install -e .[develop]
-      - shell: bash -l {0}
+      - name: Check ESMValTool version
         run: esmvaltool --help
-      - shell: bash -l {0}
+      - name: Check ESMValTool help
         run: esmvaltool version
-      - shell: bash -l {0}
+      - name: Run flake8
         run: flake8
-      - shell: bash -l {0}
+      - name: Run tests
         run: pytest -n 2 -m "not installation"
       # Automated PR
       # see https://github.com/marketplace/actions/create-pull-request
@@ -88,7 +97,4 @@ jobs:
             automatedPR
           assignees: valeriupredoi
           reviewers: valeriupredoi
-          team-reviewers: |
-            owners
-            maintainers
           draft: false
diff --git a/.github/workflows/install-from-conda.yml b/.github/workflows/install-from-conda.yml
@@ -9,41 +9,42 @@ on:
   schedule:
     - cron: '0 4 * * *'
 
+# Required shell entrypoint to have properly configured bash shell
+defaults:
+  run:
+    shell: bash -l {0}
+
 jobs:
   linux:
     runs-on: "ubuntu-latest"
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.10", "3.11"]
     name: Linux Python ${{ matrix.python-version }}
     steps:
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: conda-incubator/setup-miniconda@v3
         with:
           python-version: ${{ matrix.python-version }}
           miniforge-version: "latest"
           miniforge-variant: Mambaforge
           use-mamba: true
-      - shell: bash -l {0}
-        run: mkdir -p conda_install_linux_artifacts_python_${{ matrix.python-version }}
+      - run: mkdir -p conda_install_linux_artifacts_python_${{ matrix.python-version }}
       - name: Record versions
-        shell: bash -l {0}
         run: |
           mamba --version 2>&1 | tee conda_install_linux_artifacts_python_${{ matrix.python-version }}/conda_version.txt
           which conda 2>&1 | tee conda_install_linux_artifacts_python_${{ matrix.python-version }}/conda_path.txt
           which mamba 2>&1 | tee -a conda_install_linux_artifacts_python_${{ matrix.python-version }}/conda_path.txt
           python -V 2>&1 | tee conda_install_linux_artifacts_python_${{ matrix.python-version }}/python_version.txt
       - name: Install ESMValTool
-        shell: bash -l {0}
         run: mamba install esmvaltool 2>&1 | tee conda_install_linux_artifacts_python_${{ matrix.python-version }}/install.txt
       - name: Verify installation
-        shell: bash -l {0}
         run: |
           esmvaltool --help
           esmvaltool version 2>&1 | tee conda_install_linux_artifacts_python_${{ matrix.python-version }}/version.txt
       - name: Upload artifacts
         if: ${{ always() }}  # upload artifacts even if fail
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: Conda_Install_Linux_python_${{ matrix.python-version }}
           path: conda_install_linux_artifacts_python_${{ matrix.python-version }}
@@ -55,44 +56,33 @@ jobs:
 #    runs-on: "macos-latest"
 #    strategy:
 #      matrix:
-#        python-version: ["3.8", "3.9", "3.10"]
+#        python-version: ["3.10", "3.11"]
 #      fail-fast: false
 #    name: OSX Python ${{ matrix.python-version }}
 #    steps:
 #      - uses: actions/checkout@v2
-#      - uses: conda-incubator/setup-miniconda@v2
+#      - uses: conda-incubator/setup-miniconda@v3
 #        with:
 #          python-version: ${{ matrix.python-version }}
 #          miniconda-version: "latest"
 #          channels: conda-forge
-#      - shell: bash -l {0}
-#        run: mkdir -p conda_install_osx_artifacts_python_${{ matrix.python-version }}
-#      - shell: bash -l {0}
-#        run: conda --version 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/conda_version.txt
-#      - shell: bash -l {0}
-#        run: which conda 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/conda_path.txt
-#      - shell: bash -l {0}
-#        run: python -V 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/python_version.txt
+#      - run: mkdir -p conda_install_osx_artifacts_python_${{ matrix.python-version }}
+#      - run: conda --version 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/conda_version.txt
+#      - run: which conda 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/conda_path.txt
+#      - run: python -V 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/python_version.txt
 #      # ncurses needs to be from conda-forge and not main channel
 #      # for now it's turned off since we're not testing R/Julia installs
-#      # - shell: bash -l {0}
-#      #   run: conda uninstall -y ncurses
-#      # - shell: bash -l {0}
-#      #   run: conda list ncurses
-#      # - shell: bash -l {0}
-#      #   run: conda install -y conda-forge::ncurses
-#      # - shell: bash -l {0}
-#      #   run: conda list ncurses
-#      - shell: bash -l {0}
-#        #run: conda install esmvaltool --no-update-deps 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/install.txt
-#        run: conda install esmvaltool-python esmvaltool-ncl 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/install.txt
-#      - shell: bash -l {0}
-#        run: esmvaltool --help
-#      - shell: bash -l {0}
-#        run: esmvaltool version 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/version.txt
+#      # - run: conda uninstall -y ncurses
+#      # - run: conda list ncurses
+#      # - run: conda install -y conda-forge::ncurses
+#      # - run: conda list ncurses
+#      - run: conda install esmvaltool --no-update-deps 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/install.txt
+#      - run: conda install esmvaltool-python esmvaltool-ncl 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/install.txt
+#      - run: esmvaltool --help
+#      - run: esmvaltool version 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/version.txt
 #      - name: Upload artifacts
 #        if: ${{ always() }}  # upload artifacts even if fail
-#        uses: actions/upload-artifact@v2
+#        uses: actions/upload-artifact@v4
 #        with:
 #          name: Conda_Install_OSX_python_${{ matrix.python-version }}
 #          path: conda_install_osx_artifacts_python_${{ matrix.python-version }}