Add support for Python=3.12 (#2228)

Co-authored-by: Bouwe Andela <[email protected]>

.github/workflows/build-and-deploy-on-pypi.yml

@@ -22,10 +22,10 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - name: Set up Python 3.11
+      - name: Set up Python 3.12
         uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - name: Install pep517
         run: >-
           python -m

.github/workflows/create-condalock-file.yml

@@ -27,7 +27,7 @@ jobs:
         with:
           auto-update-conda: true
           activate-environment: esmvaltool-fromlock
-          python-version: "3.11"
+          python-version: "3.12"
           miniforge-version: "latest"
           miniforge-variant: Mambaforge
           use-mamba: true

.github/workflows/install-from-conda.yml

@@ -39,7 +39,7 @@ jobs:
     runs-on: "ubuntu-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
       # fail-fast set to False allows all other tests
       # in the workflow to run regardless of any fail
       fail-fast: false
@@ -75,7 +75,7 @@ jobs:
     runs-on: "macos-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
         architecture: ["x64"]  # need to force Intel, arm64 builds have issues
       fail-fast: false
     name: OSX Python ${{ matrix.python-version }}

.github/workflows/install-from-condalock-file.yml

@@ -29,7 +29,7 @@ jobs:
     runs-on: "ubuntu-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
       fail-fast: false
     name: Linux Python ${{ matrix.python-version }}
     steps:

.github/workflows/install-from-pypi.yml

@@ -39,7 +39,7 @@ jobs:
     runs-on: "ubuntu-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
       # fail-fast set to False allows all other tests
       # in the workflow to run regardless of any fail
       fail-fast: false
@@ -77,7 +77,7 @@ jobs:
     runs-on: "macos-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
         architecture: ["x64"]  # need to force Intel, arm64 builds have issues
       fail-fast: false
     name: OSX Python ${{ matrix.python-version }}

.github/workflows/install-from-source.yml

@@ -37,7 +37,7 @@ jobs:
     runs-on: "ubuntu-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
       fail-fast: false
     name: Linux Python ${{ matrix.python-version }}
     steps:
@@ -74,7 +74,7 @@ jobs:
     runs-on: "macos-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
         architecture: ["x64"]  # need to force Intel, arm64 builds have issues
       fail-fast: false
     name: OSX Python ${{ matrix.python-version }}

.github/workflows/run-tests-monitor.yml

@@ -22,7 +22,7 @@ jobs:
     runs-on: "ubuntu-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
       fail-fast: false
     name: Linux Python ${{ matrix.python-version }}
     steps:
@@ -55,7 +55,7 @@ jobs:
     runs-on: "macos-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
         architecture: ["x64"]  # need to force Intel, arm64 builds have issues
       fail-fast: false
     name: OSX Python ${{ matrix.python-version }}

.github/workflows/run-tests.yml

@@ -39,7 +39,7 @@ jobs:
     runs-on: "ubuntu-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
       fail-fast: false
     name: Linux Python ${{ matrix.python-version }}
     steps:
@@ -57,6 +57,7 @@ jobs:
       - run: mkdir -p test_linux_artifacts_python_${{ matrix.python-version }}
       - run: conda --version 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/conda_version.txt
       - run: python -V 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/python_version.txt
+      - run: conda list
       - run: pip install -e .[develop] 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/install.txt
       - run: conda list
       - run: flake8
@@ -72,7 +73,7 @@ jobs:
     runs-on: "macos-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
         architecture: ["x64"]  # need to force Intel, arm64 builds have issues
       fail-fast: false
     name: OSX Python ${{ matrix.python-version }}
@@ -93,6 +94,7 @@ jobs:
       - run: conda --version 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/conda_version.txt
       - run: python -V 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/python_version.txt
       - run: mamba install -c conda-forge git
+      - run: conda list
       - run: pip install -e .[develop] 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/install.txt
       - run: conda list
       - run: flake8

environment.yml

@@ -36,7 +36,7 @@ dependencies:
   - psutil
   - py-cordex
   - pybtex
-  - python >=3.9,<3.12
+  - python >=3.9
   - python-stratify >=0.3
   - pyyaml
   - requests
@@ -50,14 +50,14 @@ dependencies:
   - sphinx >=6.1.3
   - pydata-sphinx-theme
   # Python packages needed for testing
-  - flake8
   - mypy >=0.990
   - pytest >=3.9,!=6.0.0rc1,!=6.0.0
   - pytest-cov >=2.10.1
   - pytest-env
   - pytest-html !=2.1.0
   - pytest-metadata >=1.5.1
   - pytest-mock
+  - pytest-mypy
   - pytest-xdist
   # Not on conda-forge - ESMValTool_sample_data==0.0.3
   # Still for testing, MyPy library stubs
@@ -68,7 +68,11 @@ dependencies:
   - docformatter
   - isort
   - pre-commit
-  - prospector >=1.9.0
+  - pylint
+  - flake8 >= 7
+  - pydocstyle
   # Not on conda forge - vprof
   - yamllint
   - yapf
+  - pip:
+    - ESMValTool_sample_data

esmvalcore/_task.py

@@ -566,7 +566,7 @@ def _run(self, input_files):
 
         returncode = None
 
-        with resource_usage_logger(process.pid, self.resource_log),\
+        with resource_usage_logger(process.pid, self.resource_log), \
                 open(self.log, 'ab') as log:
             last_line = ['']
             while returncode is None:

setup.py

@@ -65,7 +65,7 @@
     ],
     # Test dependencies
     'test': [
-        'flake8',
+        'flake8>=7.0.0',  # not to pick up E231
         'pytest>=3.9,!=6.0.0rc1,!=6.0.0',
         'pytest-cov>=2.10.1',
         'pytest-env',
@@ -94,8 +94,10 @@
         'codespell',
         'docformatter',
         'isort',
+        'flake8>=7',
         'pre-commit',
-        'prospector[with_pyroma]>=1.9.0',
+        'pylint',
+        'pydocstyle',
         'vprof',
         'yamllint',
         'yapf',

tests/unit/main/test_main.py

@@ -24,7 +24,7 @@ def test_run_recipe_error(mocker, caplog):
     print(caplog.text)
 
     # Check that the exit status is 1
-    assert exit_.called_once_with(1)
+    exit_.assert_called_once_with(1)
 
     # Check that only the RecipeError is logged above DEBUG level
     errors = [r for r in caplog.records if r.levelname != 'DEBUG']