Use ruff formatter and pre-commit (#2524)

Co-authored-by: Valeriu Predoi <[email protected]>

.circleci/config.yml

@@ -28,7 +28,6 @@ commands:
             mkdir -p test-reports
             . /opt/conda/etc/profile.d/conda.sh
             conda activate esmvaltool
-            flake8 -j 4
             pytest -n 4 --junitxml=test-reports/report.xml
             esmvaltool version
       - store_test_results:

.editorconfig

@@ -27,4 +27,3 @@ indent_size = 2
 
 [*.{md,Rmd}]
 trim_trailing_whitespace = false
-

.github/CODEOWNERS

@@ -1,3 +1 @@
-esmvalcore/cmor @jvegasbsc
 .github/workflows @valeriupredoi
-

.github/workflows/create-condalock-file.yml

@@ -71,8 +71,6 @@ jobs:
         run: |
           esmvaltool --help
           esmvaltool version
-      - name: Run flake8
-        run: flake8
       - name: Run pytests
         run: pytest -n 2 -m "not installation"
       # Automated PR

.github/workflows/install-from-condalock-file.yml

@@ -51,7 +51,6 @@ jobs:
       - run: pip install -e .[develop]
       - run: esmvaltool --help
       - run: esmvaltool version 2>&1 | tee source_install_linux_artifacts_python_${{ matrix.python-version }}/version.txt
-      - run: flake8
       - run: pytest -n 2 -m "not installation"
       - name: Upload artifacts
         if: ${{ always() }}  # upload artifacts even if fail

.github/workflows/run-tests.yml

@@ -60,7 +60,9 @@ jobs:
       - run: conda list
       - run: pip install -e .[develop] 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/install.txt
       - run: conda list
-      - run: flake8
+      - run: |
+          pre-commit install
+          pre-commit run -a
       - run: pytest -n 2 -m "not installation" 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/test_report.txt
       - name: Upload artifacts
         if: ${{ always() }}  # upload artifacts even if fail
@@ -97,7 +99,9 @@ jobs:
       - run: conda list
       - run: pip install -e .[develop] 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/install.txt
       - run: conda list
-      - run: flake8
+      - run: |
+          pre-commit install
+          pre-commit run -a
       - run: pytest -n 2 -m "not installation" 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/test_report.txt
       - name: Upload artifacts
         if: ${{ always() }}  # upload artifacts even if fail

.pre-commit-config.yaml

@@ -8,7 +8,9 @@ exclude: |
   (?x)
   ^doc/conf.py|
   ^esmvalcore/cmor/tables/|
-  ^esmvalcore/preprocessor/ne_masks/
+  ^esmvalcore/preprocessor/ne_masks/|
+  ^esmvalcore/preprocessor/shapefiles/
+
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.4.0
@@ -29,26 +31,15 @@ repos:
     rev: 'v2.2.4'
     hooks:
       - id: codespell
-  - repo: https://github.com/PyCQA/isort
-    rev: '5.12.0'
-    hooks:
-      - id: isort
-  - repo: https://github.com/pre-commit/mirrors-yapf
-    rev: 'v0.32.0'
-    hooks:
-      - id: yapf
-        additional_dependencies:
-          - 'toml'
-  - repo: https://github.com/myint/docformatter
-    rev: 'v1.6.5'
-    hooks:
-      - id: docformatter
-  - repo: https://github.com/pycqa/flake8
-    rev: '6.0.0'
+        additional_dependencies: [tomli]  # required for Python 3.10
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: "v0.4.10"
     hooks:
-      - id: flake8
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v1.2.0'
+    rev: 'v1.11.2'
     hooks:
       - id: mypy
         additional_dependencies:

.prospector.yml

@@ -10,16 +10,21 @@ test-warnings: true
 member-warnings: false
 
 pyroma:
-    run: true
+  run: true
 
 pep8:
-    full: true
+  full: true
+  # ignore rules that conflict with ruff formatter
+  # E203: https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#slices
+  # E501: https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules
+  # W503: https://pycodestyle.pycqa.org/en/latest/intro.html#error-codes
+  disable: ['E203', 'E501', 'W503']
 
 mypy:
   run: true
 
 pep257:
-    # disable rules that are allowed by the numpy convention
-    # see https://github.com/PyCQA/pydocstyle/blob/master/src/pydocstyle/violations.py
-    # and http://pydocstyle.readthedocs.io/en/latest/error_codes.html
-    disable: ['D107', 'D203', 'D212', 'D213', 'D402', 'D413', 'D416']
+  # disable rules that are allowed by the numpy convention
+  # see https://github.com/PyCQA/pydocstyle/blob/master/src/pydocstyle/violations.py
+  # and http://pydocstyle.readthedocs.io/en/latest/error_codes.html
+  disable: ['D107', 'D203', 'D212', 'D213', 'D402', 'D413', 'D416']

.yamllint

@@ -0,0 +1,9 @@
+---
+
+extends: default
+
+rules:
+  line-length:
+    level: warning
+    max: 120
+  octal-values: enable

CITATION.cff

@@ -208,7 +208,7 @@ authors:
     affiliation: "DLR, Germany"
     family-names: Cammarano
     given-names: Diego
-  - 
+  -
     affiliation: "ACCESS-NRI, Australia"
     family-names: Yousong
     given-names: Zeng

NOTICE

@@ -50,5 +50,3 @@ In addition to using the Software, we encourage the community to join the Softwa
 To join the ESMValTool Development Team, please contact Dr. Birgit Hassler ([email protected]) and Dr. Axel Lauer ([email protected]).
 
 ==========================================
-
-

doc/api/esmvalcore.dataset.rst

@@ -1,5 +1,5 @@
 Dataset
-=======
+========
 
 .. automodule:: esmvalcore.dataset
     :no-show-inheritance:

doc/api/esmvalcore.experimental.recipe.rst

@@ -1,7 +1,7 @@
 .. _api_recipe:
 
 Recipes
-=======
+========
 
 This section describes the :py:mod:`~esmvalcore.experimental.recipe` submodule of the API (:py:mod:`esmvalcore.experimental`).
 

doc/conf.py

@@ -90,7 +90,7 @@
 templates_path = ['_templates']
 
 # The suffix of source filenames.
-source_suffix = '.rst'
+source_suffix = {'.rst': 'restructuredtext'}
 
 # The encoding of source files.
 # source_encoding = 'utf-8-sig'

doc/contributing.rst

@@ -210,6 +210,14 @@ This includes checks for invalid syntax and formatting errors.
 automatically just before you commit your code.
 It knows knows which tool to run for each filetype, and therefore provides
 a convenient way to check your code.
+Install the pre-commit hooks by running
+
+.. code-block:: bash
+
+    pre-commit install
+
+to make sure your code is formatted correctly and does not contain mistakes
+whenever you commit some changes.
 
 Python
 ~~~~~~
@@ -229,20 +237,22 @@ the repository is cloned, e.g. ``cd ESMValCore``, and run `prospector <http://pr
 
    prospector esmvalcore/preprocessor/_regrid.py
 
-In addition to prospector, we use `flake8 <https://flake8.pycqa.org/en/latest/>`_
-to automatically check for bugs and formatting mistakes and
+In addition to prospector, we use `ruff <https://docs.astral.sh/ruff/>`_
+to automatically format the code and to check for certain bugs and
 `mypy <https://mypy.readthedocs.io>`_ for checking that
 `type hints <https://mypy.readthedocs.io/en/stable/cheat_sheet_py3.html>`_ are
 correct.
 Note that `type hints`_ are completely optional, but if you do choose to add
 them, they should be correct.
+Both `ruff`_ and `mypy`_ are automatically run by pre-commit.
 
 When you make a pull request, adherence to the Python development best practices
 is checked in two ways:
 
-#. As part of the unit tests, flake8_ and mypy_ are run by
-   `CircleCI <https://app.circleci.com/pipelines/github/ESMValGroup/ESMValCore>`_,
-   see the section on Tests_ for more information.
+#. A check that the code is formatted using the pre-commit hooks and does
+   not contain any mistakes that can be found by analyzing the code without
+   running it, is performed by
+   `pre-commit.ci <https://results.pre-commit.ci/latest/github/ESMValGroup/ESMValCore/main>`_.
 #. `Codacy <https://app.codacy.com/gh/ESMValGroup/ESMValCore/pullRequests>`_
    is a service that runs prospector (and other code quality tools) on changed
    files and reports the results.
@@ -259,42 +269,25 @@ If you suspect prospector or Codacy may be wrong, please ask the
 Note that running prospector locally will give you quicker and sometimes more
 accurate results than waiting for Codacy.
 
-Most formatting issues in Python code can be fixed automatically by
-running the commands
+Formatting issues in Python code can be fixed automatically by running the
+command
 
 ::
 
-   isort some_file.py
-
-to sort the imports in `the standard way <https://www.python.org/dev/peps/pep-0008/#imports>`__
-using `isort <https://pycqa.github.io/isort/>`__ and
-
-::
-
-   yapf -i some_file.py
-
-to add/remove whitespace as required by the standard using `yapf <https://github.com/google/yapf>`__,
-
-::
-
-   docformatter -i some_file.py
-
-to run `docformatter <https://github.com/myint/docformatter>`__ which helps
-formatting the docstrings (such as line length, spaces).
+   pre-commit run --all
 
 YAML
 ~~~~
 
-Please use `yamllint <https://yamllint.readthedocs.io>`_ to check that your
-YAML files do not contain mistakes.
-``yamllint`` checks for valid syntax, common mistakes like key repetition and
-cosmetic problems such as line length, trailing spaces, wrong indentation, etc.
+We use `yamllint <https://yamllint.readthedocs.io>`_ to check that YAML files
+do not contain mistakes. This is automatically run by pre-commit.
 
 Any text file
 ~~~~~~~~~~~~~
 
 A generic tool to check for common spelling mistakes is
 `codespell <https://pypi.org/project/codespell/>`__.
+This is automatically run by pre-commit.
 
 .. _documentation:
 
@@ -379,21 +372,22 @@ the individual checks.
 To build the documentation on your own computer, go to the directory where the
 repository was cloned and run
 
-::
+.. code-block:: bash
 
    sphinx-build doc doc/build
 
 or
 
-::
+.. code-block:: bash
 
    sphinx-build -Ea doc doc/build
 
 to build it from scratch.
 
 Make sure that your newly added documentation builds without warnings or
 errors and looks correctly formatted.
-CircleCI_ will build the documentation with the command:
+`CircleCI <https://app.circleci.com/pipelines/github/ESMValGroup/ESMValCore>`_
+will build the documentation with the command:
 
 .. code-block:: bash
 
@@ -720,7 +714,7 @@ If the Codacy check keeps failing, please run prospector locally.
 If necessary, ask the pull request author to do the same and to address the
 reported issues.
 See the section on code_quality_ for more information.
-Never merge a pull request with failing CircleCI or readthedocs checks.
+Never merge a pull request with failing pre-commit, CircleCI, or readthedocs checks.
 
 
 .. _how-to-make-a-release:
@@ -743,15 +737,15 @@ Perform the steps listed below with two persons, to reduce the risk of error.
    `PyPI <https://pypi.org/project/ESMValCore/>`__, and
    `readthedocs <https://readthedocs.org/dashboard/esmvalcore/users/>`__.
 
-The release of ESMValCore is tied to the release of ESMValTool. 
+The release of ESMValCore is tied to the release of ESMValTool.
 The detailed steps can be found in the ESMValTool
 :ref:`documentation <esmvaltool:release_steps>`.
-To start the procedure, ESMValCore gets released as a 
+To start the procedure, ESMValCore gets released as a
 release candidate to test the recipes in ESMValTool. If bugs are found
-during the testing phase of the release candidate, make as many release 
-candidates for ESMValCore as needed in order to fix them. 
+during the testing phase of the release candidate, make as many release
+candidates for ESMValCore as needed in order to fix them.
 
-To make a new release of the package, be it a release candidate or the final release, 
+To make a new release of the package, be it a release candidate or the final release,
 follow these steps:
 
 1. Check that all tests and builds work
@@ -795,13 +789,13 @@ Use the script
 to create create a draft of the release notes.
 This script uses the titles and labels of merged pull requests since the
 previous release.
-Open a discussion to allow members of the development team to nominate pull 
-requests as highlights. Add the most voted pull requests as highlights at the 
-beginning of changelog. After the highlights section, list any backward 
-incompatible changes that the release may include. The 
+Open a discussion to allow members of the development team to nominate pull
+requests as highlights. Add the most voted pull requests as highlights at the
+beginning of changelog. After the highlights section, list any backward
+incompatible changes that the release may include. The
 :ref:`backward compatibility policy<esmvaltool:backward-compatibility-policy>`.
-lists the information that should be provided by the developer of any backward 
-incompatible change. Make sure to also list any deprecations that the release 
+lists the information that should be provided by the developer of any backward
+incompatible change. Make sure to also list any deprecations that the release
 may include, as well as a brief description on how to upgrade a deprecated feature.
 Review the results, and if anything needs changing, change it on GitHub and
 re-run the script until the changelog looks acceptable.

doc/gensidebar.py

@@ -38,7 +38,7 @@ def _toctree():
     def _endl():
         lines.append("")
 
-    def _write(project, desc, link, mapping=conf['intersphinx_mapping']):
+    def _write(project, desc, link, mapping=conf["intersphinx_mapping"]):
         if project != conf_api:
             if do_gen:
                 args = desc, mapping[project][0], link

doc/quickstart/install.rst

@@ -195,13 +195,15 @@ To install from source for development, follow these instructions.
    e.g. ``pip install --trusted-host=pypi.python.org --trusted-host=pypi.org --trusted-host=files.pythonhosted.org -e .[develop]``
 -  Test that your installation was successful by running
    ``esmvaltool -h``.
+-  Install the :ref:`esmvaltool:pre-commit` hooks by running:
+   ``pre-commit install``.
 
 Pre-installed versions on HPC clusters / other servers
 ------------------------------------------------------
 
 
 If you would like to use pre-installed versions on HPC clusters (currently CEDA-JASMIN and DKRZ-Levante),
-and other servers (currently Met Office Linux estate), please have a look at 
+and other servers (currently Met Office Linux estate), please have a look at
 :ref:`these instructions <esmvaltool:install_on_hpc>`.
 
 

doc/quickstart/run.rst

@@ -79,7 +79,7 @@ This feature is available for projects that are hosted on the ESGF, i.e.
 CMIP3, CMIP5, CMIP6, CORDEX, and obs4MIPs.
 
 To control the strictness of the CMOR checker and the checks during concatenation
-on auxiliary coordinates, supplementary variables, and derived coordinates, 
+on auxiliary coordinates, supplementary variables, and derived coordinates,
 use the flag ``--check_level``:
 
 .. code:: bash

doc/recipe/index.rst

@@ -8,4 +8,3 @@ The recipe format
 
     Overview <overview>
     Preprocessor <preprocessor>
-