Merge branch 'main' into update_esacci_soilmoisture

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ exclude: |
   ^esmvaltool/diag_scripts/cvdp/
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.6.0
     hooks:
       - id: check-added-large-files
       - id: check-ast
@@ -19,7 +19,7 @@ repos:
       - id: trailing-whitespace
         args: [--markdown-linebreak-ext=md]
   - repo: https://github.com/adrienverge/yamllint
-    rev: 'v1.31.0'
+    rev: 'v1.35.1'
     hooks:
       - id: yamllint
   - repo: local  # nclcodestyle is installed alongside ESMValTool
@@ -30,16 +30,16 @@ repos:
         language: system
         files: '\.(ncl|NCL)$'
   - repo: https://github.com/lorenzwalthert/precommit/  # Checks for R
-    rev: 'v0.3.2.9007'
+    rev: 'v0.4.2'
     hooks:
       - id: style-files  # styler
       - id: lintr
   - repo: https://github.com/codespell-project/codespell
-    rev: 'v2.2.4'
+    rev: 'v2.3.0'
     hooks:
       - id: codespell
   - repo: https://github.com/PyCQA/isort
-    rev: '5.12.0'
+    rev: '5.13.2'
     hooks:
       - id: isort
   - repo: https://github.com/pre-commit/mirrors-yapf
@@ -49,10 +49,10 @@ repos:
         additional_dependencies:
           - 'toml'
   - repo: https://github.com/myint/docformatter
-    rev: 'v1.6.5'
+    rev: 'v1.7.5'
     hooks:
       - id: docformatter
   - repo: https://github.com/pycqa/flake8
-    rev: '6.0.0'
+    rev: '5.0.4'
     hooks:
       - id: flake8

.zenodo.json

@@ -171,6 +171,11 @@
       "name": "Hagemann, Stefan",
       "orcid": "0000-0001-5444-2945"
     },
+    {
+      "affiliation": "University of Canterbury, New Zealand",
+      "name": "Hardacre, Catherine",
+      "orcid": "0000-0001-9093-4656"
+    },
     {
       "affiliation": "ISAC-CNR, Italy",
       "name": "von Hardenberg, Jost",
@@ -380,15 +385,20 @@
       "affiliation": "DLR, Germany",
       "name": "Bonnet, Pauline",
       "orcid": "0000-0003-3780-0784"
+    },
+    {
+      "affiliation": "MetOffice, UK",
+      "name": "Munday, Gregory",
+      "orcid": "0000-0003-4750-9923"
     }
   ],
   "description": "ESMValTool: A community diagnostic and performance metrics tool for routine evaluation of Earth system models in CMIP.",
   "license": {
     "id": "Apache-2.0"
   },
-  "publication_date": "2023-07-06",
+  "publication_date": "2024-07-04",
   "title": "ESMValTool",
-  "version": "v2.9.0",
+  "version": "v2.11.0",
   "communities": [ 
       { 
           "identifier": "is-enes3" 

CITATION.cff

@@ -391,13 +391,18 @@ authors:
     family-names: Bonnet
     given-names: Pauline
     orcid: "https://orcid.org/0000-0003-3780-0784"
+  - 
+    affiliation: "MetOffice, UK"
+    family-names: Munday
+    given-names: Gregory
+    orcid: "https://orcid.org/0000-0003-4750-9923"
 
 cff-version: 1.2.0
-date-released: 2023-12-20
+date-released: 2024-07-04
 doi: "10.5281/zenodo.3401363"
 license: "Apache-2.0"
 message: "If you use this software, please cite it using these metadata."
 repository-code: "https://github.com/ESMValGroup/ESMValTool/"
 title: ESMValTool
-version: "v2.10.0"
+version: "v2.11.0"
 ...

doc/sphinx/source/changelog.rst

@@ -3,6 +3,135 @@
 Changelog
 =========
 
+.. _changelog-v2-11-0:
+
+v2.11.0
+-------
+Highlights
+
+- Two new recipes have been added:
+
+  - Recipe :ref:`recipe_aod_aeronet_assess.yml <recipe_aod_aeronet_assess>`
+    evaluates model aerosol optical depth (AOD) climatologies against ground
+    based observations from the AeroNET measurement network.
+  - Recipe :ref:`recipe_climate_patterns.yml <recipes_climate_patterns>`
+    generates climate patterns from CMIP6 model datasets.
+
+- The ESACCI-WATERVAPOUR CMORizer now includes daily data and uses the
+  officially released CDR2 data.
+- Support for 5 new datasets have been added:
+
+  - AeroNET
+  - ANU Climate 2.0 Australian data
+  - Australian Gridded Climate Data(AGCD) precipitation
+  - NOAA-ERSST
+  - NSIDC-G02202-sh sea ice fraction
+
+- NEW TREND: First time release manager shout-outs!
+
+  - This is the first ESMValTool release managed by the Met Office! We want to
+    shout this out - and for all future first time release managers to
+    shout-out - to celebrate the growing, thriving ESMValTool community.
+
+This release includes
+
+Bug fixes
+~~~~~~~~~
+
+-  Recipe_ocean_quadmap: Update ATSR to match ESGF name (:pull:`3443`) by :user:`rbeucher`
+-  Fix recipe_bock20jgr_fig_8-10.yml (:pull:`3665`) by :user:`LisaBock`
+-  Update the list of datasets used in ``recipe_easy_ipcc.yml`` (:pull:`3710`) by :user:`bouweandela`
+
+Documentation
+~~~~~~~~~~~~~
+
+-  Improve release tools and documentation (:pull:`3462`) by :user:`bouweandela`
+-  Fix a typo in the references file (:pull:`3499`) by :user:`bouweandela`
+-  Fix recipe path in ``recipe_perfmetrics.rst`` (:pull:`3532`) by :user:`TomasTorsvik`
+-  Improved description of model evaluation recipes (:pull:`3541`) by :user:`schlunma`
+-  Remove double word in cmorizer documentation (:pull:`3553`) by :user:`bettina-gier`
+-  Fix Codacy badge (:pull:`3558`) by :user:`bouweandela`
+-  Update the release schedule for v2.11.0 (:pull:`3573`) by :user:`ehogan`
+-  Improve the formatting of the recipe documentation template (:pull:`3652`) by :user:`mo-gill`
+-  Add introduction material on the main documentation page (:pull:`3628`) by :user:`bouweandela`
+-  Avoid warning in documentation build (:pull:`3675`) by :user:`bouweandela`
+-  Update the list of broken recipes for ``v2.11.0`` (:pull:`3706`) by :user:`ehogan`
+
+Diagnostics
+~~~~~~~~~~~
+
+-  ``monitor/multi_dataset.py`` improvements: allow data w/o ``timerange`` and improve text formatting (:pull:`3528`) by :user:`schlunma`
+-  Allow datasets without ``project`` in multi_datasets.py (:pull:`3552`) by :user:`schlunma`
+-  Prevent overlapping time axis tick labels in monitoring recipe (:pull:`3682`) by :user:`schlunma`
+
+New recipe
+~~~~~~~~~~
+
+-  Add support for aerosol optical depth climatology metrics to the AutoAssess replacement (:pull:`3048`) by :user:`catherinehardacre`
+-  CMIP6 climate patterns  (:pull:`2785`) by :user:`mo-gregmunday`
+
+Observational and re-analysis dataset support
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+-  Add cmorizer scripts for NOAA-ERSST. (:pull:`1799`) by :user:`bjoernbroetz`
+-  Update OceanSODA-ETHZ CMORizer with new source file (:pull:`3535`) by :user:`TomasTorsvik`
+-  Add CMORizer script for NSIDC-G02202-sh sea ice fraction (:pull:`3512`) by :user:`flicj191`
+-  CMORizer Australian Gridded Climate Data(AGCD) precipitation (:pull:`3445`) by :user:`flicj191`
+-  Extend CMORizer NCEP-DOE-R2 (:pull:`3469`) by :user:`axel-lauer`
+-  Add comment to recipe_lauer13jclim regarding UWisc being superseded by MAC-LWP (:pull:`3537`) by :user:`rbeucher`
+-  Recipe_autoassess_landsurface_surfrad: Remove CERES-EBAF version to fix ESGF search (:pull:`3438`) by :user:`rbeucher`
+-  Updating ESACCI-WATERVAPOUR cmorizer (:pull:`3282`) by :user:`malininae`
+-  CMORiser for ANU Climate 2.0 Australian data (:pull:`3511`) by :user:`flicj191`
+-  Add AERONET cmorizer (:pull:`3227`) by :user:`zklaus`
+-  Update CRU CMORizer (:pull:`3381`) by :user:`lukruh`
+-  Fix recipe_check_obs to be aligned with DKRZ (:pull:`3673`) by :user:`LisaBock`
+-  Update AERONET data version (:pull:`3692`) by :user:`ehogan`
+
+Automatic testing
+~~~~~~~~~~~~~~~~~
+
+-  Move code into function in batch job generation script (:pull:`3491`) by :user:`bouweandela`
+-  Fix sklearn tests (:pull:`3506`) by :user:`schlunma`
+-  Pinned sklearn>=1.4.0 (:pull:`3508`) by :user:`schlunma`
+-  Update sklearn tests to be compatible with current pytest version (pytest >=8.0.0) (:pull:`3517`) by :user:`schlunma`
+-  Update sklearn tests to be compatible with current pytest version (pytest >=8.0.0) Part 2 (:pull:`3518`) by :user:`schlunma`
+-  [Circle/CI]Fix `test_installation_from_conda` Circle CI tests (:pull:`3538`) by :user:`valeriupredoi`
+-  [Github Actions] install git in OSX and add environment inspection (:pull:`3581`) by :user:`valeriupredoi`
+-  [CI Github Actions] Update (outdated) actions versions that produce Node.js warnings (:pull:`3586`) by :user:`valeriupredoi`
+-  Fix ``flake8==7`` linting issues (:pull:`3634`) by :user:`valeriupredoi`
+-  Use ``importlib`` as the import mode for ``pytest`` (:pull:`3672`) by :user:`ehogan`
+
+Installation
+~~~~~~~~~~~~
+
+-  Update dependencies (:pull:`3487`) by :user:`bouweandela`
+-  Merge v2.10.x into main (:pull:`3489`) by :user:`schlunma`
+-  Add imagehash package as an ESMValTool dependency (:pull:`3557`) by :user:`alistairsellar`
+-  Unpin ``r-akima`` (:pull:`3564`) by :user:`valeriupredoi`
+-  Adding pys2index dependency (:pull:`3577`) by :user:`ljoakim`
+-  Pin esmpy <8.6.0 (:pull:`3585`) by :user:`valeriupredoi`
+-  Pin R <4.3.0 (:pull:`3689`) by :user:`ehogan`
+-  Pin importlib_metadata <8 (:pull:`3700`) by :user:`ehogan`
+-  Pin matplotlib <3.9.0 on ESMValTool release branch (:pull:`3712`) by :user:`ehogan`
+
+Dependency updates
+~~~~~~~~~~~~~~~~~~
+
+-  Fix for ``recipe_seaice_drift.yml``: fix CRS transformer for "North Pole Stereographic" (:pull:`3531`) by :user:`flicj191`
+-  Fixed attribute handling in austral_jet/main.ncl for iris>=3.8 (:pull:`3603`) by :user:`schlunma`
+-  Fixed attribute handling in emergent constraint diagnostic for iris>=3.8 (:pull:`3605`) by :user:`schlunma`
+-  Update the name of the remapcon2 operator in R recipes (:pull:`3611`) by :user:`ehogan`
+-  Use ``iris.FUTURE.save_split_attrs = True`` to remove iris warning in many diagnostics (:pull:`3651`) by :user:`schlunma`
+-  Avoid concatenation error in recipe_pcrglobwb.yml (:pull:`3645`) by :user:`bouweandela`
+-  Update `scipy.integrate.simps` import (:pull:`3704`) by :user:`ehogan`
+
+Improvements
+~~~~~~~~~~~~
+
+-  Add native6, OBS6 and RAWOBS rootpaths to metoffice template in config-user-example.yml and remove temporary dir (:pull:`3613`) by :user:`alistairsellar`
+
+.. _changelog-v2-10-0:
+
 v2.10.0
 -------
 Highlights

doc/sphinx/source/community/release_strategy/release_strategy.rst

@@ -53,6 +53,10 @@ With the following release schedule, we strive to have three releases per year a
 
 Upcoming releases
 ^^^^^^^^^^^^^^^^^
+- 2.12.0 (TBD)
+
+Past releases
+^^^^^^^^^^^^^
 
 - 2.11.0 (Release Manager: Met Office: `Emma Hogan`_, `Chris Billows`_, `Ed Gillett`_)
 
@@ -61,16 +65,13 @@ Upcoming releases
 +============+============+========================================+=====================================+
 | 2024-04-22 |            | ESMValCore `Feature Freeze`_           |                                     |
 +------------+------------+----------------------------------------+-------------------------------------+
-| 2023-05-03 |            | ESMValCore released                    |                                     |
+| 2023-05-03 | 2024-07-03 | :esmvalcore-release:`v2.11.0` released | :ref:`esmvalcore:changelog-v2-11-0` |
 +------------+------------+----------------------------------------+-------------------------------------+
 | 2023-05-06 |            | ESMValTool `Feature Freeze`_           |                                     |
 +------------+------------+----------------------------------------+-------------------------------------+
-| 2023-05-17 |            | ESMValTool released                    |                                     |
+| 2023-05-17 | 2024-07-04 | :release:`v2.11.0` released            | :ref:`changelog-v2-11-0`            |
 +------------+------------+----------------------------------------+-------------------------------------+
 
-Past releases
-^^^^^^^^^^^^^
-
 - 2.10.0 (Release Manager: `Klaus Zimmermann`_)
 
 +------------+------------+----------------------------------------+-------------------------------------+

doc/sphinx/source/index.rst

@@ -6,11 +6,31 @@
 Welcome to ESMValTool's documentation!
 ======================================
 
+To get a first impression of what ESMValTool and ESMValCore can do for you,
+have a look at our blog posts
+`Analysis-ready climate data with ESMValCore <https://blog.esciencecenter.nl/easy-ipcc-powered-by-esmvalcore-19a0b6366ea7>`_
+and
+`ESMValTool: Recipes for solid climate science <https://blog.esciencecenter.nl/esmvaltool-recipes-for-solid-climate-science-da5b33814f69>`_.
+
+A tutorial is available on https://tutorial.esmvaltool.org.
+
+A series of video lectures has been created by `ACCESS-NRI <https://access-nri.org.au>`_.
+While these are tailored for ACCESS users, they are still very informative.
+
+.. raw:: html
+
+   <iframe width="560" height="315" src="https://www.youtube.com/embed/videoseries?si=pUXrXB8C8bLRfQHY&amp;list=PLFjfi2xLaFpJp59LvDc1upQsj_xzFlFLc" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" referrerpolicy="strict-origin-when-cross-origin" allowfullscreen></iframe>
+
+|
+
+For more detailed information, the documentation is available below.
+
+Get in touch! Contact information is available :ref:`here <Support-and-Contact>`.
+
 .. include:: _sidebar.rst.inc
 
 Indices and tables
 ==================
 
 * :ref:`genindex`
 * :ref:`search`
-

doc/sphinx/source/recipes/broken_recipe_list.rst

@@ -9,26 +9,41 @@ More details can be found in the :ref:`broken recipe policy
 <broken-recipe-policy>`.
 
 .. list-table:: Broken recipes
-   :widths: 25 25 25 25
+   :widths: 25 25 25 25 25
    :header-rows: 1
 
    * - Broken recipe
      - Affected diagnostics
+     - Broken since release
      - Problem
      - GitHub issue
-   * - `recipe_check_obs.yml`
-     - `ERA5_native6`
-     - Derivation of custom variables `rlus` and `rsus`
-     - `#1388 <https://github.com/ESMValGroup/ESMValCore/issues/1388>`_
    * - :ref:`recipe_julia.yml <recipe_examples>`
      - `example`
-     - fill values are not interpreted, resulting in an unusable plot
+     - v2.5.0
+     - Fill values are not interpreted, resulting in an unusable plot
      - `#2595 <https://github.com/ESMValGroup/ESMValTool/issues/2595>`_
-   * - :ref:`recipe_seaice_drift.yml <recipes_seaice_drift>`
-     - `sea_ice_drift_SCICEX`
-     - ``shapely 2`` issue
-     - `#3243 <https://github.com/ESMValGroup/ESMValTool/issues/3243>`_
-   * - :ref:`recipe_pysplot.yml <recipes_psyplot_diag>`
-     - `plot_map`
-     - ``shapely 2`` issue
-     - `#3483 <https://github.com/ESMValGroup/ESMValTool/issues/3483>`_
+   * - :ref:`recipe_climwip_brunner2019_med.yml <recipe_climwip>`
+     - All (preprocessor issue)
+     - v2.11.0
+     - Failed to run preprocessor function ``fix_metadata`` on the data: Unable to convert units
+     - `#3694 <https://github.com/ESMValGroup/ESMValTool/issues/3694>`_
+   * - :ref:`recipe_ocean_amoc.yml <XML_oceans>`
+     - ``diag_timeseries_amoc``, ``diag_transects``
+     - v2.11.0
+     - CESM1 CMIP5 Omon data no longer available
+     - `#3693 <https://github.com/ESMValGroup/ESMValTool/issues/3693>`_
+   * - :ref:`recipe_preprocessor_derive_test.yml <recipe_examples>`
+     - ``cmip6/toz``
+     - v2.11.0
+     - Failed to run preprocessor function ``derive`` on the data: Unable to convert units
+     - `#3709 <https://github.com/ESMValGroup/ESMValTool/issues/3709>`_
+   * - :ref:`recipe_russell18jgr.yml <nml_oceanmetrics>`
+     - ``Figure_4``
+     - v2.11.0
+     - CESM1 CMIP5 Omon data no longer available
+     - `#3693 <https://github.com/ESMValGroup/ESMValTool/issues/3693>`_
+   * - :ref:`recipe_wenzel14jgr.yml <recipe_wenzel14jgr>`
+     - ``diag_tsline_Fig2d``
+     - v2.11.0
+     - CESM1 CMIP5 Omon data no longer available
+     - `#3693 <https://github.com/ESMValGroup/ESMValTool/issues/3693>`_

doc/sphinx/source/recipes/index.rst

@@ -32,6 +32,7 @@ Atmosphere
    :maxdepth: 1
 
    recipe_miles
+   recipe_climate_patterns
    recipe_clouds
    recipe_cmug_h2o
    recipe_crem