Merge branch 'recipe_test_workflow_prototype' into 3571_review_rose_a…

…pp_conf_files

.github/workflows/test-development.yml

@@ -12,7 +12,6 @@ on:
   push:
     branches:
       - main
-      - fix_recipe_filler_bkwds_incompatibility
   schedule:
     - cron: '0 0 * * *'
 

.zenodo.json

@@ -380,6 +380,11 @@
       "affiliation": "DLR, Germany",
       "name": "Bonnet, Pauline",
       "orcid": "0000-0003-3780-0784"
+    },
+    {
+      "affiliation": "MetOffice, UK",
+      "name": "Munday, Gregory",
+      "orcid": "0000-0003-4750-9923"
     }
   ],
   "description": "ESMValTool: A community diagnostic and performance metrics tool for routine evaluation of Earth system models in CMIP.",

CITATION.cff

@@ -391,6 +391,11 @@ authors:
     family-names: Bonnet
     given-names: Pauline
     orcid: "https://orcid.org/0000-0003-3780-0784"
+  - 
+    affiliation: "MetOffice, UK"
+    family-names: Munday
+    given-names: Gregory
+    orcid: "https://orcid.org/0000-0003-4750-9923"
 
 cff-version: 1.2.0
 date-released: 2023-12-20

doc/sphinx/source/index.rst

@@ -6,11 +6,31 @@
 Welcome to ESMValTool's documentation!
 ======================================
 
+To get a first impression of what ESMValTool and ESMValCore can do for you,
+have a look at our blog posts
+`Analysis-ready climate data with ESMValCore <https://blog.esciencecenter.nl/easy-ipcc-powered-by-esmvalcore-19a0b6366ea7>`_
+and
+`ESMValTool: Recipes for solid climate science <https://blog.esciencecenter.nl/esmvaltool-recipes-for-solid-climate-science-da5b33814f69>`_.
+
+A tutorial is available on https://tutorial.esmvaltool.org.
+
+A series of video lectures has been created by `ACCESS-NRI <https://access-nri.org.au>`_.
+While these are tailored for ACCESS users, they are still very informative.
+
+.. raw:: html
+
+   <iframe width="560" height="315" src="https://www.youtube.com/embed/videoseries?si=pUXrXB8C8bLRfQHY&amp;list=PLFjfi2xLaFpJp59LvDc1upQsj_xzFlFLc" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" referrerpolicy="strict-origin-when-cross-origin" allowfullscreen></iframe>
+
+|
+
+For more detailed information, the documentation is available below.
+
+Get in touch! Contact information is available :ref:`here <Support-and-Contact>`.
+
 .. include:: _sidebar.rst.inc
 
 Indices and tables
 ==================
 
 * :ref:`genindex`
 * :ref:`search`
-

doc/sphinx/source/input.rst

@@ -232,6 +232,8 @@ A list of the datasets for which a CMORizers is available is provided in the fol
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
 | Dataset                      | Variables (MIP)                                                                                      | Tier | Script language |
 +==============================+======================================================================================================+======+=================+
+| AERONET                      | od440aer, od550aer, od870aer (AERmon)                                                                |   3  | Python          |
++------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
 | AGCD                         | pr (Amon)                                                                                            |   2  | Python          |
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
 | ANU Climate                  | pr, tas, tasmin, tasmax (Amon)                                                                       |   3  | Python          |
@@ -269,7 +271,7 @@ A list of the datasets for which a CMORizers is available is provided in the fol
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
 | CowtanWay                    | tasa (Amon)                                                                                          |   2  | Python          |
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
-| CRU                          | tas, pr (Amon)                                                                                       |   2  | Python          |
+| CRU                          | tas, tasmin, tasmax,  pr, clt (Amon), evspsblpot (Emon)                                              |   2  | Python          |
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
 | CT2019                       | co2s (Amon)                                                                                          |   2  | Python          |
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+

doc/sphinx/source/recipes/index.rst

@@ -32,6 +32,7 @@ Atmosphere
    :maxdepth: 1
 
    recipe_miles
+   recipe_climate_patterns
    recipe_clouds
    recipe_cmug_h2o
    recipe_crem

doc/sphinx/source/recipes/recipe_climate_patterns.rst

@@ -0,0 +1,107 @@
+.. _recipes_climate_patterns:
+
+Generating Climate Patterns from CMIP6 Models
+=============================================
+
+Overview
+--------
+
+The recipe recipe_climate_patterns generates climate patterns from CMIP6 model
+datasets.
+
+.. note::
+  The regrid setting in the recipe is set to a 2.5x3.75 grid. This is done to
+  match the current resolution in the IMOGEN-JULES model, but can be
+  adjusted with no issues for a finer/coarser patterns grid.
+
+
+Available recipes and diagnostics
+---------------------------------
+
+Recipes are stored in esmvaltool/recipes/
+
+* recipe_climate_patterns.yml
+
+Diagnostics are stored in esmvaltool/diag_scripts/climate_patterns/
+
+* climate_patterns.py: generates climate patterns from input datasets
+* sub_functions.py: set of sub functions to assist with driving scripts
+* plotting.py: contains all plotting functions for driving scripts
+
+
+User settings in recipe
+-----------------------
+
+#. Script climate_patterns.py
+
+   *Required settings for script*
+
+   None
+
+   *Optional settings for script*
+
+   * jules_mode: output jules-specific var names + .nc files
+   * parallelise: parallelise over models or not
+   * area: calculate the patterns globally, or over land only
+
+   *Required settings for variables*
+
+   * short_name
+   * additional_datasets
+
+   *Optional settings for variables*
+
+   None
+
+   *Required settings for preprocessor*
+
+   * monthly_statistics: converts data to mean monthly data
+
+   *Optional settings for preprocessor*
+
+   * regrid: regrids data
+
+
+Variables
+---------
+
+#. Script climate_patterns.py
+
+* tasmax (atmos, monthly, longitude latitude time)
+* tasmin (atmos, monthly, longitude latitude time)
+* tas (atmos, monthly, longitude latitude time)
+* huss (atmos, monthly, longitude latitude time)
+* pr (atmos, monthly, longitude latitude time)
+* sfcWind (atmos, monthly, longitude latitude time)
+* ps (atmos, monthly, longitude latitude time)
+* rsds (atmos, monthly, longitude latitude time)
+* rlds (atmos, monthly, longitude latitude time)
+
+
+Observations and reformat scripts
+---------------------------------
+
+None
+
+References
+----------
+
+* Huntingford, C., Cox, P. An analogue model to derive additional climate
+  change scenarios from existing GCM simulations.
+  Climate Dynamics 16, 575–586 (2000). https://doi.org/10.1007/s003820000067
+
+* Mathison, C. T. et al. A rapid application emissions-to-impacts tool
+  for scenario assessment: Probabilistic Regional Impacts from Model patterns
+  and Emissions (PRIME).
+  EGUsphere [preprint], (2024). https://doi.org/10.5194/egusphere-2023-2932
+
+Example plots
+-------------
+
+.. _fig_climate_patterns_2:
+.. figure::  /recipes/figures/climate_patterns/patterns.png
+   :align:   center
+   :width: 80%
+
+   Patterns generated for CMIP6 models, gridded view. Patterns are shown per
+   variable, for the month of January.

doc/sphinx/source/recipes/recipe_seaborn.rst

@@ -16,11 +16,11 @@ Available recipes and diagnostics
 
 Recipes are stored in recipes/
 
-   * recipe_seaborn.yml
+* recipe_seaborn.yml
 
 Diagnostics are stored in diag_scripts/
 
-   * :ref:`seaborn_diag.py <api.esmvaltool.diag_scripts.seaborn_diag>`
+* :ref:`seaborn_diag.py <api.esmvaltool.diag_scripts.seaborn_diag>`
 
 
 Variables

doc/sphinx/source/recipes/recipe_template.rst.template

@@ -14,11 +14,11 @@ Available recipes and diagnostics
 
 Recipes are stored in esmvaltool/recipes/
 
-    * recipe_<mynewrecipe>.yml
+* recipe_<mynewrecipe>.yml
 
 Diagnostics are stored in esmvaltool/diag_scripts/<mynewdiag>/
 
-    * <mynewdiag.py/.ncl/.r>: one line scription
+* <mynewdiag.py/.ncl/.r>: one line scription
 
 
 User settings in recipe

environment.yml

@@ -21,13 +21,14 @@ dependencies:
   - distributed
   - ecmwf-api-client
   - eofs
-  - esmpy !=8.1.0,<8.6.0  # https://github.com/SciTools-incubator/iris-esmf-regrid/pull/342#issuecomment-2092921514
+  - esmpy >=8.6.0  # github.com/SciTools-incubator/iris-esmf-regrid/pull/342
   - esmvalcore 2.10.*
   - fiona
   - fire
+  - fsspec
   - gdal
   - iris >=3.6.1
-  - iris-esmf-regrid >=0.7.0
+  - iris-esmf-regrid >=0.10.0  # github.com/SciTools-incubator/iris-esmf-regrid/pull/342
   - jinja2
   - joblib
   - lime

environment_osx.yml

@@ -21,13 +21,14 @@ dependencies:
   - distributed
   - ecmwf-api-client
   - eofs
-  - esmpy !=8.1.0,<8.6.0  # https://github.com/SciTools-incubator/iris-esmf-regrid/pull/342#issuecomment-2092921514
+  - esmpy >=8.6.0  # github.com/SciTools-incubator/iris-esmf-regrid/pull/342
   - esmvalcore 2.10.*
   - fiona
+  - fsspec
   - fire
   - gdal
   - iris >=3.6.1
-  - iris-esmf-regrid >=0.7.0
+  - iris-esmf-regrid >=0.10.0  # github.com/SciTools-incubator/iris-esmf-regrid/pull/342
   - jinja2
   - joblib
   - lime

esmvaltool/cmorizers/data/cmor_config/AERONET.yml

@@ -0,0 +1,28 @@
+---
+# Filename
+filename: 'AOD_Level20_Monthly_V3.tar.gz'
+
+# Common global attributes for Cmorizer output
+attributes:
+  dataset_id: AERONET
+  version: 20230610
+  tier: 3
+  modeling_realm: atmos
+  project_id: OBS6
+  source: 'https://aeronet.gsfc.nasa.gov/new_web/download_all_v3_aod.html'
+  reference: 'aeronetv3'
+  comment:
+    'Notice to users: this data has recommended guidelines for use and publication,
+    please refer to https://aeronet.gsfc.nasa.gov/new_web/data_usage.html.'
+
+# Variables to cmorize
+variables:
+  od440aer:
+    mip: AERmon
+    wavelength: 440
+  od550aer:
+    mip: AERmon
+    wavelength: 551
+  od870aer:
+    mip: AERmon
+    wavelength: 870

esmvaltool/cmorizers/data/cmor_config/CRU.yml

@@ -1,15 +1,20 @@
 ---
 # Filename (will be extended by variable name)
-filename: 'cru_ts4.02.1901.2017.{raw_name}.dat.nc'
+# filename: 'cru_ts4.02.1901.2017.{raw_name}.dat.nc'
+filename: 'cru_ts4.07.1901.2022.{raw_name}.dat.nc'
 
 # Common global attributes for Cmorizer output
 attributes:
   dataset_id: CRU
-  version: 'TS4.02'
+  # version: TS4.02
+  version: 'TS4.07'
   tier: 2
   modeling_realm: reanaly
-  project_id: OBS
-  source: 'https://crudata.uea.ac.uk/cru/data/hrg/cru_ts_4.02/cruts.1811131722.v4.02/'
+  # project_id: OBS  # v4.02
+  project_id: OBS6
+  # source: 'https://crudata.uea.ac.uk/cru/data/hrg/cru_ts_4.02/cruts.1811131722.v4.02/'
+  # source: 'https://crudata.uea.ac.uk/cru/data/hrg/cru_ts_4.06/cruts.2205201912.v4.06/'
+  source: 'https://crudata.uea.ac.uk/cru/data/hrg/cru_ts_4.07/cruts.2304141047.v4.07/'
   reference: 'cru'
   comment: ''
 
@@ -19,7 +24,23 @@ variables:
     mip: Amon
     raw: tmp
     raw_units: celsius
+  tasmin:
+    mip: Amon
+    raw: tmn
+    raw_units: celsius
+  tasmax:
+    mip: Amon
+    raw: tmx
+    raw_units: celsius
   pr:
     mip: Amon
     raw: pre
     raw_units: kg m-2 month-1
+  evspsblpot:
+    mip: Emon
+    raw: pet
+    raw_units: kg m-2 day-1
+  clt:
+    mip: Amon
+    raw: cld
+    raw_units: percent

esmvaltool/cmorizers/data/datasets.yml

@@ -1,6 +1,13 @@
 # Dataset information
 ---
 datasets:
+  AERONET:
+    tier: 3
+    source: "https://aeronet.gsfc.nasa.gov/"
+    last_access: 2023-06-13
+    info: |
+      Aerosol Optical Depth information from a worldwide network of stations.
+
   AGCD:
     tier: 2
     source: "http://dx.doi.org/10.25914/6009600786063"
@@ -274,12 +281,19 @@ datasets:
 
   CRU:
     tier: 2
-    source: https://crudata.uea.ac.uk/cru/data/hrg/cru_ts_4.02/cruts.1811131722.v4.02/
-    last_access: 2019-05-16
+    source: https://crudata.uea.ac.uk/cru/data/hrg/cru_ts_4.07/cruts.2304141047.v4.07/
+    last_access: 2023-11-06
     info: |
-      Download the following files:
-        {raw_name}/cru_ts4.02.1901.2017.{raw_name}.dat.nc.gz
+      Files can be downloaded using the download script (latest version only)
+      or manually:
+        {raw_name}/cru_ts4.07.1901.2022.{raw_name}.dat.nc.gz
       where {raw_name} is the name of the desired variable(s).
+      Previous versions can be downloaded from the corresponding folders in
+      https://crudata.uea.ac.uk/cru/data/hrg/. ESMValTools formatter can be used
+      for older versions with minor adjustments of
+      ``esmvaltool/cmorizers/data/cmor_config/CRU.yml``
+      Exact time coordinates and number of stations are not available version
+      TS4.02.
 
   CT2019:
     tier: 2