From 6685cd01e817a063281d7f773ecbeeb14ab818ec Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 2 Oct 2024 18:29:21 +0200 Subject: [PATCH 01/29] Remove obsolete config-user --- config-user-example.yml | 274 ---------------------------------------- 1 file changed, 274 deletions(-) delete mode 100644 config-user-example.yml diff --git a/config-user-example.yml b/config-user-example.yml deleted file mode 100644 index c102928db9..0000000000 --- a/config-user-example.yml +++ /dev/null @@ -1,274 +0,0 @@ -############################################################################### -# Example user configuration file for ESMValTool -############################################################################### -# -# Note for users: -# -------------- -# Site-specific entries for different HPC centers are given at the bottom of -# this file. Comment out/replace as needed. This default version of the file -# can be used in combination with the command line argument -# ``search_esgf=when_missing``. If only certain values are allowed for an -# option, these are listed after ``---``. The option in square brackets is the -# default value, i.e., the one that is used if this option is omitted in the -# file. -# -############################################################################### -# -# Note for developers: -# ------------------- -# Two identical copies of this file (``ESMValTool/config-user-example.yml`` and -# ``ESMValCore/esmvalcore/config-user.yml``) exist. If you change one of it, -# make sure to apply the changes to the other. -# -############################################################################### ---- - -# Destination directory where all output will be written -# Includes log files and performance stats. -output_dir: ~/esmvaltool_output - -# Auxiliary data directory -# Used by some recipes to look for additional datasets. -auxiliary_data_dir: ~/auxiliary_data - -# Automatic data download from ESGF --- [never]/when_missing/always -# Use automatic download of missing CMIP3, CMIP5, CMIP6, CORDEX, and obs4MIPs -# data from ESGF. ``never`` disables this feature, which is useful if you are -# working on a computer without an internet connection, or if you have limited -# disk space. ``when_missing`` enables the automatic download for files that -# are not available locally. ``always`` will always check ESGF for the latest -# version of a file, and will only use local files if they correspond to that -# latest version. -search_esgf: never - -# Directory for storing downloaded climate data -# Make sure to use a directory where you can store multiple GBs of data. Your -# home directory on a HPC is usually not suited for this purpose, so please -# change the default value in this case! -download_dir: ~/climate_data - -# Run at most this many tasks in parallel --- [null]/1/2/3/4/... -# Set to ``null`` to use the number of available CPUs. If you run out of -# memory, try setting max_parallel_tasks to ``1`` and check the amount of -# memory you need for that by inspecting the file ``run/resource_usage.txt`` in -# the output directory. Using the number there you can increase the number of -# parallel tasks again to a reasonable number for the amount of memory -# available in your system. -max_parallel_tasks: null - -# Log level of the console --- debug/[info]/warning/error -# For much more information printed to screen set log_level to ``debug``. -log_level: info - -# Exit on warning --- true/[false] -# # Only used in NCL diagnostic scripts. -exit_on_warning: false - -# Plot file format --- [png]/pdf/ps/eps/epsi -output_file_type: png - -# Remove the ``preproc`` directory if the run was successful --- [true]/false -# By default this option is set to ``true``, so all preprocessor output files -# will be removed after a successful run. Set to ``false`` if you need those -# files. -remove_preproc_dir: true - -# Use netCDF compression --- true/[false] -compress_netcdf: false - -# Save intermediary cubes in the preprocessor --- true/[false] -# Setting this to ``true`` will save the output cube from each preprocessing -# step. These files are numbered according to the preprocessing order. -save_intermediary_cubes: false - -# Path to custom ``config-developer.yml`` file -# This can be used to customise project configurations. See -# ``config-developer.yml`` for an example. Set to ``null`` to use the default. -config_developer_file: null - -# Use a profiling tool for the diagnostic run --- [false]/true -# A profiler tells you which functions in your code take most time to run. -# Only available for Python diagnostics. -profile_diagnostic: false - -# Rootpaths to the data from different projects -# This default setting will work if files have been downloaded by ESMValTool -# via ``search_esgf``. Lists are also possible. For site-specific entries and -# more examples, see below. Comment out these when using a site-specific path. -rootpath: - default: ~/climate_data - -# Directory structure for input data --- [default]/ESGF/BADC/DKRZ/ETHZ/etc. -# This default setting will work if files have been downloaded by ESMValTool -# via ``search_esgf``. See ``config-developer.yml`` for definitions. Comment -# out/replace as per needed. -drs: - CMIP3: ESGF - CMIP5: ESGF - CMIP6: ESGF - CORDEX: ESGF - obs4MIPs: ESGF - -# Example rootpaths and directory structure that showcases the different -# projects and also the use of lists -# For site-specific entries, see below. -#rootpath: -# CMIP3: [~/cmip3_inputpath1, ~/cmip3_inputpath2] -# CMIP5: [~/cmip5_inputpath1, ~/cmip5_inputpath2] -# CMIP6: [~/cmip6_inputpath1, ~/cmip6_inputpath2] -# OBS: ~/obs_inputpath -# OBS6: ~/obs6_inputpath -# obs4MIPs: ~/obs4mips_inputpath -# ana4mips: ~/ana4mips_inputpath -# native6: ~/native6_inputpath -# RAWOBS: ~/rawobs_inputpath -# default: ~/default_inputpath -#drs: -# CMIP3: default -# CMIP5: default -# CMIP6: default -# CORDEX: default -# obs4MIPs: default - -# Directory tree created by automatically downloading from ESGF -# Uncomment the lines below to locate data that has been automatically -# downloaded from ESGF (using ``search_esgf``). -#rootpath: -# CMIP3: ~/climate_data -# CMIP5: ~/climate_data -# CMIP6: ~/climate_data -# CORDEX: ~/climate_data -# obs4MIPs: ~/climate_data -#drs: -# CMIP3: ESGF -# CMIP5: ESGF -# CMIP6: ESGF -# CORDEX: ESGF -# obs4MIPs: ESGF - -# Site-specific entries: JASMIN -# Uncomment the lines below to locate data on JASMIN. -#auxiliary_data_dir: /gws/nopw/j04/esmeval/aux_data/AUX -#rootpath: -# CMIP6: /badc/cmip6/data/CMIP6 -# CMIP5: /badc/cmip5/data/cmip5/output1 -# CMIP3: /badc/cmip3_drs/data/cmip3/output -# OBS: /gws/nopw/j04/esmeval/obsdata-v2 -# OBS6: /gws/nopw/j04/esmeval/obsdata-v2 -# obs4MIPs: /gws/nopw/j04/esmeval/obsdata-v2 -# ana4mips: /gws/nopw/j04/esmeval/obsdata-v2 -# CORDEX: /badc/cordex/data/CORDEX/output -#drs: -# CMIP6: BADC -# CMIP5: BADC -# CMIP3: BADC -# CORDEX: BADC -# OBS: default -# OBS6: default -# obs4MIPs: default -# ana4mips: default - -# Site-specific entries: DKRZ-Levante -# For bd0854 members a shared download directory is available -#search_esgf: when_missing -#download_dir: /work/bd0854/DATA/ESMValTool2/download -# Uncomment the lines below to locate data on Levante at DKRZ. -#auxiliary_data_dir: /work/bd0854/DATA/ESMValTool2/AUX -#rootpath: -# CMIP6: /work/bd0854/DATA/ESMValTool2/CMIP6_DKRZ -# CMIP5: /work/bd0854/DATA/ESMValTool2/CMIP5_DKRZ -# CMIP3: /work/bd0854/DATA/ESMValTool2/CMIP3 -# CORDEX: /work/ik1017/C3SCORDEX/data/c3s-cordex/output -# OBS: /work/bd0854/DATA/ESMValTool2/OBS -# OBS6: /work/bd0854/DATA/ESMValTool2/OBS -# obs4MIPs: /work/bd0854/DATA/ESMValTool2/OBS -# ana4mips: /work/bd0854/DATA/ESMValTool2/OBS -# native6: /work/bd0854/DATA/ESMValTool2/RAWOBS -# RAWOBS: /work/bd0854/DATA/ESMValTool2/RAWOBS -#drs: -# CMIP6: DKRZ -# CMIP5: DKRZ -# CMIP3: DKRZ -# CORDEX: BADC -# obs4MIPs: default -# ana4mips: default -# OBS: default -# OBS6: default -# native6: default - -# Site-specific entries: ETHZ -# Uncomment the lines below to locate data at ETHZ. -#rootpath: -# CMIP6: /net/atmos/data/cmip6 -# CMIP5: /net/atmos/data/cmip5 -# CMIP3: /net/atmos/data/cmip3 -# OBS: /net/exo/landclim/PROJECTS/C3S/datadir/obsdir/ -#drs: -# CMIP6: ETHZ -# CMIP5: ETHZ -# CMIP3: ETHZ - -# Site-specific entries: IPSL -# Uncomment the lines below to locate data on Ciclad at IPSL. -#rootpath: -# IPSLCM: / -# CMIP5: /bdd/CMIP5/output -# CMIP6: /bdd/CMIP6 -# CMIP3: /bdd/CMIP3 -# CORDEX: /bdd/CORDEX/output -# obs4MIPs: /bdd/obs4MIPS/obs-CFMIP/observations -# ana4mips: /not_yet -# OBS: /not_yet -# OBS6: /not_yet -# RAWOBS: /not_yet -#drs: -# CMIP6: DKRZ -# CMIP5: DKRZ -# CMIP3: IPSL -# CORDEX: BADC -# obs4MIPs: IPSL -# ana4mips: default -# OBS: not_yet -# OBS6: not_yet - -# Site-specific entries: Met Office -# Uncomment the lines below to locate data at the Met Office. -#rootpath: -# CMIP5: /project/champ/data/cmip5/output1 -# CMIP6: /project/champ/data/CMIP6 -# CORDEX: /project/champ/data/cordex/output -# OBS: /data/users/esmval/ESMValTool/obs -# OBS6: /data/users/esmval/ESMValTool/obs -# obs4MIPs: /data/users/esmval/ESMValTool/obs -# ana4mips: /project/champ/data/ana4MIPs -# native6: /data/users/esmval/ESMValTool/rawobs -# RAWOBS: /data/users/esmval/ESMValTool/rawobs -#drs: -# CMIP5: BADC -# CMIP6: BADC -# CORDEX: BADC -# OBS: default -# OBS6: default -# obs4MIPs: default -# ana4mips: BADC -# native6: default - -# Site-specific entries: NCI -# Uncomment the lines below to locate data at NCI. -#rootpath: -# CMIP6: [/g/data/oi10/replicas/CMIP6, /g/data/fs38/publications/CMIP6, /g/data/xp65/public/apps/esmvaltool/replicas/CMIP6] -# CMIP5: [/g/data/r87/DRSv3/CMIP5, /g/data/al33/replicas/CMIP5/combined, /g/data/rr3/publications/CMIP5/output1, /g/data/xp65/public/apps/esmvaltool/replicas/cmip5/output1] -# CMIP3: /g/data/r87/DRSv3/CMIP3 -# OBS: /g/data/ct11/access-nri/replicas/esmvaltool/obsdata-v2 -# OBS6: /g/data/ct11/access-nri/replicas/esmvaltool/obsdata-v2 -# obs4MIPs: /g/data/ct11/access-nri/replicas/esmvaltool/obsdata-v2 -# ana4mips: /g/data/ct11/access-nri/replicas/esmvaltool/obsdata-v2 -# native6: /g/data/xp65/public/apps/esmvaltool/native6 -# -#drs: -# CMIP6: NCI -# CMIP5: NCI -# CMIP3: NCI -# CORDEX: ESGF -# obs4MIPs: default -# ana4mips: default From d15683e9780a1e6d5d30f3a33abceb85d5778be9 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 2 Oct 2024 18:29:54 +0200 Subject: [PATCH 02/29] Fix CI test --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5957a5e7e3..c0280138d6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -216,8 +216,8 @@ jobs: conda activate esmvaltool mkdir -p ~/climate_data esmvaltool config get_config_user - echo "search_esgf: when_missing" >> ~/.esmvaltool/config-user.yml - cat ~/.esmvaltool/config-user.yml + echo "search_esgf: when_missing" >> ~/.config/esmvaltool/config-user.yml + cat ~/.config/esmvaltool/config-user.yml for recipe in esmvaltool/recipes/testing/recipe_*.yml; do esmvaltool run "$recipe" done From 68910074301e553e3e1bafeb532fa633b2c40083 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 2 Oct 2024 18:30:13 +0200 Subject: [PATCH 03/29] Adapt doc --- doc/sphinx/source/community/dataset.rst | 9 ++--- doc/sphinx/source/community/diagnostic.rst | 23 ++++++------ .../detailed_release_procedure.rst | 4 +-- doc/sphinx/source/community/upgrading.rst | 6 ++-- doc/sphinx/source/develop/dataset.rst | 35 ++++++++++--------- doc/sphinx/source/faq.rst | 7 ---- doc/sphinx/source/functionalities.rst | 8 ++--- doc/sphinx/source/input.rst | 12 +++---- .../source/quickstart/configuration.rst | 19 +++++----- doc/sphinx/source/quickstart/output.rst | 33 +++++++++-------- doc/sphinx/source/quickstart/running.rst | 8 ++--- .../source/recipes/recipe_carvalhais14nat.rst | 32 ++++++++--------- doc/sphinx/source/recipes/recipe_climwip.rst | 7 ++-- doc/sphinx/source/recipes/recipe_gier20bg.rst | 14 ++++---- .../source/recipes/recipe_hydrology.rst | 8 ++--- .../source/recipes/recipe_ipccwg1ar6ch3.rst | 28 +++++++-------- doc/sphinx/source/recipes/recipe_kcs.rst | 4 +-- .../recipes/recipe_model_evaluation.rst | 6 ++-- doc/sphinx/source/recipes/recipe_monitor.rst | 6 ++-- doc/sphinx/source/recipes/recipe_oceans.rst | 12 +++---- .../source/recipes/recipe_shapeselect.rst | 2 +- .../source/recipes/recipe_wenzel14jgr.rst | 4 +-- .../source/recipes/recipe_wenzel16nat.rst | 19 +++++----- doc/sphinx/source/utils.rst | 9 ++--- 24 files changed, 156 insertions(+), 159 deletions(-) diff --git a/doc/sphinx/source/community/dataset.rst b/doc/sphinx/source/community/dataset.rst index 424d4d4694..620d46a3c2 100644 --- a/doc/sphinx/source/community/dataset.rst +++ b/doc/sphinx/source/community/dataset.rst @@ -42,14 +42,15 @@ and run the recipe, to make sure the CMOR checks pass without warnings or errors To test a pull request for a new CMORizer script: -#. Download the data following the instructions included in the script and place - it in the ``RAWOBS`` path specified in your ``config-user.yml`` +#. Download the data following the instructions included in the script and + place it in the ``RAWOBS`` `rootpath` specified in your :ref:`configuration + ` #. If available, use the downloading script by running ``esmvaltool data download --config_file `` #. Run the cmorization by running ``esmvaltool data format `` #. Copy the resulting data to the ``OBS`` (for CMIP5 compliant data) or ``OBS6`` - (for CMIP6 compliant data) path specified in your - ``config-user.yml`` + (for CMIP6 compliant data) `rootpath` specified in your :ref:`configuration + ` #. Run ``recipes/examples/recipe_check_obs.yml`` with the new dataset to check that the data can be used diff --git a/doc/sphinx/source/community/diagnostic.rst b/doc/sphinx/source/community/diagnostic.rst index 285815f7cf..1be820f7b8 100644 --- a/doc/sphinx/source/community/diagnostic.rst +++ b/doc/sphinx/source/community/diagnostic.rst @@ -64,7 +64,7 @@ If it is just a few simple scripts or packaging is not possible (i.e. for NCL) y and paste the source code into the ``esmvaltool/diag_scripts`` directory. If you have existing code in a compiled language like -C, C++, or Fortran that you want to re-use, the recommended way to proceed is to add Python bindings and publish +C, C++, or Fortran that you want to reuse, the recommended way to proceed is to add Python bindings and publish the package on PyPI so it can be installed as a Python dependency. You can then call the functions it provides using a Python diagnostic. @@ -134,9 +134,8 @@ Diagnostic output Typically, diagnostic scripts create plots, but any other output such as e.g. text files or tables is also possible. Figures should be saved in the ``plot_dir``, either in both ``.pdf`` and -``.png`` format (preferred), or -respect the ``output_file_type`` specified in the -:ref:`esmvalcore:user configuration file`. +``.png`` format (preferred), or respect the :ref:`configuration option +` ``output_file_type`` . Data should be saved in the ``work_dir``, preferably as a ``.nc`` (`NetCDF `__) file, following the `CF-Conventions `__ as much as possible. @@ -181,7 +180,7 @@ human inspection. In addition to provenance information, a caption is also added to the plots. Provenance information from the recipe is automatically recorded by ESMValCore, whereas -diagnostic scripts must include code specifically to record provenance. See below for +diagnostic scripts must include code specifically to record provenance. See below for documentation of provenance attributes that can be included in a recipe. When contributing a diagnostic, please make sure it records the provenance, and that no warnings related to provenance are generated when running the recipe. @@ -252,7 +251,7 @@ for example plot_types: errorbar: error bar plot -To use these items, include them in the provenance record dictionary in the form +To use these items, include them in the provenance record dictionary in the form :code:`key: [value]` i.e. for the example above as :code:`'plot_types': ['errorbar']`. @@ -275,8 +274,8 @@ Always use :func:`esmvaltool.diag_scripts.shared.run_diagnostic` at the end of y with run_diagnostic() as config: main(config) -Create a ``provenance_record`` for each diagnostic file (i.e. image or data -file) that the diagnostic script outputs. The ``provenance_record`` is a +Create a ``provenance_record`` for each diagnostic file (i.e. image or data +file) that the diagnostic script outputs. The ``provenance_record`` is a dictionary of provenance items, for example: .. code-block:: python @@ -296,15 +295,15 @@ dictionary of provenance items, for example: 'statistics': ['mean'], } -To save a matplotlib figure, use the convenience function -:func:`esmvaltool.diag_scripts.shared.save_figure`. Similarly, to save Iris cubes use +To save a matplotlib figure, use the convenience function +:func:`esmvaltool.diag_scripts.shared.save_figure`. Similarly, to save Iris cubes use :func:`esmvaltool.diag_scripts.shared.save_data`. Both of these functions take ``provenance_record`` as an argument and log the provenance accordingly. Have a look at the example Python diagnostic in `esmvaltool/diag_scripts/examples/diagnostic.py `_ for a complete example. -For any other files created, you will need to make use of a +For any other files created, you will need to make use of a :class:`esmvaltool.diag_scripts.shared.ProvenanceLogger` to log provenance. Include the following code directly after the file is saved: @@ -489,7 +488,7 @@ This includes the following items: * In-code documentation (comments, docstrings) * Code quality (e.g. no hardcoded pathnames) * No Codacy errors reported -* Re-use of existing functions whenever possible +* Reuse of existing functions whenever possible * Provenance implemented Run recipe diff --git a/doc/sphinx/source/community/release_strategy/detailed_release_procedure.rst b/doc/sphinx/source/community/release_strategy/detailed_release_procedure.rst index a73643f454..d0d7f74672 100644 --- a/doc/sphinx/source/community/release_strategy/detailed_release_procedure.rst +++ b/doc/sphinx/source/community/release_strategy/detailed_release_procedure.rst @@ -49,7 +49,7 @@ and attach it in the release testing issue; to record the environment in a yaml Modifications to configuration files need to be documented as well. To test recipes, it is recommended to only use the default options and DKRZ data directories, simply by uncommenting -the DKRZ-Levante block of a newly generated ``config-user.yml`` file. +the DKRZ-Levante block of a :ref:`newly generated configuration file `. Submit run scripts - test recipe runs ------------------------------------- @@ -61,7 +61,7 @@ You will have to set the name of your environment, your email address (if you wa More information on running jobs with SLURM on DKRZ/Levante can be found in the DKRZ `documentation `_. -You can also specify the path to your ``config-user.yml`` file where ``max_parallel_tasks`` can be set. The script was found to work well with ``max_parallel_tasks=8``. Some recipes need to be run with ``max_parallel_tasks=1`` (large memory requirements, CMIP3 data, diagnostic issues, ...). These recipes are listed in `ONE_TASK_RECIPES`. +You can also specify the path to your configuration directory where ``max_parallel_tasks`` can be set in a YAML file. The script was found to work well with ``max_parallel_tasks=8``. Some recipes need to be run with ``max_parallel_tasks=1`` (large memory requirements, CMIP3 data, diagnostic issues, ...). These recipes are listed in `ONE_TASK_RECIPES`. Some recipes need other job requirements, you can add their headers in the `SPECIAL_RECIPES` dictionary. Otherwise the header will be written following the template that is written in the lines below. If you want to exclude recipes, you can do so by uncommenting the `exclude` lines. diff --git a/doc/sphinx/source/community/upgrading.rst b/doc/sphinx/source/community/upgrading.rst index 9ed7f8b5b1..9a9b37f178 100644 --- a/doc/sphinx/source/community/upgrading.rst +++ b/doc/sphinx/source/community/upgrading.rst @@ -145,7 +145,7 @@ Many operations previously performed by the diagnostic scripts, are now included The backend operations are fully controlled by the ``preprocessors`` section in the recipe. Here, a number of preprocessor sets can be defined, with different options for each of the operations. The sets defined in this section are applied in the ``diagnostics`` section to preprocess a given variable. -It is recommended to proceed step by step, porting and testing each operation separately before proceeding with the next one. A useful setting in the user configuration file (``config-private.yml``) called ``write_intermediary_cube`` allows writing out the variable field after each preprocessing step, thus facilitating the comparison with the old version (e.g., after CMORization, level selection, after regridding, etc.). The CMORization step of the new backend exactly corresponds to the operation performed by the old backend (and stored in the ``climo`` directory, now called ``preprec``): this is the very first step to be checked, by simply comparing the intermediary file produced by the new backend after CMORization with the output of the old backend in the ``climo`` directorsy (see "Testing" below for instructions). +It is recommended to proceed step by step, porting and testing each operation separately before proceeding with the next one. A useful setting in the configuration called ``write_intermediary_cube`` allows writing out the variable field after each preprocessing step, thus facilitating the comparison with the old version (e.g., after CMORization, level selection, after regridding, etc.). The CMORization step of the new backend exactly corresponds to the operation performed by the old backend (and stored in the ``climo`` directory, now called ``preprec``): this is the very first step to be checked, by simply comparing the intermediary file produced by the new backend after CMORization with the output of the old backend in the ``climo`` directorsy (see "Testing" below for instructions). The new backend also performs variable derivation, replacing the ``calculate`` function in the ``variable_defs`` scripts. If the recipe which is being ported makes use of derived variables, the corresponding calculation must be ported from the ``./variable_defs/.ncl`` file to ``./esmvaltool/preprocessor/_derive.py``. @@ -159,7 +159,7 @@ In the new version, all settings are centralized in the recipe, completely repla Make sure the diagnostic script writes NetCDF output ====================================================== -Each diagnostic script is required to write the output of the anaylsis in one or more NetCDF files. This is to give the user the possibility to further look into the results, besides the plots, but (most importantly) for tagging purposes when publishing the data in a report and/or on a website. +Each diagnostic script is required to write the output of the analysis in one or more NetCDF files. This is to give the user the possibility to further look into the results, besides the plots, but (most importantly) for tagging purposes when publishing the data in a report and/or on a website. For each of the plot produced by the diagnostic script a single NetCDF file has to be generated. The variable saved in this file should also contain all the necessary metadata that documents the plot (dataset names, units, statistical methods, etc.). The files have to be saved in the work directory (defined in `cfg['work_dir']` and `config_user_info@work_dir`, for the python and NCL diagnostics, respectively). @@ -209,7 +209,7 @@ Before submitting a pull request, the code should be cleaned to adhere to the co Update the documentation ======================== -If necessary, add or update the documentation for your recipes in the corrsponding rst file, which is now in ``doc\sphinx\source\recipes``. Do not forget to also add the documentation file to the list in ``doc\sphinx\source\annex_c`` to make sure it actually appears in the documentation. +If necessary, add or update the documentation for your recipes in the corresponding rst file, which is now in ``doc\sphinx\source\recipes``. Do not forget to also add the documentation file to the list in ``doc\sphinx\source\annex_c`` to make sure it actually appears in the documentation. Open a pull request =================== diff --git a/doc/sphinx/source/develop/dataset.rst b/doc/sphinx/source/develop/dataset.rst index f3c168a17c..f624a44feb 100644 --- a/doc/sphinx/source/develop/dataset.rst +++ b/doc/sphinx/source/develop/dataset.rst @@ -76,7 +76,7 @@ for downloading (e.g. providing contact information, licence agreements) and using the observations. The unformatted (raw) observations should then be stored in the appropriate of these three folders. -For each additional dataset, an entry needs to be made to the file +For each additional dataset, an entry needs to be made to the file `datasets.yml `_. The dataset entry should contain: @@ -92,10 +92,10 @@ of the cmorizing script (see Section `4. Create a cmorizer for the dataset`_). 3.1 Downloader script (optional) -------------------------------- -A Python script can be written to download raw observations +A Python script can be written to download raw observations from source and store the data in the appropriate tier subdirectory of the folder ``RAWOBS`` automatically. -There are many downloading scripts available in +There are many downloading scripts available in `/esmvaltool/cmorizers/data/downloaders/datasets/ `_ where several data download mechanisms are provided: @@ -108,18 +108,18 @@ Note that the name of this downloading script has to be identical to the name of the dataset. Depending on the source server, the downloading script needs to contain paths to -raw observations, filename patterns and various necessary fields to retrieve +raw observations, filename patterns and various necessary fields to retrieve the data. -Default ``start_date`` and ``end_date`` can be provided in cases where raw data +Default ``start_date`` and ``end_date`` can be provided in cases where raw data are stored in daily, monthly, and yearly files. The downloading script for the given dataset can be run with: .. code-block:: console - esmvaltool data download --config_file + esmvaltool data download --config_dir -The options ``--start`` and ``--end`` can be added to the command above to +The options ``--start`` and ``--end`` can be added to the command above to restrict the download of raw data to a time range. They will be ignored if a specific dataset does not support it (i.e. because it is provided as a single file). Valid formats are ``YYYY``, ``YYYYMM`` and ``YYYYMMDD``. By default, already downloaded data are not overwritten @@ -128,7 +128,7 @@ unless the option ``--overwrite=True`` is used. 4. Create a cmorizer for the dataset ==================================== -There are many cmorizing scripts available in +There are many cmorizing scripts available in `/esmvaltool/cmorizers/data/formatters/datasets/ `_ where solutions to many kinds of format issues with observational data are @@ -158,7 +158,7 @@ configuration file: `MTE.yml `_ in the directory ``ESMValTool/esmvaltool/cmorizers/data/cmor_config/``. Note that both the name of this configuration file and the cmorizing script have to be -identical to the name of your dataset. +identical to the name of your dataset. It is recommended that you set ``project`` to ``OBS6`` in the configuration file. That way, the variables defined in the CMIP6 CMOR table, augmented with the custom variables described above, are available to your script. @@ -188,7 +188,8 @@ The main body of the CMORizer script must contain a function called with this exact call signature. Here, ``in_dir`` corresponds to the input directory of the raw files, ``out_dir`` to the output directory of final reformatted data set, ``cfg`` to the dataset-specific configuration file, -``cfg_user`` to the user configuration file, ``start_date`` to the start +``cfg_user`` to the configuration object (which behaves basically like a +dictionary), ``start_date`` to the start of the period to format, and ``end_date`` to the end of the period to format. If not needed, the last three arguments can be ignored using underscores. The return value of this function is ignored. All @@ -256,9 +257,9 @@ The cmorizing script for the given dataset can be run with: .. code-block:: console - esmvaltool data format --config_file + esmvaltool data format --config_dir -The options ``--start`` and ``--end`` can be added to the command above to +The options ``--start`` and ``--end`` can be added to the command above to restrict the formatting of raw data to a time range. They will be ignored if a specific dataset does not support it (i.e. because it is provided as a single file). Valid formats are ``YYYY``, ``YYYYMM`` and ``YYYYMMDD``. @@ -267,12 +268,12 @@ does not support it (i.e. because it is provided as a single file). Valid format The output path given in the configuration file is the path where your cmorized dataset will be stored. The ESMValTool will create a folder - with the correct tier information + with the correct tier information (see Section `2. Edit your configuration file`_) if that tier folder is not - already available, and then a folder named after the dataset. + already available, and then a folder named after the dataset. In this folder the cmorized data set will be stored as a NetCDF file. The cmorized dataset will be automatically moved to the correct tier - subfolder of your OBS or OBS6 directory if the option + subfolder of your OBS or OBS6 directory if the option ``--install=True`` is used in the command above and no such directory was already created. @@ -284,9 +285,9 @@ the cmorizing scripts can be run in a single command with: .. code-block:: console - esmvaltool data prepare --config_file + esmvaltool data prepare --config_dir -Note that options from the ```esmvaltool data download`` and +Note that options from the ```esmvaltool data download`` and ``esmvaltool data format`` commands can be passed to the above command. 6. Naming convention of the observational data files diff --git a/doc/sphinx/source/faq.rst b/doc/sphinx/source/faq.rst index 10c72bd2cb..4b7e79b5f6 100644 --- a/doc/sphinx/source/faq.rst +++ b/doc/sphinx/source/faq.rst @@ -59,13 +59,6 @@ This is a useful functionality because it allows the user to `fix` things on-the quitting the Ipython console, code execution continues as per normal. -Use multiple config-user.yml files -================================== - -The user selects the configuration yaml file at run time. It's possible to -have several configurations files. For instance, it may be practical to have one -config file for debugging runs and another for production runs. - Create a symbolic link to the latest output directory ===================================================== diff --git a/doc/sphinx/source/functionalities.rst b/doc/sphinx/source/functionalities.rst index 5b49c118a2..0098d95ded 100644 --- a/doc/sphinx/source/functionalities.rst +++ b/doc/sphinx/source/functionalities.rst @@ -12,9 +12,9 @@ that it can: - execute the workflow; and - output the desired collective data and media. -To facilitate these four steps, the user has control over the tool via -two main input files: the :ref:`user configuration file ` -and the :ref:`recipe `. The configuration file sets +To facilitate these four steps, the user has control over the tool via the +:ref:`configuration ` and the :ref:`recipe +`. The configuration sets user and site-specific parameters (like input and output paths, desired output graphical formats, logging level, etc.), whereas the recipe file sets data, preprocessing and diagnostic-specific parameters (data @@ -27,7 +27,7 @@ recyclable; the recipe file can be used for a large number of applications, since it may include as many datasets, preprocessors and diagnostics sections as the user deems useful. -Once the user configuration files and the recipe are at hand, the user +Once the configuration files and the recipe are at hand, the user can start the tool. A schematic overview of the ESMValTool workflow is depicted in the figure below. diff --git a/doc/sphinx/source/input.rst b/doc/sphinx/source/input.rst index 65aef57cd8..bd66ef89b3 100644 --- a/doc/sphinx/source/input.rst +++ b/doc/sphinx/source/input.rst @@ -76,7 +76,7 @@ For example, run to run the default example recipe and automatically download the required data to the directory ``~/climate_data``. -The data only needs to be downloaded once, every following run will re-use +The data only needs to be downloaded once, every following run will reuse previously downloaded data stored in this directory. See :ref:`esmvalcore:config-esgf` for a more in depth explanation and the available configuration options. @@ -117,7 +117,7 @@ OBS and OBS6 data is stored in the `esmeval` Group Workspace (GWS), and to be gr GWS, one must apply at https://accounts.jasmin.ac.uk/services/group_workspaces/esmeval/ ; after permission has been granted, the user is encouraged to use the data locally, and not move it elsewhere, to minimize both data transfers and stale disk usage; to note that Tier 3 data is subject to data protection restrictions; for further inquiries, -the GWS is adminstered by [Valeriu Predoi](mailto:valeriu.predoi@ncas.ac.uk). +the GWS is administered by [Valeriu Predoi](mailto:valeriu.predoi@ncas.ac.uk). Using a CMORizer script ----------------------- @@ -193,8 +193,8 @@ To CMORize one or more datasets, run: esmvaltool data format --config_file [CONFIG_FILE] [DATASET_LIST] -The path to the raw data to be CMORized must be specified in the :ref:`user -configuration file` as RAWOBS. +The `rootpath` to the raw data to be CMORized must be specified in the +:ref:`configuration ` as ``RAWOBS``. Within this path, the data are expected to be organized in subdirectories corresponding to the data tier: Tier2 for freely-available datasets (other than obs4MIPs and ana4mips) and Tier3 for restricted datasets (i.e., dataset which @@ -492,8 +492,8 @@ A list of all currently supported native datasets is :ref:`provided here A detailed description of how to include new native datasets is given :ref:`here `. -To use this functionality, users need to provide a path in the -:ref:`esmvalcore:user configuration file` for the ``native6`` project data +To use this functionality, users need to provide a ``rootpath`` in the +:ref:`configuration ` for the ``native6`` project data and/or the dedicated project used for the native dataset, e.g., ``ICON``. Then, in the recipe, they can refer to those projects. For example: diff --git a/doc/sphinx/source/quickstart/configuration.rst b/doc/sphinx/source/quickstart/configuration.rst index 34c29aac5c..4b28f748c8 100644 --- a/doc/sphinx/source/quickstart/configuration.rst +++ b/doc/sphinx/source/quickstart/configuration.rst @@ -1,28 +1,29 @@ -.. _config-user: +.. _config: ************* Configuration ************* The ``esmvaltool`` command is provided by the ESMValCore package, the -documentation on configuring ESMValCore can be found -:ref:`here `. -In particular, it is recommended to read the section on the -:ref:`User configuration file ` -and the section on +documentation on configuring ESMValCore can be found :ref:`here +`. +An overview of all configuration options can be found :ref:`here +`. +In particular, it is recommended to read the section on how to :ref:`specify +configuration options ` and the section on :ref:`Finding data `. -To install the default configuration file in the default location, run +To install the default configuration in the default location, run .. code:: bash esmvaltool config get_config_user -Note that this file needs to be customized using the instructions above, so +Note that this needs to be customized using the instructions above, so the ``esmvaltool`` command can find the data on your system, before it can run a recipe. There is a lesson available in the `ESMValTool tutorial `_ -that describes how to personalize the configuration file. It can be found +that describes how to personalize the configuration. It can be found `at this site `_. diff --git a/doc/sphinx/source/quickstart/output.rst b/doc/sphinx/source/quickstart/output.rst index 4a33e8ca42..33836f1c9a 100644 --- a/doc/sphinx/source/quickstart/output.rst +++ b/doc/sphinx/source/quickstart/output.rst @@ -5,8 +5,9 @@ Output ****** ESMValTool automatically generates a new output directory with every run. The -location is determined by the output_dir option in the config-user.yml file, -the recipe name, and the date and time, using the the format: YYYYMMDD_HHMMSS. +location is determined by the :ref:`configuration option +` ``output_dir``, the recipe name, and the date and +time, using the the format: YYYYMMDD_HHMMSS. For instance, a typical output location would be: output_directory/recipe_ocean_amoc_20190118_1027/ @@ -33,13 +34,15 @@ The preprocessed datasets will be stored to the preproc/ directory. Each variable in each diagnostic will have its own the `metadata.yml`_ interface files saved in the preproc directory. -If the option ``save_intermediary_cubes`` is set to ``true`` in the -config-user.yml file, then the intermediary cubes will also be saved here. -This option is set to false in the default ``config-user.yml`` file. +If the :ref:`configuration option ` +``save_intermediary_cubes`` is set to ``true`` , then the intermediary cubes +will also be saved here. +This option is set to ``false`` by default. -If the option ``remove_preproc_dir`` is set to ``true`` in the config-user.yml -file, then the preproc directory will be deleted after the run completes. This -option is set to true in the default ``config-user.yml`` file. +If the :ref:`configuration option ` +``remove_preproc_dir`` is set to ``true`` , then the preproc directory will be +deleted after the run completes. +This option is set to ``true`` by default. Run @@ -70,8 +73,8 @@ Plots ===== The plots directory is where diagnostics save their output figures. These -plots are saved in the format requested by the option `output_file_type` in the -config-user.yml file. +plots are saved in the format requested by the :ref:`configuration option +` ``output_file_type``. Settings.yml @@ -81,10 +84,10 @@ The settings.yml file is automatically generated by ESMValCore. For each diagnos a unique settings.yml file will be produced. The settings.yml file passes several global level keys to diagnostic scripts. -This includes several flags from the config-user.yml file (such as -'write_netcdf', 'write_plots', etc...), several paths which are specific to the -diagnostic being run (such as 'plot_dir' and 'run_dir') and the location on -disk of the metadata.yml file (described below). +This includes several flags from the configuration (such as +``write_netcdf``, ``write_plots``, etc...), several paths which are specific to +the diagnostic being run (such as ``plot_dir`` and ``run_dir``) and the +location on disk of the metadata.yml file (described below). .. code-block:: yaml @@ -147,5 +150,5 @@ As you can see, this is effectively a dictionary with several items including data paths, metadata and other information. There are several tools available in python which are built to read and parse -these files. The tools are avaialbe in the shared directory in the diagnostics +these files. The tools are available in the shared directory in the diagnostics directory. diff --git a/doc/sphinx/source/quickstart/running.rst b/doc/sphinx/source/quickstart/running.rst index 7f9cadbaa1..20cb8620b0 100644 --- a/doc/sphinx/source/quickstart/running.rst +++ b/doc/sphinx/source/quickstart/running.rst @@ -39,20 +39,20 @@ from ESGF to the local directory ``~/climate_data``, run The ``--search_esgf=when_missing`` option tells ESMValTool to search for and download the necessary climate data files, if they cannot be found locally. -The data only needs to be downloaded once, every following run will re-use +The data only needs to be downloaded once, every following run will reuse previously downloaded data. If you have all required data available locally, you can run the tool with ``--search_esgf=never`` argument (the default). Note that in that case the required data should be located in the directories -specified in your user configuration file. +specified in the configuration (see :ref:`esmvalcore:config_option_rootpath`). A third option ``--search_esgf=always`` is available. With this option, the tool will first check the ESGF for the needed data, regardless of any local data availability; if the data found on ESGF is newer than the local data (if any) or the user specifies a version of the data that is available only from the ESGF, then that data will be downloaded; otherwise, local data will be used. -Recall that the chapter :ref:`Configuring ESMValTool ` -provides an explanation of how to create your own config-user.yml file. +Recall that the chapter on :ref:`configuring ESMValTool ` +provides an explanation of how to set up the configuration. See :ref:`running esmvaltool ` in the ESMValCore documentation for a more complete introduction to the ``esmvaltool`` command. diff --git a/doc/sphinx/source/recipes/recipe_carvalhais14nat.rst b/doc/sphinx/source/recipes/recipe_carvalhais14nat.rst index dc26a745e2..b551bbbdc5 100644 --- a/doc/sphinx/source/recipes/recipe_carvalhais14nat.rst +++ b/doc/sphinx/source/recipes/recipe_carvalhais14nat.rst @@ -73,7 +73,7 @@ The settings needed for loading the observational dataset in all diagnostics are provided in the recipe through `obs_info` within `obs_details` section. * ``obs_data_subdir``: subdirectory of auxiliary_data_dir (set in - config-user file) where observation data are stored {e.g., + configuration) where observation data are stored {e.g., data_ESMValTool_Carvalhais2014}. * ``source_label``: source data label {'Carvalhais2014'}. * ``variant_label``: variant of the observation {'BE'} for best estimate. @@ -112,7 +112,7 @@ Script land_carbon_cycle/diag_global_turnover.py * ``y0``: {``float``, 1.0} Y - coordinate of the upper edge of the figure. * ``wp``: {``float``, 1 / number of models} - width of each map. * ``hp``: {``float``, = wp} - height of each map. - * ``xsp``: {``float``, 0} - spacing betweeen maps in X - direction. + * ``xsp``: {``float``, 0} - spacing between maps in X - direction. * ``ysp``: {``float``, -0.03} - spacing between maps in Y -direction. Negative to reduce the spacing below default. * ``aspect_map``: {``float``, 0.5} - aspect of the maps. @@ -217,10 +217,10 @@ Due to inherent dependence of the diagnostic on uncertainty estimates in observation, the data needed for each diagnostic script are processed at different spatial resolutions (as in Carvalhais et al., 2014), and provided in 11 different resolutions (see Table 1). Note that the uncertainties were -estimated at the resolution of the selected models, and, thus, only the -pre-processed observed data can be used with the recipe. -It is not possible to use regridding functionalities of ESMValTool to regrid -the observational data to other spatial resolutions, as the uncertainty +estimated at the resolution of the selected models, and, thus, only the +pre-processed observed data can be used with the recipe. +It is not possible to use regridding functionalities of ESMValTool to regrid +the observational data to other spatial resolutions, as the uncertainty estimates cannot be regridded. Table 1. A summary of the observation datasets at different resolutions. @@ -309,7 +309,7 @@ Example plots Comparison of latitudinal (zonal) variations of pearson correlation between turnover time and climate: turnover time and precipitation, controlled for - temperature (left) and vice-versa (right). Reproduces figures 2c and 2d in + temperature (left) and vice-versa (right). Reproduces figures 2c and 2d in `Carvalhais et al. (2014)`_. .. _fig_carvalhais14nat_2: @@ -320,7 +320,7 @@ Example plots Comparison of observation-based and modelled ecosystem carbon turnover time. Along the diagnonal, tau_ctotal are plotted, above the bias, and below - density plots. The inset text in density plots indicate the correlation. + density plots. The inset text in density plots indicate the correlation. .. _fig_carvalhais14nat_3: @@ -328,11 +328,11 @@ Example plots :align: center :width: 80% - Global distributions of multimodel bias and model agreement. Multimodel bias - is calculated as the ratio of multimodel median turnover time and that from - observation. Stippling indicates the regions where only less than one - quarter of the models fall within the range of observational uncertainties - (`5^{th}` and `95^{th}` percentiles). Reproduces figure 3 in `Carvalhais et + Global distributions of multimodel bias and model agreement. Multimodel bias + is calculated as the ratio of multimodel median turnover time and that from + observation. Stippling indicates the regions where only less than one + quarter of the models fall within the range of observational uncertainties + (`5^{th}` and `95^{th}` percentiles). Reproduces figure 3 in `Carvalhais et al. (2014)`_. .. _fig_carvalhais14nat_4: @@ -341,7 +341,7 @@ Example plots :align: center :width: 80% - Comparison of latitudinal (zonal) variations of observation-based and - modelled ecosystem carbon turnover time. The zonal turnover time is - calculated as the ratio of zonal `ctotal` and `gpp`. Reproduces figures 2a + Comparison of latitudinal (zonal) variations of observation-based and + modelled ecosystem carbon turnover time. The zonal turnover time is + calculated as the ratio of zonal `ctotal` and `gpp`. Reproduces figures 2a and 2b in `Carvalhais et al. (2014)`_. diff --git a/doc/sphinx/source/recipes/recipe_climwip.rst b/doc/sphinx/source/recipes/recipe_climwip.rst index 0928ba939f..45e23fe685 100644 --- a/doc/sphinx/source/recipes/recipe_climwip.rst +++ b/doc/sphinx/source/recipes/recipe_climwip.rst @@ -43,9 +43,8 @@ Using shapefiles for cutting scientific regions To use shapefiles for selecting SREX or AR6 regions by name it is necessary to download them, e.g., from the sources below and reference the file using the `shapefile` parameter. This can either be a -absolute or a relative path. In the example recipes they are stored in a subfolder `shapefiles` -in the `auxiliary_data_dir` (with is specified in the -`config-user.yml `_). +absolute or a relative path. In the example recipes they are stored in a subfolder `shapefiles` +in the :ref:`configuration option ` `auxiliary_data_dir`. SREX regions (AR5 reference regions): http://www.ipcc-data.org/guidelines/pages/ar5_regions.html @@ -249,7 +248,7 @@ Brunner et al. (2020) recipe and example independence weighting The recipe uses an additional step between pre-processor and weight calculation to calculate anomalies relative to the global mean (e.g., tas_ANOM = tas_CLIM - global_mean(tas_CLIM)). This means we do not use the absolute temperatures of a model as performance criterion but rather the horizontal temperature distribution (see `Brunner et al. 2020 `_ for a discussion). -This recipe also implements a somewhat general independence weighting for CMIP6. In contrast to model performance (which should be case specific) model independence can largely be seen as only dependet on the multi-model ensemble in use but not the target variable or region. This means that the configuration used should be valid for similar subsets of CMIP6 as used in this recipe: +This recipe also implements a somewhat general independence weighting for CMIP6. In contrast to model performance (which should be case specific) model independence can largely be seen as only dependent on the multi-model ensemble in use but not the target variable or region. This means that the configuration used should be valid for similar subsets of CMIP6 as used in this recipe: .. code-block:: yaml diff --git a/doc/sphinx/source/recipes/recipe_gier20bg.rst b/doc/sphinx/source/recipes/recipe_gier20bg.rst index bb11770a24..b8f8fb9b8e 100644 --- a/doc/sphinx/source/recipes/recipe_gier20bg.rst +++ b/doc/sphinx/source/recipes/recipe_gier20bg.rst @@ -53,7 +53,7 @@ User settings in recipe * Optional diag_script_info attributes: * ``styleset``: styleset for color coding panels - * ``output_file_type``: output file type for plots, default: config_user -> png + * ``output_file_type``: output file type for plots, default: png * ``var_plotname``: NCL string formatting how variable should be named in plots defaults to short_name if not assigned. @@ -64,7 +64,7 @@ User settings in recipe amplitude contour plot * Optional diag_script_info attributes: - * ``output_file_type``: output file type for plots, default: config_user -> png + * ``output_file_type``: output file type for plots, default: png #. Script xco2_analysis/main.ncl: @@ -77,7 +77,7 @@ User settings in recipe accounting for the ensemble member named in "ensemble_refs" * Optional diag_script_info attributes: - * ``output_file_type``: output file type for plots, default: config_user -> png + * ``output_file_type``: output file type for plots, default: png * ``ensemble_refs``: list of model-ensemble pairs to denote which ensemble member to use for calculating multi-model mean. required if ensemble_mean = true @@ -97,17 +97,17 @@ User settings in recipe * ``plot_var2_mean``: If True adds mean of seasonal cycle to panel as string. * Optional diag_script_info attributes: - * ``output_file_type``: output file type for plots, default: config_user -> png + * ``output_file_type``: output file type for plots, default: png * ``var_plotname``: String formatting how variable should be named in plots defaults to short_name if not assigned #. Script xco2_analysis/sat_masks.ncl: * Optional diag_script_info attributes: - * ``output_file_type``: output file type for plots, default: config_user -> png + * ``output_file_type``: output file type for plots, default: png * ``var_plotname``: String formatting how variable should be named in plots defaults to short_name if not assigned - * ``c3s_plots``: Missing value plots seperated by timeseries of c3s satellites + * ``c3s_plots``: Missing value plots separated by timeseries of c3s satellites #. Script xco2_analysis/station_comparison.ncl: @@ -116,7 +116,7 @@ User settings in recipe first, then 2D variable, followed by surface stations * Optional diag_script_info attributes: - * ``output_file_type``: output file type for plots, default: config_user -> png + * ``output_file_type``: output file type for plots, default: png * ``var_plotnames``: String formatting how variables should be named in plots defaults to short_name if not assigned * ``overwrite_altitudes``: Give other altitude values than the ones attached in diff --git a/doc/sphinx/source/recipes/recipe_hydrology.rst b/doc/sphinx/source/recipes/recipe_hydrology.rst index d0e2e0bcb3..995a70b3ae 100644 --- a/doc/sphinx/source/recipes/recipe_hydrology.rst +++ b/doc/sphinx/source/recipes/recipe_hydrology.rst @@ -62,13 +62,13 @@ Diagnostics are stored in esmvaltool/diag_scripts/hydrology * wflow.py * lisflood.py * hype.py - * globwat.py + * globwat.py User settings in recipe ----------------------- -All hydrological recipes require a shapefile as an input to produce forcing data. This shapefile determines the shape of the basin for which the data will be cut out and processed. All recipes are tested with `the shapefiles `_ that are used for the eWaterCycle project. In principle any shapefile can be used, for example, the freely available basin shapefiles from the `HydroSHEDS project `_. +All hydrological recipes require a shapefile as an input to produce forcing data. This shapefile determines the shape of the basin for which the data will be cut out and processed. All recipes are tested with `the shapefiles `_ that are used for the eWaterCycle project. In principle any shapefile can be used, for example, the freely available basin shapefiles from the `HydroSHEDS project `_. #. recipe_pcrglobwb.yml @@ -87,7 +87,7 @@ All hydrological recipes require a shapefile as an input to produce forcing data *extract_shape:* - * shapefile: Meuse.shp (MARRMoT is a hydrological Lumped model that needs catchment-aggregated forcing data. The catchment is provided as a shapefile, the path can be relative to ``auxiliary_data_dir`` as defined in config-user.yml.). + * shapefile: Meuse.shp (MARRMoT is a hydrological Lumped model that needs catchment-aggregated forcing data. The catchment is provided as a shapefile, the path can be relative to :ref:`configuration option ` ``auxiliary_data_dir``). * method: contains * crop: true @@ -107,7 +107,7 @@ All hydrological recipes require a shapefile as an input to produce forcing data * dem_file: netcdf file containing a digital elevation model with elevation in meters and coordinates latitude and longitude. A wflow example dataset is available at: https://github.com/openstreams/wflow/tree/master/examples/wflow_rhine_sbm - The example dem_file can be obtained from https://github.com/openstreams/wflow/blob/master/examples/wflow_rhine_sbm/staticmaps/wflow_dem.map + The example dem_file can be obtained from https://github.com/openstreams/wflow/blob/master/examples/wflow_rhine_sbm/staticmaps/wflow_dem.map * regrid: the regridding scheme for regridding to the digital elevation model. Choose ``area_weighted`` (slow) or ``linear``. #. recipe_lisflood.yml diff --git a/doc/sphinx/source/recipes/recipe_ipccwg1ar6ch3.rst b/doc/sphinx/source/recipes/recipe_ipccwg1ar6ch3.rst index 42bedcec09..ebfb4e5cfd 100644 --- a/doc/sphinx/source/recipes/recipe_ipccwg1ar6ch3.rst +++ b/doc/sphinx/source/recipes/recipe_ipccwg1ar6ch3.rst @@ -6,7 +6,7 @@ IPCC AR6 Chapter 3 (selected figures) Overview -------- -This recipe collects selected diagnostics used in IPCC AR6 WGI Chapter 3: +This recipe collects selected diagnostics used in IPCC AR6 WGI Chapter 3: Human influence on the climate system (`Eyring et al., 2021`_). Plots from IPCC AR6 can be readily reproduced and compared to previous versions. The aim is to be able to start with what was available now the next time allowing us to focus @@ -15,7 +15,8 @@ on developing more innovative analysis methods rather than constantly having to Processing of CMIP3 models currently works only in serial mode, due to an issue in the input data still under investigation. To run the recipe for Fig 3.42a -and Fig. 3.43 set "max_parallel_tasks: 1" in the config-user.yml file. +and Fig. 3.43 set the :ref:`configuration option ` +``max_parallel_tasks: 1``. The plots are produced collecting the diagnostics from individual recipes. The following figures from `Eyring et al. (2021)`_ can currently be reproduced: @@ -43,10 +44,9 @@ To reproduce Fig. 3.9 you need the shapefile of the `AR6 reference regions (`Iturbide et al., 2020 `_). Please download the file `IPCC-WGI-reference-regions-v4_shapefile.zip `_, -unzip and store it in `/IPCC-regions/` (the `auxiliary_data_dir` -is defined in the `config-user.yml -`_ -file). +unzip and store it in `/IPCC-regions/` (where +`auxiliary_data_dir` is given as :ref:`configuration option +`). .. _`Eyring et al., 2021`: https://www.ipcc.ch/report/ar6/wg1/chapter/chapter-3/ .. _`Eyring et al. (2021)`: https://www.ipcc.ch/report/ar6/wg1/chapter/chapter-3/ @@ -179,7 +179,7 @@ User settings in recipe * start_year: start year in figure * end_year: end year in figure - * panels: list of variable blocks for each panel + * panels: list of variable blocks for each panel *Optional settings for script* @@ -205,7 +205,7 @@ User settings in recipe * plot_units: variable unit for plotting * y-min: set min of y-axis * y-max: set max of y-axis - * order: order in which experiments should be plotted + * order: order in which experiments should be plotted * stat_shading: if true: shading of statistic range * ref_shading: if true: shading of reference period @@ -225,7 +225,7 @@ User settings in recipe * plot_legend: if true, plot legend will be plotted * plot_units: variable unit for plotting - * multi_model_mean: if true, multi-model mean and uncertaintiy will be + * multi_model_mean: if true, multi-model mean and uncertaintiy will be plotted *Optional settings for variables* @@ -304,7 +304,7 @@ User settings in recipe * labels: List of labels for each variable on the x-axis * model_spread: if True, model spread is shaded * plot_median: if True, median is plotted - * project_order: give order of projects + * project_order: give order of projects Variables @@ -452,7 +452,7 @@ Example plots 2013). For line colours see the legend of Figure 3.4. Additionally, the multi-model mean (red) and standard deviation (grey shading) are shown. Observational and model datasets were detrended by removing the - least-squares quadratic trend. + least-squares quadratic trend. .. figure:: /recipes/figures/ipccwg1ar6ch3/tas_anom_damip_global_1850-2020.png :align: center @@ -467,7 +467,7 @@ Example plots anomalies are shown relative to 1950-2010 for Antarctica and relative to 1850-1900 for other continents. CMIP6 historical simulations are expanded by the SSP2-4.5 scenario simulations. All available ensemble members were used. - Regions are defined by Iturbide et al. (2020). + Regions are defined by Iturbide et al. (2020). .. figure:: /recipes/figures/ipccwg1ar6ch3/model_bias_pr_annualclim_CMIP6.png :align: center @@ -487,7 +487,7 @@ Example plots show a change greater than the variability threshold; crossed lines indicate regions with conflicting signal, where >=66% of models show change greater than the variability threshold and <80% of all models agree on the sign of - change. + change. .. figure:: /recipes/figures/ipccwg1ar6ch3/precip_anom_1950-2014.png :align: center @@ -511,7 +511,7 @@ Example plots forcings (brown) and natural forcings only (blue). Observed trends for each observational product are shown as horizontal lines. Panel (b) shows annual mean precipitation rate (mm day-1) of GHCN version 2 for the years 1950-2014 - over land areas used to compute the plots. + over land areas used to compute the plots. .. figure:: /recipes/figures/ipccwg1ar6ch3/zonal_westerly_winds.png :align: center diff --git a/doc/sphinx/source/recipes/recipe_kcs.rst b/doc/sphinx/source/recipes/recipe_kcs.rst index fa07f0a167..1ed117ecb6 100644 --- a/doc/sphinx/source/recipes/recipe_kcs.rst +++ b/doc/sphinx/source/recipes/recipe_kcs.rst @@ -30,7 +30,7 @@ In the second diagnostic, for both the control and future periods, the N target 2. Further constrain the selection by picking samples that represent either high or low changes in summer precipitation and summer and winter temperature, by limiting the remaining samples to certain percentile ranges: relatively wet/cold in the control and dry/warm in the future, or vice versa. The percentile ranges are listed in table 1 of Lenderink 2014's supplement. This should result is approximately 50 remaining samples for each scenario, for both control and future. 3. Use a Monte-Carlo method to make a final selection of 8 resamples with minimal reuse of the same ensemble member/segment. -Datasets have been split in two parts: the CMIP datasets and the target model datasets. An example use case for this recipe is to compare between CMIP5 and CMIP6, for example. The recipe can work with a target model that is not part of CMIP, provided that the data are CMOR compatible, and using the same data referece syntax as the CMIP data. Note that you can specify :ref:`multiple data paths` in the user configuration file. +Datasets have been split in two parts: the CMIP datasets and the target model datasets. An example use case for this recipe is to compare between CMIP5 and CMIP6, for example. The recipe can work with a target model that is not part of CMIP, provided that the data are CMOR compatible, and using the same data reference syntax as the CMIP data. Note that you can specify :ref:`multiple data paths` in the configuration. Available recipes and diagnostics @@ -128,7 +128,7 @@ AND highlighting the selected steering parameters and resampling periods: .. figure:: /recipes/figures/kcs/global_matching.png :align: center -The diagnostic ``local_resampling`` procudes a number of output files: +The diagnostic ``local_resampling`` produces a number of output files: * ``season_means_.nc``: intermediate results, containing the season means for each segment of the original target model ensemble. * ``top1000_.csv``: intermediate results, containing the 1000 combinations that have been selected based on winter mean precipitation. diff --git a/doc/sphinx/source/recipes/recipe_model_evaluation.rst b/doc/sphinx/source/recipes/recipe_model_evaluation.rst index 9e199815e0..c61f34aa62 100644 --- a/doc/sphinx/source/recipes/recipe_model_evaluation.rst +++ b/doc/sphinx/source/recipes/recipe_model_evaluation.rst @@ -35,9 +35,9 @@ User settings ------------- It is recommended to use a vector graphic file type (e.g., SVG) for the output -format when running this recipe, i.e., run the recipe with the command line -option ``--output_file_type=svg`` or use ``output_file_type: svg`` in your -:ref:`esmvalcore:user configuration file`. +format when running this recipe, i.e., run the recipe with the +:ref:`configuration options ` ``output_file_type: +svg``. Note that map and profile plots are rasterized by default. Use ``rasterize: false`` in the recipe to disable this. diff --git a/doc/sphinx/source/recipes/recipe_monitor.rst b/doc/sphinx/source/recipes/recipe_monitor.rst index ee3b9b44fa..8f4893fc12 100644 --- a/doc/sphinx/source/recipes/recipe_monitor.rst +++ b/doc/sphinx/source/recipes/recipe_monitor.rst @@ -36,9 +36,9 @@ User settings ------------- It is recommended to use a vector graphic file type (e.g., SVG) for the output -files when running this recipe, i.e., run the recipe with the command line -option ``--output_file_type=svg`` or use ``output_file_type: svg`` in your -:ref:`esmvalcore:user configuration file`. +format when running this recipe, i.e., run the recipe with the +:ref:`configuration options ` ``output_file_type: +svg``. Note that map and profile plots are rasterized by default. Use ``rasterize_maps: false`` or ``rasterize: false`` (see `Recipe settings`_) in the recipe to disable this. diff --git a/doc/sphinx/source/recipes/recipe_oceans.rst b/doc/sphinx/source/recipes/recipe_oceans.rst index d8bf3143e1..2c96425794 100644 --- a/doc/sphinx/source/recipes/recipe_oceans.rst +++ b/doc/sphinx/source/recipes/recipe_oceans.rst @@ -458,7 +458,7 @@ and a latitude and longitude coordinates. This diagnostic also includes the optional arguments, `maps_range` and `diff_range` to manually define plot ranges. Both arguments are a list of two floats -to set plot range minimun and maximum values respectively for Model and Observations +to set plot range minimum and maximum values respectively for Model and Observations maps (Top panels) and for the Model minus Observations panel (bottom left). Note that if input data have negative values the Model over Observations map (bottom right) is not produced. @@ -491,14 +491,14 @@ diagnostic_maps_multimodel.py The diagnostic_maps_multimodel.py_ diagnostic makes model(s) vs observations maps and if data are not provided it draws only model field. -It is always nessary to define the overall layout trough the argument `layout_rowcol`, +It is always necessary to define the overall layout through the argument `layout_rowcol`, which is a list of two integers indicating respectively the number of rows and columns to organize the plot. Observations has not be accounted in here as they are automatically added at the top of the figure. This diagnostic also includes the optional arguments, `maps_range` and `diff_range` to manually define plot ranges. Both arguments are a list of two floats -to set plot range minimun and maximum values respectively for variable data and +to set plot range minimum and maximum values respectively for variable data and the Model minus Observations range. Note that this diagnostic assumes that the preprocessors do the bulk of the @@ -748,7 +748,7 @@ These tools are: - bgc_units: converts to sensible units where appropriate (ie Celsius, mmol/m3) - timecoord_to_float: Converts time series to decimal time ie: Midnight on January 1st 1970 is 1970.0 - add_legend_outside_right: a plotting tool, which adds a legend outside the axes. -- get_image_format: loads the image format, as defined in the global user config.yml. +- get_image_format: loads the image format, as defined in the global configuration. - get_image_path: creates a path for an image output. - make_cube_layer_dict: makes a dictionary for several layers of a cube. @@ -762,8 +762,8 @@ A note on the auxiliary data directory Some of these diagnostic scripts may not function on machines with no access to the internet, as cartopy may try to download the shape files. The solution to this issue is the put the relevant cartopy shapefiles in a directory which -is visible to esmvaltool, then link that path to ESMValTool via -the `auxiliary_data_dir` variable in your config-user.yml file. +is visible to esmvaltool, then link that path to ESMValTool via the +:ref:`configuration option ` `auxiliary_data_dir`. The cartopy masking files can be downloaded from: https://www.naturalearthdata.com/downloads/ diff --git a/doc/sphinx/source/recipes/recipe_shapeselect.rst b/doc/sphinx/source/recipes/recipe_shapeselect.rst index 63afbcae6c..b443059ddc 100644 --- a/doc/sphinx/source/recipes/recipe_shapeselect.rst +++ b/doc/sphinx/source/recipes/recipe_shapeselect.rst @@ -29,7 +29,7 @@ User settings in recipe *Required settings (scripts)* - * shapefile: path to the user provided shapefile. A relative path is relative to the auxiliary_data_dir as configured in config-user.yml. + * shapefile: path to the user provided shapefile. A relative path is relative to the :ref:`configuration option ` `auxiliary_data_dir``. * weighting_method: the preferred weighting method 'mean_inside' - mean of all grid points inside polygon; 'representative' - one point inside or close to the polygon is used to represent the complete area. diff --git a/doc/sphinx/source/recipes/recipe_wenzel14jgr.rst b/doc/sphinx/source/recipes/recipe_wenzel14jgr.rst index 3c7fa86a3a..4faa05c2a9 100644 --- a/doc/sphinx/source/recipes/recipe_wenzel14jgr.rst +++ b/doc/sphinx/source/recipes/recipe_wenzel14jgr.rst @@ -28,8 +28,8 @@ User settings .. note:: - Make sure to run this recipe setting ``max_parallel_tasks: 1`` in the ``config_user.yml`` - file or using the CLI flag ``--max_parallel_tasks=1``. + Make sure to run this recipe with the :ref:`configuration option + ` ``max_parallel_tasks: 1``. User setting files (cfg files) are stored in nml/cfg_carbon/ diff --git a/doc/sphinx/source/recipes/recipe_wenzel16nat.rst b/doc/sphinx/source/recipes/recipe_wenzel16nat.rst index 03bb822545..a661844e70 100644 --- a/doc/sphinx/source/recipes/recipe_wenzel16nat.rst +++ b/doc/sphinx/source/recipes/recipe_wenzel16nat.rst @@ -35,9 +35,8 @@ User settings .. note:: - Make sure to run this recipe setting ``output_file_type: pdf`` in the ``config_user.yml`` - file or using the CLI flag ``--output_file_type=pdf``. - + Make sure to run this recipe with the :ref:`configuration option + ` ``max_parallel_tasks: 1``. #. Script carbon_beta.ncl @@ -58,7 +57,7 @@ User settings none -#. Script carbon_co2_cycle.ncl +#. Script carbon_co2_cycle.ncl *Required Settings (scripts)* @@ -72,7 +71,7 @@ User settings *Required settings (variables)* - * reference_dataset: name of reference datatset (observations) + * reference_dataset: name of reference dataset (observations) *Optional settings (variables)* @@ -102,15 +101,15 @@ Example plots ------------- .. figure:: /recipes/figures/wenzel16nat/fig_1.png - :width: 12 cm + :width: 12 cm :align: center - + Comparison of CO\ :sub:`2` seasonal amplitudes for CMIP5 historical simulations and observations showing annual mean atmospheric CO\ :sub:`2` versus the amplitudes of the CO\ :sub:`2` seasonal cycle at Pt. Barrow, Alaska (produced with carbon_co2_cycle.ncl, similar to Fig. 1a from Wenzel et al. (2016)). - + .. figure:: /recipes/figures/wenzel16nat/fig_2.png - :width: 12 cm + :width: 12 cm :align: center - + Barchart showing the gradient of the linear correlations for the comparison of CO\ :sub:`2` seasonal amplitudes for CMIP5 historical for at Pt. Barrow, Alaska (produced with carbon_co2_cycle.ncl, similar to Fig. 1b from Wenzel et al. (2016)). .. figure:: /recipes/figures/wenzel16nat/fig_3.png diff --git a/doc/sphinx/source/utils.rst b/doc/sphinx/source/utils.rst index 71de0e01f6..573f7420ef 100644 --- a/doc/sphinx/source/utils.rst +++ b/doc/sphinx/source/utils.rst @@ -135,10 +135,11 @@ This suite is configured to work with versions of cylc older than 8.0.0 . To prepare for using this tool: #. Log in to a system that uses `slurm `_ -#. Make sure the required CMIP and observational datasets are available and configured in config-user.yml +#. Make sure the required CMIP and observational datasets are available and + their ``rootpath`` and ``drs`` is properly set up in the :ref:`configuration + ` #. Make sure the required auxiliary data is available (see :ref:`recipe documentation `) #. Install ESMValTool -#. Update config-user.yml so it points to the right data locations Next, get started with `cylc `_: @@ -181,7 +182,7 @@ The following parameters have to be set in the script in order to make it run: Optionally, the following parameters can be edited: -* ``config_file``, *str*: Path to ``config-user.yml`` if default ``~/.esmvaltool/config-user.yml`` not used. +* ``config_dir``, *str*: Path to :ref:`configuration directory `, by default ``~/.config/esmvaltool/``. * ``partition``, *str*: Name of the DKRZ partition used to run jobs. Default is ``interactive`` to minimize computing cost compared to ``compute`` for which nodes cannot be shared. * ``memory``, *str*: Amount of memory requested for each run. Default is ``64G`` to allow to run 4 recipes on the same node in parallel. * ``time``, *str*: Time limit. Default is ``04:00:00`` to increase the job priority. Jobs can run for up to 8 hours and 12 hours on the compute and interactive partitions, respectively. @@ -230,7 +231,7 @@ script as well as a list of all available recipes. To generate the list, run the for recipe in $(esmvaltool recipes list | grep '\.yml$'); do echo $(basename "$recipe"); done > all_recipes.txt -To keep the script execution fast, it is recommended to use ``log_level: info`` in your config-user.yml file so that SLURM +To keep the script execution fast, it is recommended to use ``log_level: info`` in the configuration so that SLURM output files are rather small. .. _overview_page: From 42403fa4f84271b411593ab510cceec741e9d53a Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 2 Oct 2024 18:31:03 +0200 Subject: [PATCH 04/29] Adapt recipe filler --- esmvaltool/utils/recipe_filler.py | 47 ++++++++++++------------- tests/integration/test_recipe_filler.py | 23 ++++++------ 2 files changed, 34 insertions(+), 36 deletions(-) diff --git a/esmvaltool/utils/recipe_filler.py b/esmvaltool/utils/recipe_filler.py index 40f637c6d5..14350ae08d 100755 --- a/esmvaltool/utils/recipe_filler.py +++ b/esmvaltool/utils/recipe_filler.py @@ -30,7 +30,8 @@ - `start_year` and `end_year` are mandatory and are used to filter out the datasets that don't have data in the interval; if you want all possible years hence no filtering on years just use "*" for start and end years; -- `config-user: rootpath: CMIPX` may be a list, rootpath lists are supported; +- `rootpath: CMIPX` in the configuration may be a list, rootpath lists are + supported; Caveats: @@ -52,7 +53,6 @@ import esmvalcore import yaml - from esmvalcore import __version__ as core_ver from esmvalcore.cmor.table import CMOR_TABLES, read_cmor_tables from packaging import version as pkg_version @@ -183,13 +183,13 @@ def _normalize_path(path): return os.path.abspath(os.path.expanduser(os.path.expandvars(path))) -def read_config_user_file(config_file, folder_name, options=None): - """Read config user file and store settings in a dictionary.""" +def read_config_file(config_file, folder_name, options=None): + """Read configuration file and store settings in a dictionary.""" if not config_file: - config_file = '~/.esmvaltool/config-user.yml' + config_file = '~/.config/esmvaltool/config-user.yml' config_file = os.path.abspath( os.path.expandvars(os.path.expanduser(config_file))) - # Read user config file + # Read config file if not os.path.exists(config_file): print(f"ERROR: Config file {config_file} does not exist") @@ -292,14 +292,14 @@ def read_config_user_file(config_file, folder_name, options=None): def _get_download_dir(yamlconf, cmip_era): - """Get the Download Directory from user config file.""" + """Get the Download Directory from configuration file.""" if 'download_dir' in yamlconf: return os.path.join(yamlconf['download_dir'], cmip_era) return False def _get_site_rootpath(cmip_era): - """Get site (drs) from config-user.yml.""" + """Get site (drs) from configuration.""" config_yml = get_args().config_file with open(config_yml, 'r') as yamf: yamlconf = yaml.safe_load(yamf) @@ -541,7 +541,7 @@ def list_all_files(file_dict, cmip_era): if not new_path.startswith(os.sep): raise ValueError( "Could not expand ~ to user home dir " - "please expand it in the config user file!") + "please expand it in the configuration file!") logger.info("Expanding path to %s", new_path) # Globs all the wildcards into a list of files. @@ -656,20 +656,20 @@ def _check_recipe(recipe_dict): raise ValueError("Please fix the issues in recipe and rerun") -def _check_config_file(user_config_file): +def _check_config_file(config_file): """Perform a quick recipe check for mandatory fields.""" do_exit = False - if "rootpath" not in user_config_file: + if "rootpath" not in config_file: logger.error("Config file missing rootpath section.") do_exit = True - if "drs" not in user_config_file: + if "drs" not in config_file: logger.error("Config file missing drs section.") do_exit = True for proj in cmip_eras: - if proj not in user_config_file["rootpath"].keys(): + if proj not in config_file["rootpath"].keys(): logger.error("Config file missing rootpath for %s", proj) do_exit = True - if proj not in user_config_file["drs"].keys(): + if proj not in config_file["drs"].keys(): logger.error("Config file missing drs for %s", proj) do_exit = True if do_exit: @@ -790,8 +790,9 @@ def get_args(): parser.add_argument('recipe', help='Path/name of yaml pilot recipe file') parser.add_argument('-c', '--config-file', - default=os.path.join(os.environ["HOME"], '.esmvaltool', - 'config-user.yml'), + default=os.path.expanduser( + '~/.config/esmvaltool/config-user.yml' + ), help='User configuration file') parser.add_argument('-o', @@ -835,25 +836,23 @@ def run(): cmip_eras = ["CMIP5", "CMIP6"] # read the config file - config_user = read_config_user_file(args.config_file, - 'recipe_filler', - options={}) + cfg = read_config_file(args.config_file, 'recipe_filler', options={}) # configure logger - run_dir = os.path.join(config_user['output_dir'], 'recipe_filler') + run_dir = os.path.join(cfg['output_dir'], 'recipe_filler') if not os.path.isdir(run_dir): os.makedirs(run_dir) log_files = configure_logging(output_dir=run_dir, - console_log_level=config_user['log_level']) + console_log_level=cfg['log_level']) logger.info(HEADER) - logger.info("Using user configuration file: %s", args.config_file) + logger.info("Using configuration file: %s", args.config_file) logger.info("Using pilot recipe file: %s", input_recipe) logger.info("Writing filled out recipe to: %s", output_recipe) log_files = "\n".join(log_files) logger.info("Writing program log files to:\n%s", log_files) - # check config user file - _check_config_file(config_user) + # check configuration file + _check_config_file(cfg) # parse recipe with open(input_recipe, 'r') as yamlfile: diff --git a/tests/integration/test_recipe_filler.py b/tests/integration/test_recipe_filler.py index b78ac8c5f8..8dfa900416 100644 --- a/tests/integration/test_recipe_filler.py +++ b/tests/integration/test_recipe_filler.py @@ -10,7 +10,6 @@ from esmvaltool.utils.recipe_filler import run - # Load test configuration with open(os.path.join(os.path.dirname(__file__), 'recipe_filler.yml')) as file: @@ -65,7 +64,7 @@ def create_tree(path, filenames=None, symlinks=None): os.symlink(symlink['target'], link_name) -def write_config_user_file(dirname, file_path, drs): +def write_config_file(dirname, file_path, drs): config_file = dirname / 'config-user.yml' cfg = { 'log_level': 'info', @@ -102,7 +101,7 @@ def root(): def setup_files(tmp_path, root, cfg): """Create config, recipe ,output recipe etc.""" - user_config_file = write_config_user_file(tmp_path, root, cfg['drs']) + config_file = write_config_file(tmp_path, root, cfg['drs']) diagnostics = {} diagnostics["test_diagnostic"] = {} diagnostics["test_diagnostic"]["variables"] = {} @@ -110,7 +109,7 @@ def setup_files(tmp_path, root, cfg): recipe = write_recipe(tmp_path, diagnostics) output_recipe = str(tmp_path / "recipe_auto.yml") - return user_config_file, recipe, output_recipe + return config_file, recipe, output_recipe @pytest.mark.parametrize('cfg', CONFIG['has_additional_datasets']) @@ -119,13 +118,13 @@ def test_adding_datasets(tmp_path, root, cfg): create_tree(root, cfg.get('available_files'), cfg.get('available_symlinks')) - user_config_file, recipe, output_recipe = setup_files(tmp_path, root, cfg) + config_file, recipe, output_recipe = setup_files(tmp_path, root, cfg) with arguments( 'recipe_filler', recipe, '-c', - user_config_file, + config_file, '-o', output_recipe, ): @@ -144,13 +143,13 @@ def test_not_adding_datasets(tmp_path, root, cfg): create_tree(root, cfg.get('available_files'), cfg.get('available_symlinks')) - user_config_file, recipe, output_recipe = setup_files(tmp_path, root, cfg) + config_file, recipe, output_recipe = setup_files(tmp_path, root, cfg) with arguments( 'recipe_filler', recipe, '-c', - user_config_file, + config_file, '-o', output_recipe, ): @@ -166,7 +165,7 @@ def test_not_adding_datasets(tmp_path, root, cfg): def test_bad_var(tmp_path, root): """Test a bad variable in the works.""" cfg = CONFIG['bad_variable'][0] - user_config_file, recipe, output_recipe = setup_files(tmp_path, root, cfg) + config_file, recipe, output_recipe = setup_files(tmp_path, root, cfg) # this doesn't fail and it shouldn't since it can go on # and look for data for other valid variables @@ -174,7 +173,7 @@ def test_bad_var(tmp_path, root): 'recipe_filler', recipe, '-c', - user_config_file, + config_file, '-o', output_recipe, ): @@ -190,7 +189,7 @@ def test_bad_var(tmp_path, root): def test_no_short_name(tmp_path, root): """Test a bad variable in the works.""" cfg = CONFIG['no_short_name'][0] - user_config_file, recipe, output_recipe = setup_files(tmp_path, root, cfg) + config_file, recipe, output_recipe = setup_files(tmp_path, root, cfg) # this doesn't fail and it shouldn't since it can go on # and look for data for other valid variables @@ -198,7 +197,7 @@ def test_no_short_name(tmp_path, root): 'recipe_filler', recipe, '-c', - user_config_file, + config_file, '-o', output_recipe, ): From 6328ade8d0144f917374529bcab8a592305fc129 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 2 Oct 2024 18:31:25 +0200 Subject: [PATCH 05/29] Adapt CMORizers --- esmvaltool/cmorizers/data/cmorizer.py | 108 +++++++++++++++--- esmvaltool/cmorizers/data/datasets.yml | 30 ++--- .../download_scripts/download_era_interim.py | 9 +- esmvaltool/cmorizers/data/downloaders/cds.py | 2 +- .../data/downloaders/datasets/aeronet.py | 2 +- .../data/downloaders/datasets/aphro_ma.py | 2 +- .../data/downloaders/datasets/aura_tes.py | 2 +- .../downloaders/datasets/berkeleyearth.py | 2 +- .../downloaders/datasets/calipso_goccp.py | 2 +- .../downloaders/datasets/calipso_icecloud.py | 2 +- .../datasets/cds_satellite_albedo.py | 2 +- .../datasets/cds_satellite_lai_fapar.py | 2 +- .../datasets/cds_satellite_soil_moisture.py | 4 +- .../data/downloaders/datasets/cds_uerra.py | 2 +- .../data/downloaders/datasets/cds_xch4.py | 2 +- .../data/downloaders/datasets/cowtanway.py | 2 +- .../data/downloaders/datasets/cru.py | 2 +- .../data/downloaders/datasets/ct2019.py | 2 +- .../downloaders/datasets/duveiller2018.py | 2 +- .../data/downloaders/datasets/e_obs.py | 2 +- .../downloaders/datasets/eppley_vgpm_modis.py | 2 +- .../downloaders/datasets/esacci_aerosol.py | 2 +- .../data/downloaders/datasets/esacci_cloud.py | 2 +- .../data/downloaders/datasets/esacci_fire.py | 2 +- .../downloaders/datasets/esacci_landcover.py | 2 +- .../data/downloaders/datasets/esacci_oc.py | 2 +- .../data/downloaders/datasets/esacci_ozone.py | 2 +- .../datasets/esacci_sea_surface_salinity.py | 2 +- .../datasets/esacci_soilmoisture.py | 2 +- .../data/downloaders/datasets/ghcn.py | 2 +- .../data/downloaders/datasets/ghcn_cams.py | 2 +- .../data/downloaders/datasets/gistemp.py | 2 +- .../data/downloaders/datasets/glodap.py | 2 +- .../data/downloaders/datasets/gpcc.py | 2 +- .../data/downloaders/datasets/gpcp_sg.py | 2 +- .../data/downloaders/datasets/hadcrut3.py | 2 +- .../data/downloaders/datasets/hadcrut4.py | 2 +- .../data/downloaders/datasets/hadcrut5.py | 2 +- .../data/downloaders/datasets/hadisst.py | 2 +- .../data/downloaders/datasets/haloe.py | 2 +- .../data/downloaders/datasets/isccp_fh.py | 2 +- .../data/downloaders/datasets/jra_25.py | 2 +- .../data/downloaders/datasets/jra_55.py | 4 +- .../data/downloaders/datasets/kadow2020.py | 2 +- .../downloaders/datasets/landflux_eval.py | 2 +- .../downloaders/datasets/landschuetzer2016.py | 2 +- .../downloaders/datasets/landschuetzer2020.py | 2 +- .../data/downloaders/datasets/merra.py | 2 +- .../data/downloaders/datasets/merra2.py | 2 +- .../downloaders/datasets/mobo_dic2004_2019.py | 2 +- .../downloaders/datasets/mobo_dic_mpim.py | 2 +- .../data/downloaders/datasets/ncep_doe_r2.py | 2 +- .../data/downloaders/datasets/ncep_ncar_r1.py | 2 +- .../data/downloaders/datasets/ndp.py | 2 +- .../datasets/noaa_cires_20cr_v2.py | 2 +- .../datasets/noaa_cires_20cr_v3.py | 2 +- .../downloaders/datasets/noaa_ersstv3b.py | 3 +- .../data/downloaders/datasets/noaa_ersstv5.py | 3 +- .../data/downloaders/datasets/noaa_mbl_ch4.py | 2 +- .../downloaders/datasets/noaaglobaltemp.py | 2 +- .../downloaders/datasets/nsidc_0116_nh.py | 2 +- .../downloaders/datasets/nsidc_0116_sh.py | 2 +- .../downloaders/datasets/nsidc_g02202_sh.py | 3 +- .../downloaders/datasets/oceansoda_ethz.py | 2 +- .../data/downloaders/datasets/osi_450_nh.py | 2 +- .../data/downloaders/datasets/osi_450_sh.py | 2 +- .../data/downloaders/datasets/patmos_x.py | 2 +- .../data/downloaders/datasets/persiann_cdr.py | 2 +- .../data/downloaders/datasets/phc.py | 2 +- .../data/downloaders/datasets/regen.py | 2 +- .../downloaders/datasets/scripps_co2_kum.py | 2 +- .../data/downloaders/datasets/tcom_ch4.py | 2 +- .../data/downloaders/datasets/tcom_n2o.py | 2 +- .../data/downloaders/datasets/woa.py | 2 +- .../cmorizers/data/downloaders/downloader.py | 2 +- esmvaltool/cmorizers/data/downloaders/ftp.py | 4 +- .../data/formatters/datasets/ct2019.py | 2 +- .../data/formatters/datasets/merra.ncl | 7 +- .../data/formatters/datasets/mls_aura.py | 2 +- tests/integration/test_cmorizer.py | 50 ++++++-- 80 files changed, 239 insertions(+), 124 deletions(-) diff --git a/esmvaltool/cmorizers/data/cmorizer.py b/esmvaltool/cmorizers/data/cmorizer.py index 16b7666350..ef4fd31cbb 100755 --- a/esmvaltool/cmorizers/data/cmorizer.py +++ b/esmvaltool/cmorizers/data/cmorizer.py @@ -10,6 +10,7 @@ import os import shutil import subprocess +import warnings from pathlib import Path import esmvalcore @@ -18,13 +19,14 @@ from esmvalcore.config import CFG from esmvalcore.config._logging import configure_logging +from esmvaltool import ESMValToolDeprecationWarning from esmvaltool.cmorizers.data.utilities import read_cmor_config logger = logging.getLogger(__name__) datasets_file = os.path.join(os.path.dirname(__file__), 'datasets.yml') -class Formatter(): +class _Formatter(): """ Class to manage the download and formatting of datasets. @@ -39,26 +41,38 @@ def __init__(self, info): self.datasets_info = info self.config = '' - def start(self, command, datasets, config_file, options): + def start(self, command, datasets, config_file, config_dir, options): """Read configuration and set up formatter for data processing. Parameters ---------- command: str - Name of the command to execute + Name of the command to execute. datasets: str - List of datasets to process, comma separated + List of datasets to process, comma separated. config_file: str - Config file to use + Config file to use. Option will be removed in v2.14.0. + config_dir: str + Config directory to use. options: dict() - Extra options to overwrite config user file + Extra options to overwrite configuration. + """ if isinstance(datasets, str): self.datasets = datasets.split(',') else: self.datasets = datasets - CFG.load_from_file(config_file) + if config_file is not None: # remove in v2.14.0 + CFG.load_from_file(config_file) + elif config_dir is not None: + config_dir = Path(config_dir).expanduser().absolute() + if not config_dir.is_dir(): + raise NotADirectoryError( + f"Invalid --config_dir given: {config_dir} is not an " + f"existing directory" + ) + CFG.update_from_dirs([config_dir]) CFG.update(options) self.config = CFG.start_session(f'data_{command}') @@ -400,7 +414,7 @@ class DataCommand(): def __init__(self): with open(datasets_file, 'r', encoding='utf8') as data: self._info = yaml.safe_load(data) - self.formatter = Formatter(self._info) + self.formatter = _Formatter(self._info) def _has_downloader(self, dataset): return 'Yes' if self.formatter.has_downloader(dataset) else "No" @@ -441,28 +455,48 @@ def download(self, start=None, end=None, overwrite=False, + config_dir=None, **kwargs): """Download datasets. Parameters ---------- - datasets : list(str) + datasets: list(str) List of datasets to format - config_file : str, optional - Path to ESMValTool's config user file, by default None - start : str, optional + config_file: str, optional + Path to ESMValTool's config user file, by default None. + + .. deprecated:: 2.12.0 + This option has been deprecated in ESMValTool version 2.12.0 + and is scheduled for removal in version 2.14.0. Please use the + option `config_dir` instead. + start: str, optional Start of the interval to process, by default None. Valid formats are YYYY, YYYYMM and YYYYMMDD. - end : str, optional + end: str, optional End of the interval to process, by default None. Valid formats are YYYY, YYYYMM and YYYYMMDD. - overwrite : bool, optional + overwrite: bool, optional If true, download already present data again + config_dir: str, optional + Path to additional ESMValTool configuration directory. See + :ref:`esmvalcore:config_yaml_files` for details. + """ + if config_file is not None: + msg = ( + "The option `config_file` has been deprecated in ESMValTool " + "version 2.12.0 and is scheduled for removal in version " + "2.14.0. Please use the option ``config_dir`` instead." + ) + warnings.warn(msg, ESMValToolDeprecationWarning) + start = self._parse_date(start) end = self._parse_date(end) - self.formatter.start('download', datasets, config_file, kwargs) + self.formatter.start( + 'download', datasets, config_file, config_dir, kwargs + ) self.formatter.download(start, end, overwrite) def format(self, @@ -471,6 +505,7 @@ def format(self, start=None, end=None, install=False, + config_dir=None, **kwargs): """Format datasets. @@ -480,6 +515,11 @@ def format(self, List of datasets to format config_file : str, optional Path to ESMValTool's config user file, by default None + + .. deprecated:: 2.12.0 + This option has been deprecated in ESMValTool version 2.12.0 + and is scheduled for removal in version 2.14.0. Please use the + option `config_dir` instead. start : str, optional Start of the interval to process, by default None. Valid formats are YYYY, YYYYMM and YYYYMMDD. @@ -488,11 +528,25 @@ def format(self, are YYYY, YYYYMM and YYYYMMDD. install : bool, optional If true, move processed data to the folder, by default False + config_dir: str, optional + Path to additional ESMValTool configuration directory. See + :ref:`esmvalcore:config_yaml_files` for details. + """ + if config_file is not None: + msg = ( + "The option `config_file` has been deprecated in ESMValTool " + "version 2.12.0 and is scheduled for removal in version " + "2.14.0. Please use the option ``config_dir`` instead." + ) + warnings.warn(msg, ESMValToolDeprecationWarning) + start = self._parse_date(start) end = self._parse_date(end) - self.formatter.start('formatting', datasets, config_file, kwargs) + self.formatter.start( + 'formatting', datasets, config_file, config_dir, kwargs + ) self.formatter.format(start, end, install) def prepare(self, @@ -502,6 +556,7 @@ def prepare(self, end=None, overwrite=False, install=False, + config_dir=None, **kwargs): """Download and format a set of datasets. @@ -511,6 +566,11 @@ def prepare(self, List of datasets to format config_file : str, optional Path to ESMValTool's config user file, by default None + + .. deprecated:: 2.12.0 + This option has been deprecated in ESMValTool version 2.12.0 + and is scheduled for removal in version 2.14.0. Please use the + option `config_dir` instead. start : str, optional Start of the interval to process, by default None. Valid formats are YYYY, YYYYMM and YYYYMMDD. @@ -521,11 +581,25 @@ def prepare(self, If true, move processed data to the folder, by default False overwrite : bool, optional If true, download already present data again + config_dir: str, optional + Path to additional ESMValTool configuration directory. See + :ref:`esmvalcore:config_yaml_files` for details. + """ + if config_file is not None: + msg = ( + "The option `config_file` has been deprecated in ESMValTool " + "version 2.12.0 and is scheduled for removal in version " + "2.14.0. Please use the option ``config_dir`` instead." + ) + warnings.warn(msg, ESMValToolDeprecationWarning) + start = self._parse_date(start) end = self._parse_date(end) - self.formatter.start('preparation', datasets, config_file, kwargs) + self.formatter.start( + 'preparation', datasets, config_file, config_dir, kwargs + ) if self.formatter.download(start, end, overwrite): self.formatter.format(start, end, install) else: diff --git a/esmvaltool/cmorizers/data/datasets.yml b/esmvaltool/cmorizers/data/datasets.yml index 8fcb6adc21..27240e4d61 100644 --- a/esmvaltool/cmorizers/data/datasets.yml +++ b/esmvaltool/cmorizers/data/datasets.yml @@ -17,16 +17,16 @@ datasets: analyses covering analysis of monthly rainfall. The dataset provides consistent temporal and spatial analyses across Australia for each observed data variable. This accounts for spatial and temporal gaps in observations. Where possible, the gridded analysis techniques provide useful estimates in data-sparse regions - such as central Australia. - + such as central Australia. + Time coverage: Site-based data are used to provide gridded climate data at the monthly timescale for rainfall (1900+). Reference: Evans, A., Jones, D.A., Smalley, R., and Lellyett, S. 2020. An enhanced gridded rainfall analysis scheme for Australia. Bureau of Meteorology Research Report. No. 41. National Computational Infrastructure (NCI) - Catalogue Record: http://dx.doi.org/10.25914/6009600786063. - Data from NCI (National Computing Infrastructure Australia https://nci.org.au/), + Data from NCI (National Computing Infrastructure Australia https://nci.org.au/), requires an NCI account and access to Gadi(Supercomputer in Canberra) and the project found in catalogue record. Access can be requested through NCI. NCI is an ESGF node (https://esgf.nci.org.au/projects/esgf-nci/) - + ANUClimate: tier: 3 source: "https://dx.doi.org/10.25914/60a10aa56dd1b" @@ -35,7 +35,7 @@ datasets: Data from NCI project requiring an NCI account and access to GADI ANUClimate 2.0 consists of gridded daily and monthly climate variables across the terrestrial landmass of Australia - from at least 1970 to the present. Rainfall grids are generated from 1900 to the present. The underpinning spatial + from at least 1970 to the present. Rainfall grids are generated from 1900 to the present. The underpinning spatial models have been developed at the Fenner School of Environment and Society of the Australian National University. APHRO-MA: @@ -301,7 +301,7 @@ datasets: last_access: 2020-03-23 info: | Create a new empty directory ``$RAWOBSPATH/Tier2/CT2019`` (where - ``$RAWOBSPATH`` is given by your user configuration file) where the raw + ``$RAWOBSPATH`` is given by your user configuration) where the raw data will be stored. The download of the data is automatically handled by this script. If data is already present in this directory, the download is skipped (to force a new download delete your old files). @@ -479,11 +479,11 @@ datasets: Download and processing instructions: Use the following CLI to download all the files: esmvaltool data download ESACCI-LANDCOVER - The underlying downloader is located here: + The underlying downloader is located here: /ESMValTool/esmvaltool/cmorizers/data/downloaders/datasets/esacci_landcover.py - and it will download all the files currently available on CEDA (1992-2020) + and it will download all the files currently available on CEDA (1992-2020) under a single directory as follow: ${RAWOBS}/Tier2/ESACCI-LANDCOVER - + ESACCI-LST: tier: 2 source: On CEDA-JASMIN, /gws/nopw/j04/esacci_lst/public @@ -554,7 +554,7 @@ datasets: source: https://wui.cmsaf.eu/safira/action/viewDoiDetails?acronym=COMBI_V001 last_access: 2024-02-21 info: | - CDR2 requires registration at EUMETSAT CM SAF, the information on how to + CDR2 requires registration at EUMETSAT CM SAF, the information on how to download the order will be emailed once the order is ready. All files need to be in one directory, not in yearly subdirectories. @@ -903,7 +903,7 @@ datasets: Select "Data Access" -> "Subset/Get Data" -> "Get Data" and follow the "Instructions for downloading". All *.he5 files need to be saved in the $RAWOBS/Tier3/MLS-AURA directory, where $RAWOBS refers to the RAWOBS - directory defined in the user configuration file. Apply this procedure to + directory defined in the user configuration. Apply this procedure to both links provided above. The temperature fields are necessary for quality control of the RHI data (see Data Quality Document for MLS-AURA for more information). @@ -1076,7 +1076,7 @@ datasets: last_access: 2023-12-04 info: | Download the following files: - ersst.yyyymm.nc + ersst.yyyymm.nc for years 1854 to 2020 NOAA-ERSSTv5: @@ -1085,7 +1085,7 @@ datasets: last_access: 2023-12-04 info: | Download the following files: - ersst.v5.yyyymm.nc + ersst.v5.yyyymm.nc for years 1854 onwards NOAAGlobalTemp: @@ -1112,13 +1112,13 @@ datasets: Download daily data from: https://nsidc.org/data/NSIDC-0116 Login required for download, and also requires citation only to use - + NSIDC-G02202-sh: tier: 3 source: https://polarwatch.noaa.gov/erddap/griddap/nsidcG02202v4shmday last_access: 2023-05-13 info: | - Download monthly data. + Download monthly data. Login required for download, and also requires citation only to use OceanSODA-ETHZ: diff --git a/esmvaltool/cmorizers/data/download_scripts/download_era_interim.py b/esmvaltool/cmorizers/data/download_scripts/download_era_interim.py index 72cf8d98af..374c750ef6 100644 --- a/esmvaltool/cmorizers/data/download_scripts/download_era_interim.py +++ b/esmvaltool/cmorizers/data/download_scripts/download_era_interim.py @@ -12,8 +12,13 @@ 4. Copy/paste the text in https://api.ecmwf.int/v1/key/ into a blank text file and save it as $HOME/.ecmwfapirc -5. Use ESMValCore/esmvalcore/config-user.yml as an template -and set the rootpath of the output directory in RAWOBS +5. Copy the default configuration file with + +```bash +esmvaltool config get_config_user --path=config-user.yml +``` + +and set the ``rootpath`` for the RAWOBS project. 6. Check the description of the variables at https://apps.ecmwf.int/codes/grib/param-db diff --git a/esmvaltool/cmorizers/data/downloaders/cds.py b/esmvaltool/cmorizers/data/downloaders/cds.py index af1ad62501..e717bc4c43 100644 --- a/esmvaltool/cmorizers/data/downloaders/cds.py +++ b/esmvaltool/cmorizers/data/downloaders/cds.py @@ -19,7 +19,7 @@ class CDSDownloader(BaseDownloader): product_name : str Name of the product in the CDS config : dict - ESMValTool's user configuration + ESMValTool's configuration request_dictionary : dict Common CDS request parameters dataset : str diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/aeronet.py b/esmvaltool/cmorizers/data/downloaders/datasets/aeronet.py index 668a688bb6..fb4cba80f2 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/aeronet.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/aeronet.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/aphro_ma.py b/esmvaltool/cmorizers/data/downloaders/datasets/aphro_ma.py index 425bd93e88..77662358c2 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/aphro_ma.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/aphro_ma.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/aura_tes.py b/esmvaltool/cmorizers/data/downloaders/datasets/aura_tes.py index 9f7e195397..85b1c5e2e8 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/aura_tes.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/aura_tes.py @@ -10,7 +10,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/berkeleyearth.py b/esmvaltool/cmorizers/data/downloaders/datasets/berkeleyearth.py index 01ce0e6fa3..a29a9802f3 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/berkeleyearth.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/berkeleyearth.py @@ -10,7 +10,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/calipso_goccp.py b/esmvaltool/cmorizers/data/downloaders/datasets/calipso_goccp.py index 1c93d85b61..4040e65565 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/calipso_goccp.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/calipso_goccp.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/calipso_icecloud.py b/esmvaltool/cmorizers/data/downloaders/datasets/calipso_icecloud.py index c8992e3d38..04a6d3c3a6 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/calipso_icecloud.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/calipso_icecloud.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_albedo.py b/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_albedo.py index a789f9b227..fc8ae46bad 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_albedo.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_albedo.py @@ -15,7 +15,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_lai_fapar.py b/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_lai_fapar.py index 65c2bafa0f..80c8039a51 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_lai_fapar.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_lai_fapar.py @@ -15,7 +15,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_soil_moisture.py b/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_soil_moisture.py index 8b017321e1..99b92ecb23 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_soil_moisture.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_soil_moisture.py @@ -16,7 +16,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict @@ -89,7 +89,7 @@ def get_downloader(config, dataset, dataset_info, overwrite, sensor, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/cds_uerra.py b/esmvaltool/cmorizers/data/downloaders/datasets/cds_uerra.py index 7641aea6ad..644a0d5804 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/cds_uerra.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/cds_uerra.py @@ -15,7 +15,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/cds_xch4.py b/esmvaltool/cmorizers/data/downloaders/datasets/cds_xch4.py index 241db20f2d..c7455ca4f5 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/cds_xch4.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/cds_xch4.py @@ -11,7 +11,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/cowtanway.py b/esmvaltool/cmorizers/data/downloaders/datasets/cowtanway.py index edffbeb18c..03417fcd45 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/cowtanway.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/cowtanway.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/cru.py b/esmvaltool/cmorizers/data/downloaders/datasets/cru.py index 8fbce3e9a3..010dc3f5ce 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/cru.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/cru.py @@ -15,7 +15,7 @@ def download_dataset( Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/ct2019.py b/esmvaltool/cmorizers/data/downloaders/datasets/ct2019.py index 4eed312963..7c838962b2 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/ct2019.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/ct2019.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/duveiller2018.py b/esmvaltool/cmorizers/data/downloaders/datasets/duveiller2018.py index 8cf91a15ee..bb12d1c6cb 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/duveiller2018.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/duveiller2018.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/e_obs.py b/esmvaltool/cmorizers/data/downloaders/datasets/e_obs.py index fd12127e43..2b6d395dbc 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/e_obs.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/e_obs.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/eppley_vgpm_modis.py b/esmvaltool/cmorizers/data/downloaders/datasets/eppley_vgpm_modis.py index f70ee8ab35..e6ab8280d4 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/eppley_vgpm_modis.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/eppley_vgpm_modis.py @@ -17,7 +17,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_aerosol.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_aerosol.py index 09d2616c9c..19427bc82c 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_aerosol.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_aerosol.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_cloud.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_cloud.py index a14255e143..2f14a2bc92 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_cloud.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_cloud.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_fire.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_fire.py index 82fbc743ad..ea17b99cad 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_fire.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_fire.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_landcover.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_landcover.py index efffa2aaaa..427ea1fbb0 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_landcover.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_landcover.py @@ -12,7 +12,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_oc.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_oc.py index 64d4209477..6347977d3f 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_oc.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_oc.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_ozone.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_ozone.py index 50d80e06f3..dd2e2c9197 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_ozone.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_ozone.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sea_surface_salinity.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sea_surface_salinity.py index dc38903624..c8ea036476 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sea_surface_salinity.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sea_surface_salinity.py @@ -17,7 +17,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_soilmoisture.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_soilmoisture.py index 0d29e96ff9..d06bc1310b 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_soilmoisture.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_soilmoisture.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/ghcn.py b/esmvaltool/cmorizers/data/downloaders/datasets/ghcn.py index ceeb44de30..872523c8df 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/ghcn.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/ghcn.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/ghcn_cams.py b/esmvaltool/cmorizers/data/downloaders/datasets/ghcn_cams.py index 40b3b1a8a8..96ef17e023 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/ghcn_cams.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/ghcn_cams.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/gistemp.py b/esmvaltool/cmorizers/data/downloaders/datasets/gistemp.py index 2bf73fa4ed..2b2d8cf031 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/gistemp.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/gistemp.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/glodap.py b/esmvaltool/cmorizers/data/downloaders/datasets/glodap.py index f4de91f47f..534499c7f2 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/glodap.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/glodap.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/gpcc.py b/esmvaltool/cmorizers/data/downloaders/datasets/gpcc.py index 6279970be0..f38023a3c5 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/gpcc.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/gpcc.py @@ -17,7 +17,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/gpcp_sg.py b/esmvaltool/cmorizers/data/downloaders/datasets/gpcp_sg.py index 91e1b4e47b..2b364a1085 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/gpcp_sg.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/gpcp_sg.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut3.py b/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut3.py index 6481537442..a4172a601a 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut3.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut3.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut4.py b/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut4.py index 986e0bd2e7..a0f36275d9 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut4.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut4.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut5.py b/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut5.py index 6c8eda7c57..5c6d9f35cc 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut5.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut5.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/hadisst.py b/esmvaltool/cmorizers/data/downloaders/datasets/hadisst.py index 1ac279f284..cabe34c2a9 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/hadisst.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/hadisst.py @@ -15,7 +15,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/haloe.py b/esmvaltool/cmorizers/data/downloaders/datasets/haloe.py index fd396119fb..e8bdd43fca 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/haloe.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/haloe.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/isccp_fh.py b/esmvaltool/cmorizers/data/downloaders/datasets/isccp_fh.py index 9f2c15b794..b23880097d 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/isccp_fh.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/isccp_fh.py @@ -15,7 +15,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/jra_25.py b/esmvaltool/cmorizers/data/downloaders/datasets/jra_25.py index 27e3dbc97c..1f37a08d40 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/jra_25.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/jra_25.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/jra_55.py b/esmvaltool/cmorizers/data/downloaders/datasets/jra_55.py index a5dc5b851c..c68b5e0fce 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/jra_55.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/jra_55.py @@ -1,14 +1,12 @@ """Script to download JRA-55 from RDA.""" import logging import os - from datetime import datetime from dateutil import relativedelta from esmvaltool.cmorizers.data.downloaders.wget import WGetDownloader - logger = logging.getLogger(__name__) @@ -19,7 +17,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/kadow2020.py b/esmvaltool/cmorizers/data/downloaders/datasets/kadow2020.py index 2c08dd3c04..f05d40ee07 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/kadow2020.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/kadow2020.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/landflux_eval.py b/esmvaltool/cmorizers/data/downloaders/datasets/landflux_eval.py index 1d8e84cb9d..b8bd3784ad 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/landflux_eval.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/landflux_eval.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/landschuetzer2016.py b/esmvaltool/cmorizers/data/downloaders/datasets/landschuetzer2016.py index e08bae771e..0d9eb75b94 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/landschuetzer2016.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/landschuetzer2016.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/landschuetzer2020.py b/esmvaltool/cmorizers/data/downloaders/datasets/landschuetzer2020.py index 88929cfc7a..cce6a6319c 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/landschuetzer2020.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/landschuetzer2020.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/merra.py b/esmvaltool/cmorizers/data/downloaders/datasets/merra.py index df1d0ff7e9..d18371b127 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/merra.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/merra.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/merra2.py b/esmvaltool/cmorizers/data/downloaders/datasets/merra2.py index 28ddebf720..762cbf09ed 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/merra2.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/merra2.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/mobo_dic2004_2019.py b/esmvaltool/cmorizers/data/downloaders/datasets/mobo_dic2004_2019.py index 1299981811..711bffb8dd 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/mobo_dic2004_2019.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/mobo_dic2004_2019.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/mobo_dic_mpim.py b/esmvaltool/cmorizers/data/downloaders/datasets/mobo_dic_mpim.py index 58af5689d7..3e16c76de5 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/mobo_dic_mpim.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/mobo_dic_mpim.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/ncep_doe_r2.py b/esmvaltool/cmorizers/data/downloaders/datasets/ncep_doe_r2.py index 704493554f..6c7f2b8a37 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/ncep_doe_r2.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/ncep_doe_r2.py @@ -15,7 +15,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/ncep_ncar_r1.py b/esmvaltool/cmorizers/data/downloaders/datasets/ncep_ncar_r1.py index f9b0816efe..b8f51d779f 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/ncep_ncar_r1.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/ncep_ncar_r1.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/ndp.py b/esmvaltool/cmorizers/data/downloaders/datasets/ndp.py index 391e973d28..2a21f9d8c9 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/ndp.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/ndp.py @@ -15,7 +15,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_cires_20cr_v2.py b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_cires_20cr_v2.py index fb2d733f06..5d02e93d7b 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_cires_20cr_v2.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_cires_20cr_v2.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_cires_20cr_v3.py b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_cires_20cr_v3.py index 67f1a38f33..190a48f895 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_cires_20cr_v3.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_cires_20cr_v3.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv3b.py b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv3b.py index 0ac6a3e012..167c28be54 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv3b.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv3b.py @@ -1,6 +1,7 @@ """Script to download NOAA-ERSST-v3b.""" import logging from datetime import datetime + from dateutil import relativedelta from esmvaltool.cmorizers.data.downloaders.wget import WGetDownloader @@ -15,7 +16,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv5.py b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv5.py index f995f9d2c7..a6cee113a3 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv5.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv5.py @@ -1,6 +1,7 @@ """Script to download NOAA-ERSST-V5.""" import logging from datetime import datetime + from dateutil import relativedelta from esmvaltool.cmorizers.data.downloaders.wget import WGetDownloader @@ -15,7 +16,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_mbl_ch4.py b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_mbl_ch4.py index 3cbf701c97..11c90919ab 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_mbl_ch4.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_mbl_ch4.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/noaaglobaltemp.py b/esmvaltool/cmorizers/data/downloaders/datasets/noaaglobaltemp.py index cf946df2cc..c63e104278 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/noaaglobaltemp.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/noaaglobaltemp.py @@ -15,7 +15,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_0116_nh.py b/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_0116_nh.py index c4bb4e3afe..f83ed1e221 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_0116_nh.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_0116_nh.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_0116_sh.py b/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_0116_sh.py index adc8acd5ee..b9aa93acfc 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_0116_sh.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_0116_sh.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_g02202_sh.py b/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_g02202_sh.py index 798decda96..b7c908119d 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_g02202_sh.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_g02202_sh.py @@ -1,6 +1,7 @@ """Script to download NSIDC-G02202-sh.""" import logging from datetime import datetime + from dateutil import relativedelta from esmvaltool.cmorizers.data.downloaders.wget import WGetDownloader @@ -15,7 +16,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/oceansoda_ethz.py b/esmvaltool/cmorizers/data/downloaders/datasets/oceansoda_ethz.py index 2e099814dd..f6fb85d505 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/oceansoda_ethz.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/oceansoda_ethz.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/osi_450_nh.py b/esmvaltool/cmorizers/data/downloaders/datasets/osi_450_nh.py index 826548503f..3c58e254c8 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/osi_450_nh.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/osi_450_nh.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/osi_450_sh.py b/esmvaltool/cmorizers/data/downloaders/datasets/osi_450_sh.py index 5e3fd383f4..030f76982d 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/osi_450_sh.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/osi_450_sh.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/patmos_x.py b/esmvaltool/cmorizers/data/downloaders/datasets/patmos_x.py index 20b70e7fb9..f516a7d72d 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/patmos_x.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/patmos_x.py @@ -15,7 +15,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/persiann_cdr.py b/esmvaltool/cmorizers/data/downloaders/datasets/persiann_cdr.py index 5aa19ef257..27f8900e54 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/persiann_cdr.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/persiann_cdr.py @@ -15,7 +15,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/phc.py b/esmvaltool/cmorizers/data/downloaders/datasets/phc.py index 3938735e83..d8883a0959 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/phc.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/phc.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/regen.py b/esmvaltool/cmorizers/data/downloaders/datasets/regen.py index 4b4727c110..e585f981f7 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/regen.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/regen.py @@ -15,7 +15,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/scripps_co2_kum.py b/esmvaltool/cmorizers/data/downloaders/datasets/scripps_co2_kum.py index e9b202ea0a..8fa0fc4559 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/scripps_co2_kum.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/scripps_co2_kum.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/tcom_ch4.py b/esmvaltool/cmorizers/data/downloaders/datasets/tcom_ch4.py index c317ec7f06..ae5858c323 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/tcom_ch4.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/tcom_ch4.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/tcom_n2o.py b/esmvaltool/cmorizers/data/downloaders/datasets/tcom_n2o.py index 38d4515f97..809ae73c71 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/tcom_n2o.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/tcom_n2o.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/woa.py b/esmvaltool/cmorizers/data/downloaders/datasets/woa.py index da8529a680..f32de80f72 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/woa.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/woa.py @@ -15,7 +15,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/downloader.py b/esmvaltool/cmorizers/data/downloaders/downloader.py index f9f36c8ea5..8b40b7d2d0 100644 --- a/esmvaltool/cmorizers/data/downloaders/downloader.py +++ b/esmvaltool/cmorizers/data/downloaders/downloader.py @@ -9,7 +9,7 @@ class BaseDownloader(): Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Dataset to download dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/ftp.py b/esmvaltool/cmorizers/data/downloaders/ftp.py index 9f0cd5e8f9..3c9dae0e61 100644 --- a/esmvaltool/cmorizers/data/downloaders/ftp.py +++ b/esmvaltool/cmorizers/data/downloaders/ftp.py @@ -25,7 +25,7 @@ class FTPDownloader(BaseDownloader): Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration server : str FTP server URL dataset : str @@ -169,7 +169,7 @@ class CCIDownloader(FTPDownloader): Parameters ---------- config : dict - ESMValTool's user configuration + ESMValTool's configuration dataset : str Dataset to download dataset_info : dict diff --git a/esmvaltool/cmorizers/data/formatters/datasets/ct2019.py b/esmvaltool/cmorizers/data/formatters/datasets/ct2019.py index 33f56f234d..64f64f4e82 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/ct2019.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/ct2019.py @@ -11,7 +11,7 @@ Download and processing instructions Create a new empty directory ``$RAWOBSPATH/Tier2/CT2019`` (where - ``$RAWOBSPATH`` is given by your user configuration file) where the raw + ``$RAWOBSPATH`` is given in the configuration) where the raw data will be stored. The download of the data is automatically handled by this script. If data is already present in this directory, the download is skipped (to force a new download delete your old files). diff --git a/esmvaltool/cmorizers/data/formatters/datasets/merra.ncl b/esmvaltool/cmorizers/data/formatters/datasets/merra.ncl index b57bca6a09..d9fbf761df 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/merra.ncl +++ b/esmvaltool/cmorizers/data/formatters/datasets/merra.ncl @@ -14,7 +14,7 @@ ; Download and processing instructions ; (requires EarthData login; see https://urs.earthdata.nasa.gov/) ; Use ESMValTool automatic download: -; esmvaltool data download --config_file MERRA +; esmvaltool data download MERRA ; ; Modification history ; 20230818-lauer_axel: added output of clwvi (iwp + lwp) @@ -209,7 +209,7 @@ begin delete(tmp) - ; calcuation of outgoing fluxes: out = in - net + ; calculation of outgoing fluxes: out = in - net if ((VAR(vv) .eq. "rsut") .or. (VAR(vv) .eq. "rsutcs")) then tmp = f->SWTDN if (isatt(tmp, "scale_factor") .or. isatt(tmp, "add_offset")) then @@ -220,7 +220,8 @@ begin delete(tmp) end if - ; calcuation of total precipitation flux = large-scale+convective+anvil + ; calculation of total precipitation flux = + ; large-scale+convective+anvil if (VAR(vv) .eq. "pr") then tmp = f->PRECCON ; surface precipitation flux from convection if (isatt(tmp, "scale_factor") .or. isatt(tmp, "add_offset")) then diff --git a/esmvaltool/cmorizers/data/formatters/datasets/mls_aura.py b/esmvaltool/cmorizers/data/formatters/datasets/mls_aura.py index 5b500e9087..0a5031b243 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/mls_aura.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/mls_aura.py @@ -14,7 +14,7 @@ Select "Data Access" -> "Subset/Get Data" -> "Get Data" and follow the "Instructions for downloading". All *.he5 files need to be saved in the $RAWOBS/Tier3/MLS-AURA directory, where $RAWOBS refers to the RAWOBS - directory defined in the user configuration file. Apply this procedure to + directory defined in the configuration. Apply this procedure to both links provided above. The temperature fields are necessary for quality control of the RHI data (see Data Quality Document for MLS-AURA for more information). diff --git a/tests/integration/test_cmorizer.py b/tests/integration/test_cmorizer.py index 11bade4190..13d636cbd3 100644 --- a/tests/integration/test_cmorizer.py +++ b/tests/integration/test_cmorizer.py @@ -14,6 +14,7 @@ import yaml from cf_units import Unit +from esmvaltool import ESMValToolDeprecationWarning from esmvaltool.cmorizers.data.cmorizer import DataCommand @@ -28,8 +29,8 @@ def keep_cwd(): os.chdir(curr_path) -def write_config_user_file(dirname): - """Replace config_user file values for testing.""" +def write_config_file(dirname): + """Replace configuration values for testing.""" config_file = dirname / 'config-user.yml' cfg = { 'output_dir': str(dirname / 'output_dir'), @@ -143,14 +144,48 @@ def arguments(*args): sys.argv = backup -def test_cmorize_obs_woa_no_data(tmp_path): +# Remove in v2.14.0 +def test_cmorize_obs_woa_no_data_config_file(tmp_path): """Test for example run of cmorize_obs command.""" + config_file = write_config_file(tmp_path) + os.makedirs(os.path.join(tmp_path, 'raw_stuff', 'Tier2')) + with keep_cwd(): + with pytest.raises(Exception): + with pytest.warns(ESMValToolDeprecationWarning): + DataCommand().format('WOA', config_file=config_file) + + log_dir = os.path.join(tmp_path, 'output_dir') + log_file = os.path.join(log_dir, + os.listdir(log_dir)[0], 'run', 'main_log.txt') + check_log_file(log_file, no_data=True) - config_user_file = write_config_user_file(tmp_path) + +# Remove in v2.14.0 +def test_cmorize_obs_woa_data_config_file(tmp_path): + """Test for example run of cmorize_obs command.""" + config_file = write_config_file(tmp_path) + data_path = os.path.join(tmp_path, 'raw_stuff', 'Tier2', 'WOA') + put_dummy_data(data_path) + with keep_cwd(): + with pytest.warns(ESMValToolDeprecationWarning): + DataCommand().format('WOA', config_file=config_file) + + log_dir = os.path.join(tmp_path, 'output_dir') + log_file = os.path.join(log_dir, + os.listdir(log_dir)[0], 'run', 'main_log.txt') + check_log_file(log_file, no_data=False) + output_path = os.path.join(log_dir, os.listdir(log_dir)[0], 'Tier2', 'WOA') + check_output_exists(output_path) + check_conversion(output_path) + + +def test_cmorize_obs_woa_no_data(tmp_path): + """Test for example run of cmorize_obs command.""" + write_config_file(tmp_path) os.makedirs(os.path.join(tmp_path, 'raw_stuff', 'Tier2')) with keep_cwd(): with pytest.raises(Exception): - DataCommand().format('WOA', config_user_file) + DataCommand().format('WOA', config_dir=str(tmp_path)) log_dir = os.path.join(tmp_path, 'output_dir') log_file = os.path.join(log_dir, @@ -160,12 +195,11 @@ def test_cmorize_obs_woa_no_data(tmp_path): def test_cmorize_obs_woa_data(tmp_path): """Test for example run of cmorize_obs command.""" - - config_user_file = write_config_user_file(tmp_path) + write_config_file(tmp_path) data_path = os.path.join(tmp_path, 'raw_stuff', 'Tier2', 'WOA') put_dummy_data(data_path) with keep_cwd(): - DataCommand().format('WOA', config_user_file) + DataCommand().format('WOA', config_dir=str(tmp_path)) log_dir = os.path.join(tmp_path, 'output_dir') log_file = os.path.join(log_dir, From 9e185bc0e31c48a5ea85b9e273203b92f238fdfc Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 2 Oct 2024 18:31:40 +0200 Subject: [PATCH 06/29] Adapt diags --- .../diag_scripts/kcs/local_resampling.py | 4 +- .../diag_scripts/monitor/compute_eofs.py | 4 +- esmvaltool/diag_scripts/monitor/monitor.py | 4 +- .../diag_scripts/monitor/multi_datasets.py | 4 +- .../russell18jgr/russell18jgr-fig6a.ncl | 6 +-- .../russell18jgr/russell18jgr-fig6b.ncl | 13 +++-- .../russell18jgr/russell18jgr-fig7i.ncl | 9 ++-- .../russell18jgr/russell18jgr-fig9c.ncl | 9 ++-- tests/integration/test_diagnostic_run.py | 54 +++++++++++++++++-- 9 files changed, 73 insertions(+), 34 deletions(-) diff --git a/esmvaltool/diag_scripts/kcs/local_resampling.py b/esmvaltool/diag_scripts/kcs/local_resampling.py index 9eb2ea28ed..0bf6260d65 100644 --- a/esmvaltool/diag_scripts/kcs/local_resampling.py +++ b/esmvaltool/diag_scripts/kcs/local_resampling.py @@ -292,7 +292,7 @@ def select_final_subset(cfg, subsets, prov=None): Final set of eight samples should have with minimal reuse of the same ensemble member for the same period. From 10.000 randomly - selected sets of 8 samples, count and penalize re-used segments (1 + selected sets of 8 samples, count and penalize reused segments (1 for 3*reuse, 5 for 4*reuse). Choose the set with the lowest penalty. """ n_samples = cfg['n_samples'] @@ -387,7 +387,7 @@ def _get_climatology(cfg, scenario_name, table, prov=None): resampled_control = _recombine(segments_control, table['control']) resampled_future = _recombine(segments_future, table['future']) - # Store the resampled contol climates + # Store the resampled control climates filename = get_diagnostic_filename(f'resampled_control_{scenario_name}', cfg, extension='nc') diff --git a/esmvaltool/diag_scripts/monitor/compute_eofs.py b/esmvaltool/diag_scripts/monitor/compute_eofs.py index dea5d63b9a..a07ca835c0 100644 --- a/esmvaltool/diag_scripts/monitor/compute_eofs.py +++ b/esmvaltool/diag_scripts/monitor/compute_eofs.py @@ -24,10 +24,10 @@ Path to the folder to store figures. Defaults to ``{plot_dir}/../../{dataset}/{exp}/{modeling_realm}/{real_name}``. All tags (i.e., the entries in curly brackets, e.g., ``{dataset}``, are - replaced with the corresponding tags). ``{plot_dir}`` is replaced with the + replaced with the corresponding tags). ``{plot_dir}`` is replaced with the default ESMValTool plot directory (i.e., ``output_dir/plots/diagnostic_name/script_name/``, see - :ref:`esmvalcore:user configuration file`). + :ref:`esmvalcore:outputdata`). rasterize_maps: bool, optional (default: True) If ``True``, use `rasterization `_ for diff --git a/esmvaltool/diag_scripts/monitor/monitor.py b/esmvaltool/diag_scripts/monitor/monitor.py index 59e37b9842..dda5aa4f3d 100644 --- a/esmvaltool/diag_scripts/monitor/monitor.py +++ b/esmvaltool/diag_scripts/monitor/monitor.py @@ -52,10 +52,10 @@ Path to the folder to store figures. Defaults to ``{plot_dir}/../../{dataset}/{exp}/{modeling_realm}/{real_name}``. All tags (i.e., the entries in curly brackets, e.g., ``{dataset}``, are - replaced with the corresponding tags). ``{plot_dir}`` is replaced with the + replaced with the corresponding tags). ``{plot_dir}`` is replaced with the default ESMValTool plot directory (i.e., ``output_dir/plots/diagnostic_name/script_name/``, see - :ref:`esmvalcore:user configuration file`). + :ref:`esmvalcore:outputdata`). rasterize_maps: bool, optional (default: True) If ``True``, use `rasterization `_ for diff --git a/esmvaltool/diag_scripts/monitor/multi_datasets.py b/esmvaltool/diag_scripts/monitor/multi_datasets.py index 879346954c..32f654b3b6 100644 --- a/esmvaltool/diag_scripts/monitor/multi_datasets.py +++ b/esmvaltool/diag_scripts/monitor/multi_datasets.py @@ -100,10 +100,10 @@ Path to the folder to store figures. Defaults to ``{plot_dir}/../../{dataset}/{exp}/{modeling_realm}/{real_name}``. All tags (i.e., the entries in curly brackets, e.g., ``{dataset}``, are - replaced with the corresponding tags). ``{plot_dir}`` is replaced with the + replaced with the corresponding tags). ``{plot_dir}`` is replaced with the default ESMValTool plot directory (i.e., ``output_dir/plots/diagnostic_name/script_name/``, see - :ref:`esmvalcore:user configuration file`). + :ref:`esmvalcore:outputdata`). savefig_kwargs: dict, optional Optional keyword arguments for :func:`matplotlib.pyplot.savefig`. By default, uses ``bbox_inches: tight, dpi: 300, orientation: landscape``. diff --git a/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig6a.ncl b/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig6a.ncl index bd672ed3cf..0f1b49c224 100644 --- a/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig6a.ncl +++ b/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig6a.ncl @@ -151,10 +151,8 @@ begin fx_variable = "volcello" error_msg("f", "russell18jgr-fig6.ncl", " ", "volcello file for " \ + vo_datasets(iii) \ - + " not found in the metadata file, please add "\ - + "'fx_files: [volcello]' to the variable dictionary in the " \ - + "recipe or add the location of file to input directory " \ - + "in config-user.yml ") + + " not found in the metadata file, please specify " \ + + "'volcello' as supplementary variable in the recipe.") end if dataset_so_time = read_data(so_items[iii]) diff --git a/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig6b.ncl b/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig6b.ncl index 6b019625f0..71323f411d 100644 --- a/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig6b.ncl +++ b/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig6b.ncl @@ -45,10 +45,10 @@ ; ; Caveats ; -; - MIROC-ESM and BNU-ESM doesnot work as depth variable is not called lev. -; - MRI_ESM1 doesnot work as the data is ofset by 80 degrees in longitude +; - MIROC-ESM and BNU-ESM does not work as depth variable is not called lev. +; - MRI_ESM1 does not work as the data is offset by 80 degrees in longitude ; and causes problem in interpolation. -; - CCSM4 ans CESM1-CAM5 dont work as the units for so is 1, not accepted +; - CCSM4 and CESM1-CAM5 dont work as the units for so is 1, not accepted ; by ESMValTool. ; - Transport is very small in case of NorESM1-M and ME as volcello ; values look incorrect(very small). @@ -153,11 +153,10 @@ begin if (all(ismissing(fx_var))) then fx_variable = "volcello" - error_msg("f", "russell_fig-7i.ncl", " ", "areacello file for " + \ + error_msg("f", "russell_fig-7i.ncl", " ", "volcello file for " + \ vo_datasets(iii) \ - + " not found in the metadata file, please " + \ - "add 'fx_files: [volcello]' to the variable dictionary in" + \ - " the recipe or add the location of file to config-user.yml") + + " not found in the metadata file, please specify " \ + + "'volcello' as supplementary variable in the recipe.") end if dataset_so_time = read_data(so_items[iii]) diff --git a/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig7i.ncl b/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig7i.ncl index 86ce4bee70..aefcf323d9 100644 --- a/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig7i.ncl +++ b/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig7i.ncl @@ -156,9 +156,8 @@ begin fx_variable = "areacello" error_msg("f", "russell_fig-7i.ncl", " ", "areacello file for " + \ datasetnames(iii) + " not found in the metadata file," + \ - " please add 'fx_files: [areacello]' to the variable " + \ - "dictionary in the recipe or add the location of " + \ - " file to config-user.yml") + + " not found in the metadata file, please specify " \ + + "'areacello' as supplementary variable in the recipe.") end if areacello_2d = fx_var delete(fx_var) @@ -212,9 +211,9 @@ begin "lgPerimOn" : False ; no perimeter "lgItemCount" : dimsizes(annots) ; how many "lgLineLabelStrings" : annots ; labels - "lgLabelsOn" : False ; no default lables + "lgLabelsOn" : False ; no default labsels "lgLineLabelFontHeightF" : 0.0085 ; font height - "lgDashIndexes" : dashes ; line paterns + "lgDashIndexes" : dashes ; line patterns "lgLineColors" : colors "lgMonoLineLabelFontColor" : True ; one label color end create diff --git a/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig9c.ncl b/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig9c.ncl index 2fe0cc3e4a..017b70103a 100644 --- a/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig9c.ncl +++ b/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig9c.ncl @@ -227,9 +227,8 @@ begin if (all(ismissing(fx_var))) then error_msg("f", "russell18jgr-fig9c.ncl", " ", "areacello file for " + \ datasetnames(iii) + " not found in the metadata file, " + \ - "please add 'fx_files: [areacello]' to the variable " + \ - "dictionary in the recipe or add the location of " + \ - " file to config-user.yml ") + + " not found in the metadata file, please specify " \ + + "'areacello' as supplementary variable in the recipe.") end if areacello_2d = fx_var @@ -304,9 +303,9 @@ begin "lgPerimOn" : False ; no perimeter "lgItemCount" : dimsizes(annots) ; how many "lgLabelStrings" : annots ; labels - "lgLabelsOn" : True ; no default lables + "lgLabelsOn" : True ; no default labels "lgLabelFontHeightF" : 0.001 ; font height - "lgItemType" : "markers" ; line paterns + "lgItemType" : "markers" ; line patterns "lgMarkerColors" : colors "lgMarkerIndexes" : markers ; one label color end create diff --git a/tests/integration/test_diagnostic_run.py b/tests/integration/test_diagnostic_run.py index b0c606f4ee..a9b52a25d4 100644 --- a/tests/integration/test_diagnostic_run.py +++ b/tests/integration/test_diagnostic_run.py @@ -8,9 +8,10 @@ import pytest import yaml from esmvalcore._main import run +from esmvalcore.exceptions import ESMValCoreDeprecationWarning -def write_config_user_file(dirname): +def write_config_file(dirname): config_file = dirname / 'config-user.yml' cfg = { 'output_dir': str(dirname / 'output_dir'), @@ -68,10 +69,10 @@ def check(result_file): ] +# Remove in v2.14.0 @pytest.mark.installation @pytest.mark.parametrize('script_file', SCRIPTS) -def test_diagnostic_run(tmp_path, script_file): - +def test_diagnostic_run_config_file(tmp_path, script_file): local_script_file = Path(__file__).parent / script_file recipe_file = tmp_path / 'recipe_test.yml' @@ -96,12 +97,55 @@ def test_diagnostic_run(tmp_path, script_file): """.format(script_file, result_file)) recipe_file.write_text(str(recipe)) - config_user_file = write_config_user_file(tmp_path) + config_file = write_config_file(tmp_path) with arguments( 'esmvaltool', 'run', '--config_file', - config_user_file, + config_file, + str(recipe_file), + ): + with pytest.warns(ESMValCoreDeprecationWarning): + run() + + check(result_file) + + +@pytest.mark.installation +@pytest.mark.parametrize('script_file', SCRIPTS) +def test_diagnostic_run(tmp_path, script_file): + local_script_file = Path(__file__).parent / script_file + + recipe_file = tmp_path / 'recipe_test.yml' + script_file = tmp_path / script_file + result_file = tmp_path / 'result.yml' + config_dir = tmp_path / 'config' + config_dir.mkdir(exist_ok=True, parents=True) + + shutil.copy(local_script_file, script_file) + + # Create recipe + recipe = dedent(""" + documentation: + title: Test recipe + description: Recipe with no data. + authors: [andela_bouwe] + + diagnostics: + diagnostic_name: + scripts: + script_name: + script: {} + setting_name: {} + """.format(script_file, result_file)) + recipe_file.write_text(str(recipe)) + + write_config_file(config_dir) + with arguments( + 'esmvaltool', + 'run', + '--config_dir', + str(config_dir), str(recipe_file), ): run() From 0119432d7734daca0d2f0232239363cfdc9856ab Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 2 Oct 2024 18:31:50 +0200 Subject: [PATCH 07/29] Adapt recipes --- esmvaltool/recipes/examples/recipe_extract_shape.yml | 2 +- esmvaltool/recipes/hydrology/recipe_hydro_forcing.yml | 4 ++-- esmvaltool/recipes/hydrology/recipe_lisflood.yml | 3 ++- esmvaltool/recipes/hydrology/recipe_marrmot.yml | 3 ++- .../ipccwg1ar6ch3/recipe_ipccwg1ar6ch3_fig_3_42_a.yml | 2 +- esmvaltool/recipes/recipe_carvalhais14nat.yml | 2 +- esmvaltool/recipes/recipe_runoff_et.yml | 2 +- esmvaltool/recipes/recipe_sea_surface_salinity.yml | 5 ++--- esmvaltool/recipes/recipe_shapeselect.yml | 3 +-- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/esmvaltool/recipes/examples/recipe_extract_shape.yml b/esmvaltool/recipes/examples/recipe_extract_shape.yml index 79f04371b5..08d1bab490 100644 --- a/esmvaltool/recipes/examples/recipe_extract_shape.yml +++ b/esmvaltool/recipes/examples/recipe_extract_shape.yml @@ -7,7 +7,7 @@ documentation: The example shapefile(s) can be copied from esmvaltool/diag_scripts/shapeselect/testdata/Elbe.* and - placed in the auxiliary_data_dir defined in config-user.yml. + placed in the auxiliary_data_dir defined in the configuration. title: Example recipe extracting precipitation in the Elbe catchment. diff --git a/esmvaltool/recipes/hydrology/recipe_hydro_forcing.yml b/esmvaltool/recipes/hydrology/recipe_hydro_forcing.yml index f68a597733..925d9bd420 100644 --- a/esmvaltool/recipes/hydrology/recipe_hydro_forcing.yml +++ b/esmvaltool/recipes/hydrology/recipe_hydro_forcing.yml @@ -9,7 +9,7 @@ documentation: used to: 1. Plot a timeseries of the raw daily data - 2. Plot monthly aggregrated data over a certain period + 2. Plot monthly aggregated data over a certain period 3. Plot the monthly climate statistics over a certain period authors: @@ -33,7 +33,7 @@ datasets: preprocessors: daily: extract_shape: &extract_shape - # In aux (config-user.yml) + # Relative to auxiliary_data_dir defined in configuration shapefile: Lorentz_Basin_Shapefiles/Meuse/Meuse.shp method: contains crop: true diff --git a/esmvaltool/recipes/hydrology/recipe_lisflood.yml b/esmvaltool/recipes/hydrology/recipe_lisflood.yml index ffecbc37be..3acb4be481 100644 --- a/esmvaltool/recipes/hydrology/recipe_lisflood.yml +++ b/esmvaltool/recipes/hydrology/recipe_lisflood.yml @@ -37,7 +37,8 @@ preprocessors: scheme: linear extract_shape: # Perhaps a single shapefile needs to be created covering multiple basins - shapefile: Lorentz_Basin_Shapefiles/Meuse/Meuse.shp # (config-user, aux) + # Relative to auxiliary_data_dir defined in configuration + shapefile: Lorentz_Basin_Shapefiles/Meuse/Meuse.shp method: contains crop: true # set to false to keep the entire globe (memory intensive!) daily_water: diff --git a/esmvaltool/recipes/hydrology/recipe_marrmot.yml b/esmvaltool/recipes/hydrology/recipe_marrmot.yml index dd6eef0a49..e85a66d9b9 100644 --- a/esmvaltool/recipes/hydrology/recipe_marrmot.yml +++ b/esmvaltool/recipes/hydrology/recipe_marrmot.yml @@ -28,7 +28,8 @@ preprocessors: daily: &daily extract_shape: # Lumped model: needs catchment-aggregated input data - shapefile: Meuse/Meuse.shp # In aux (config-user.yml) + # Relative to auxiliary_data_dir defined in configuration + shapefile: Meuse/Meuse.shp method: contains crop: true diff --git a/esmvaltool/recipes/ipccwg1ar6ch3/recipe_ipccwg1ar6ch3_fig_3_42_a.yml b/esmvaltool/recipes/ipccwg1ar6ch3/recipe_ipccwg1ar6ch3_fig_3_42_a.yml index 20b0402a23..55c53147ec 100644 --- a/esmvaltool/recipes/ipccwg1ar6ch3/recipe_ipccwg1ar6ch3_fig_3_42_a.yml +++ b/esmvaltool/recipes/ipccwg1ar6ch3/recipe_ipccwg1ar6ch3_fig_3_42_a.yml @@ -10,7 +10,7 @@ documentation: Contribution to the Sixth Assessment Report: Chapter 3 Processing of CMIP3 models currently works only in serial mode, due to an issue in the input data still under investigation. To run the recipe - set: max_parallel_tasks: 1 in the config-user.yml file. + set the configuration option ``max_parallel_tasks: 1``. authors: - bock_lisa diff --git a/esmvaltool/recipes/recipe_carvalhais14nat.yml b/esmvaltool/recipes/recipe_carvalhais14nat.yml index 9ec0811c00..63bfbb1edd 100644 --- a/esmvaltool/recipes/recipe_carvalhais14nat.yml +++ b/esmvaltool/recipes/recipe_carvalhais14nat.yml @@ -8,7 +8,7 @@ documentation: Carvalhais et al., 2014, Nature. The data required in the obs_details section can be obtained at http://www.bgc-jena.mpg.de/geodb/BGI/tau4ESMValTool.php - and have to be stored in the auxiliary_data_dir defined i config-user.yml, + and have to be stored in the auxiliary_data_dir defined in the configuration in a subdirectory obs_data_subdir specified in the obs_details section below. diff --git a/esmvaltool/recipes/recipe_runoff_et.yml b/esmvaltool/recipes/recipe_runoff_et.yml index 6924321c7c..0a83213caa 100644 --- a/esmvaltool/recipes/recipe_runoff_et.yml +++ b/esmvaltool/recipes/recipe_runoff_et.yml @@ -8,7 +8,7 @@ documentation: water balance components for different catchments and compares the results against observations. Currently, the required catchment mask needs to be downloaded manually at https://doi.org/10.5281/zenodo.2025776 and saved in - the auxiliary_data_dir defined in config-user.yml. + the auxiliary_data_dir defined in configuration. authors: - hagemann_stefan diff --git a/esmvaltool/recipes/recipe_sea_surface_salinity.yml b/esmvaltool/recipes/recipe_sea_surface_salinity.yml index 4e670eec7f..43ec0e6b5e 100644 --- a/esmvaltool/recipes/recipe_sea_surface_salinity.yml +++ b/esmvaltool/recipes/recipe_sea_surface_salinity.yml @@ -20,8 +20,7 @@ documentation: preprocessors: timeseries: extract_shape: - # Relative paths are relative to 'auxiliary_data_dir' as configured in - # the config-user.yml file. + # Relative paths are relative to the configuration option 'auxiliary_data_dir'. # The example shapefile can be downloaded from # https://marineregions.org/download_file.php?name=World_Seas_IHO_v3.zip # but any shapefile can be used @@ -50,7 +49,7 @@ datasets: - {<<: *cmip6, dataset: MPI-ESM1-2-HR, alias: MPI-ESM1-2-HR} - {<<: *cmip6, dataset: NorESM2-MM, alias: NorESM2-MM} - {<<: *cmip6, dataset: GISS-E2-2-H, alias: GISS-E2-2-H, institute: NASA-GISS} - + diagnostics: compare_salinity: diff --git a/esmvaltool/recipes/recipe_shapeselect.yml b/esmvaltool/recipes/recipe_shapeselect.yml index 0fb22c0d5d..ee56810f03 100644 --- a/esmvaltool/recipes/recipe_shapeselect.yml +++ b/esmvaltool/recipes/recipe_shapeselect.yml @@ -36,8 +36,7 @@ diagnostics: script: shapeselect/diag_shapeselect.py # Example shapefiles can be found in: # esmvaltool/diag_scripts/shapeselect/testdata/ - # Relative paths are relative to 'auxiliary_data_dir' as configured in - # the config-user.yml file. + # Relative paths are relative to configuration option 'auxiliary_data_dir'. shapefile: 'Thames.shp' weighting_method: 'mean_inside' write_xlsx: true From 1aad2420a9c4a3aa8ae0d8209d27c0bb182a6e70 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 2 Oct 2024 18:32:55 +0200 Subject: [PATCH 08/29] Adapt generate.py --- esmvaltool/utils/batch-jobs/generate.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/esmvaltool/utils/batch-jobs/generate.py b/esmvaltool/utils/batch-jobs/generate.py index afba37906f..06f9819456 100644 --- a/esmvaltool/utils/batch-jobs/generate.py +++ b/esmvaltool/utils/batch-jobs/generate.py @@ -9,7 +9,7 @@ - conda_path 2) If needed, edit optional parameters: - outputs -- config_file +- config_dir 3) SLURM settings This script is configured to optimize the computing footprint of the recipe testing. It is not necessary to edit @@ -49,11 +49,11 @@ # Full path to the mambaforge/etc/profile.d/conda.sh executable # Set the path to conda conda_path = 'PATH_TO/mambaforge/etc/profile.d/conda.sh' -# Full path to config_file -# If none, ~/.esmvaltool/config-user.yml is used -config_file = '' +# Full path to configuration directory +# If none, ~/.config/esmvaltool/ +config_dir = '' # Set max_parallel_tasks -# If none, read from config_file +# If none, read from configuration default_max_parallel_tasks = 8 # List of recipes that require non-default SLURM options set above @@ -315,11 +315,11 @@ def generate_submit(): file.write(f'. {conda_path}\n') file.write(f'conda activate {env}\n') file.write('\n') - if not config_file: + if not config_dir: file.write(f'esmvaltool run {str(recipe)}') else: - file.write(f'esmvaltool run --config_file ' - f'{str(config_file)} {str(recipe)}') + file.write(f'esmvaltool run --config_dir ' + f'{str(config_dir)} {str(recipe)}') # set max_parallel_tasks max_parallel_tasks = MAX_PARALLEL_TASKS.get( recipe.stem, From d3eab268fc32156e095e4c8766faa1c8a13cf003 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 2 Oct 2024 18:33:06 +0200 Subject: [PATCH 09/29] Adapt NCL logging --- esmvaltool/interface_scripts/logging.ncl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/esmvaltool/interface_scripts/logging.ncl b/esmvaltool/interface_scripts/logging.ncl index 6333479f96..35c3167341 100644 --- a/esmvaltool/interface_scripts/logging.ncl +++ b/esmvaltool/interface_scripts/logging.ncl @@ -61,9 +61,9 @@ procedure log_debug(output_string[*]:string) ; output_string: the text to be output as message on screen ; ; Description -; Write a debug message to the log file (only if log_level = debug in -; config-user.yml). If the input is an array, each element will be -; written on different lines. +; Write a debug message to the log file (only if log_level = debug in the +; configuration). If the input is an array, each element will be written on +; different lines. ; ; Caveats ; From 78f83158ca6971f2b5c65de953faed5164ababb7 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 2 Oct 2024 18:38:45 +0200 Subject: [PATCH 10/29] Undo changes in individual CMORizers --- esmvaltool/cmorizers/data/downloaders/cds.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/aeronet.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/aphro_ma.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/aura_tes.py | 2 +- .../cmorizers/data/downloaders/datasets/berkeleyearth.py | 2 +- .../cmorizers/data/downloaders/datasets/calipso_goccp.py | 2 +- .../cmorizers/data/downloaders/datasets/calipso_icecloud.py | 2 +- .../data/downloaders/datasets/cds_satellite_albedo.py | 2 +- .../data/downloaders/datasets/cds_satellite_lai_fapar.py | 2 +- .../data/downloaders/datasets/cds_satellite_soil_moisture.py | 4 ++-- esmvaltool/cmorizers/data/downloaders/datasets/cds_uerra.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/cds_xch4.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/cowtanway.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/cru.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/ct2019.py | 2 +- .../cmorizers/data/downloaders/datasets/duveiller2018.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/e_obs.py | 2 +- .../cmorizers/data/downloaders/datasets/eppley_vgpm_modis.py | 2 +- .../cmorizers/data/downloaders/datasets/esacci_aerosol.py | 2 +- .../cmorizers/data/downloaders/datasets/esacci_cloud.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/esacci_fire.py | 2 +- .../cmorizers/data/downloaders/datasets/esacci_landcover.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/esacci_oc.py | 2 +- .../cmorizers/data/downloaders/datasets/esacci_ozone.py | 2 +- .../data/downloaders/datasets/esacci_sea_surface_salinity.py | 2 +- .../data/downloaders/datasets/esacci_soilmoisture.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/ghcn.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/ghcn_cams.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/gistemp.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/glodap.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/gpcc.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/gpcp_sg.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/hadcrut3.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/hadcrut4.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/hadcrut5.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/hadisst.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/haloe.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/isccp_fh.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/jra_25.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/jra_55.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/kadow2020.py | 2 +- .../cmorizers/data/downloaders/datasets/landflux_eval.py | 2 +- .../cmorizers/data/downloaders/datasets/landschuetzer2016.py | 2 +- .../cmorizers/data/downloaders/datasets/landschuetzer2020.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/merra.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/merra2.py | 2 +- .../cmorizers/data/downloaders/datasets/mobo_dic2004_2019.py | 2 +- .../cmorizers/data/downloaders/datasets/mobo_dic_mpim.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/ncep_doe_r2.py | 2 +- .../cmorizers/data/downloaders/datasets/ncep_ncar_r1.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/ndp.py | 2 +- .../cmorizers/data/downloaders/datasets/noaa_cires_20cr_v2.py | 2 +- .../cmorizers/data/downloaders/datasets/noaa_cires_20cr_v3.py | 2 +- .../cmorizers/data/downloaders/datasets/noaa_ersstv3b.py | 2 +- .../cmorizers/data/downloaders/datasets/noaa_ersstv5.py | 2 +- .../cmorizers/data/downloaders/datasets/noaa_mbl_ch4.py | 2 +- .../cmorizers/data/downloaders/datasets/noaaglobaltemp.py | 2 +- .../cmorizers/data/downloaders/datasets/nsidc_0116_nh.py | 2 +- .../cmorizers/data/downloaders/datasets/nsidc_0116_sh.py | 2 +- .../cmorizers/data/downloaders/datasets/nsidc_g02202_sh.py | 2 +- .../cmorizers/data/downloaders/datasets/oceansoda_ethz.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/osi_450_nh.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/osi_450_sh.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/patmos_x.py | 2 +- .../cmorizers/data/downloaders/datasets/persiann_cdr.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/phc.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/regen.py | 2 +- .../cmorizers/data/downloaders/datasets/scripps_co2_kum.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/tcom_ch4.py | 2 +- esmvaltool/cmorizers/data/downloaders/datasets/tcom_n2o.py | 2 +- esmvaltool/cmorizers/data/downloaders/downloader.py | 2 +- esmvaltool/cmorizers/data/downloaders/ftp.py | 4 ++-- 72 files changed, 74 insertions(+), 74 deletions(-) diff --git a/esmvaltool/cmorizers/data/downloaders/cds.py b/esmvaltool/cmorizers/data/downloaders/cds.py index e717bc4c43..af1ad62501 100644 --- a/esmvaltool/cmorizers/data/downloaders/cds.py +++ b/esmvaltool/cmorizers/data/downloaders/cds.py @@ -19,7 +19,7 @@ class CDSDownloader(BaseDownloader): product_name : str Name of the product in the CDS config : dict - ESMValTool's configuration + ESMValTool's user configuration request_dictionary : dict Common CDS request parameters dataset : str diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/aeronet.py b/esmvaltool/cmorizers/data/downloaders/datasets/aeronet.py index fb4cba80f2..668a688bb6 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/aeronet.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/aeronet.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/aphro_ma.py b/esmvaltool/cmorizers/data/downloaders/datasets/aphro_ma.py index 77662358c2..425bd93e88 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/aphro_ma.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/aphro_ma.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/aura_tes.py b/esmvaltool/cmorizers/data/downloaders/datasets/aura_tes.py index 85b1c5e2e8..9f7e195397 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/aura_tes.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/aura_tes.py @@ -10,7 +10,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/berkeleyearth.py b/esmvaltool/cmorizers/data/downloaders/datasets/berkeleyearth.py index a29a9802f3..01ce0e6fa3 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/berkeleyearth.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/berkeleyearth.py @@ -10,7 +10,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/calipso_goccp.py b/esmvaltool/cmorizers/data/downloaders/datasets/calipso_goccp.py index 4040e65565..1c93d85b61 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/calipso_goccp.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/calipso_goccp.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/calipso_icecloud.py b/esmvaltool/cmorizers/data/downloaders/datasets/calipso_icecloud.py index 04a6d3c3a6..c8992e3d38 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/calipso_icecloud.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/calipso_icecloud.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_albedo.py b/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_albedo.py index fc8ae46bad..a789f9b227 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_albedo.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_albedo.py @@ -15,7 +15,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_lai_fapar.py b/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_lai_fapar.py index 80c8039a51..65c2bafa0f 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_lai_fapar.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_lai_fapar.py @@ -15,7 +15,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_soil_moisture.py b/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_soil_moisture.py index 99b92ecb23..8b017321e1 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_soil_moisture.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/cds_satellite_soil_moisture.py @@ -16,7 +16,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict @@ -89,7 +89,7 @@ def get_downloader(config, dataset, dataset_info, overwrite, sensor, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/cds_uerra.py b/esmvaltool/cmorizers/data/downloaders/datasets/cds_uerra.py index 644a0d5804..7641aea6ad 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/cds_uerra.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/cds_uerra.py @@ -15,7 +15,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/cds_xch4.py b/esmvaltool/cmorizers/data/downloaders/datasets/cds_xch4.py index c7455ca4f5..241db20f2d 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/cds_xch4.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/cds_xch4.py @@ -11,7 +11,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/cowtanway.py b/esmvaltool/cmorizers/data/downloaders/datasets/cowtanway.py index 03417fcd45..edffbeb18c 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/cowtanway.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/cowtanway.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/cru.py b/esmvaltool/cmorizers/data/downloaders/datasets/cru.py index 010dc3f5ce..8fbce3e9a3 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/cru.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/cru.py @@ -15,7 +15,7 @@ def download_dataset( Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/ct2019.py b/esmvaltool/cmorizers/data/downloaders/datasets/ct2019.py index 7c838962b2..4eed312963 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/ct2019.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/ct2019.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/duveiller2018.py b/esmvaltool/cmorizers/data/downloaders/datasets/duveiller2018.py index bb12d1c6cb..8cf91a15ee 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/duveiller2018.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/duveiller2018.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/e_obs.py b/esmvaltool/cmorizers/data/downloaders/datasets/e_obs.py index 2b6d395dbc..fd12127e43 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/e_obs.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/e_obs.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/eppley_vgpm_modis.py b/esmvaltool/cmorizers/data/downloaders/datasets/eppley_vgpm_modis.py index e6ab8280d4..f70ee8ab35 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/eppley_vgpm_modis.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/eppley_vgpm_modis.py @@ -17,7 +17,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_aerosol.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_aerosol.py index 19427bc82c..09d2616c9c 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_aerosol.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_aerosol.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_cloud.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_cloud.py index 2f14a2bc92..a14255e143 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_cloud.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_cloud.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_fire.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_fire.py index ea17b99cad..82fbc743ad 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_fire.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_fire.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_landcover.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_landcover.py index 427ea1fbb0..efffa2aaaa 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_landcover.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_landcover.py @@ -12,7 +12,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_oc.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_oc.py index 6347977d3f..64d4209477 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_oc.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_oc.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_ozone.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_ozone.py index dd2e2c9197..50d80e06f3 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_ozone.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_ozone.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sea_surface_salinity.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sea_surface_salinity.py index c8ea036476..dc38903624 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sea_surface_salinity.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_sea_surface_salinity.py @@ -17,7 +17,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_soilmoisture.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_soilmoisture.py index d06bc1310b..0d29e96ff9 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_soilmoisture.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_soilmoisture.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/ghcn.py b/esmvaltool/cmorizers/data/downloaders/datasets/ghcn.py index 872523c8df..ceeb44de30 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/ghcn.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/ghcn.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/ghcn_cams.py b/esmvaltool/cmorizers/data/downloaders/datasets/ghcn_cams.py index 96ef17e023..40b3b1a8a8 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/ghcn_cams.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/ghcn_cams.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/gistemp.py b/esmvaltool/cmorizers/data/downloaders/datasets/gistemp.py index 2b2d8cf031..2bf73fa4ed 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/gistemp.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/gistemp.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/glodap.py b/esmvaltool/cmorizers/data/downloaders/datasets/glodap.py index 534499c7f2..f4de91f47f 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/glodap.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/glodap.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/gpcc.py b/esmvaltool/cmorizers/data/downloaders/datasets/gpcc.py index f38023a3c5..6279970be0 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/gpcc.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/gpcc.py @@ -17,7 +17,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/gpcp_sg.py b/esmvaltool/cmorizers/data/downloaders/datasets/gpcp_sg.py index 2b364a1085..91e1b4e47b 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/gpcp_sg.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/gpcp_sg.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut3.py b/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut3.py index a4172a601a..6481537442 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut3.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut3.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut4.py b/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut4.py index a0f36275d9..986e0bd2e7 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut4.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut4.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut5.py b/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut5.py index 5c6d9f35cc..6c8eda7c57 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut5.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/hadcrut5.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/hadisst.py b/esmvaltool/cmorizers/data/downloaders/datasets/hadisst.py index cabe34c2a9..1ac279f284 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/hadisst.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/hadisst.py @@ -15,7 +15,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/haloe.py b/esmvaltool/cmorizers/data/downloaders/datasets/haloe.py index e8bdd43fca..fd396119fb 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/haloe.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/haloe.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/isccp_fh.py b/esmvaltool/cmorizers/data/downloaders/datasets/isccp_fh.py index b23880097d..9f2c15b794 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/isccp_fh.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/isccp_fh.py @@ -15,7 +15,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/jra_25.py b/esmvaltool/cmorizers/data/downloaders/datasets/jra_25.py index 1f37a08d40..27e3dbc97c 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/jra_25.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/jra_25.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/jra_55.py b/esmvaltool/cmorizers/data/downloaders/datasets/jra_55.py index c68b5e0fce..7a9e374136 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/jra_55.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/jra_55.py @@ -17,7 +17,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/kadow2020.py b/esmvaltool/cmorizers/data/downloaders/datasets/kadow2020.py index f05d40ee07..2c08dd3c04 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/kadow2020.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/kadow2020.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/landflux_eval.py b/esmvaltool/cmorizers/data/downloaders/datasets/landflux_eval.py index b8bd3784ad..1d8e84cb9d 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/landflux_eval.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/landflux_eval.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/landschuetzer2016.py b/esmvaltool/cmorizers/data/downloaders/datasets/landschuetzer2016.py index 0d9eb75b94..e08bae771e 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/landschuetzer2016.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/landschuetzer2016.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/landschuetzer2020.py b/esmvaltool/cmorizers/data/downloaders/datasets/landschuetzer2020.py index cce6a6319c..88929cfc7a 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/landschuetzer2020.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/landschuetzer2020.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/merra.py b/esmvaltool/cmorizers/data/downloaders/datasets/merra.py index d18371b127..df1d0ff7e9 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/merra.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/merra.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/merra2.py b/esmvaltool/cmorizers/data/downloaders/datasets/merra2.py index 762cbf09ed..28ddebf720 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/merra2.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/merra2.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/mobo_dic2004_2019.py b/esmvaltool/cmorizers/data/downloaders/datasets/mobo_dic2004_2019.py index 711bffb8dd..1299981811 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/mobo_dic2004_2019.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/mobo_dic2004_2019.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/mobo_dic_mpim.py b/esmvaltool/cmorizers/data/downloaders/datasets/mobo_dic_mpim.py index 3e16c76de5..58af5689d7 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/mobo_dic_mpim.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/mobo_dic_mpim.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/ncep_doe_r2.py b/esmvaltool/cmorizers/data/downloaders/datasets/ncep_doe_r2.py index 6c7f2b8a37..704493554f 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/ncep_doe_r2.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/ncep_doe_r2.py @@ -15,7 +15,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/ncep_ncar_r1.py b/esmvaltool/cmorizers/data/downloaders/datasets/ncep_ncar_r1.py index b8f51d779f..f9b0816efe 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/ncep_ncar_r1.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/ncep_ncar_r1.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/ndp.py b/esmvaltool/cmorizers/data/downloaders/datasets/ndp.py index 2a21f9d8c9..391e973d28 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/ndp.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/ndp.py @@ -15,7 +15,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_cires_20cr_v2.py b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_cires_20cr_v2.py index 5d02e93d7b..fb2d733f06 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_cires_20cr_v2.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_cires_20cr_v2.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_cires_20cr_v3.py b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_cires_20cr_v3.py index 190a48f895..67f1a38f33 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_cires_20cr_v3.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_cires_20cr_v3.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv3b.py b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv3b.py index 167c28be54..5a54080be4 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv3b.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv3b.py @@ -16,7 +16,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv5.py b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv5.py index a6cee113a3..7dbeccfe12 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv5.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv5.py @@ -16,7 +16,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_mbl_ch4.py b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_mbl_ch4.py index 11c90919ab..3cbf701c97 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_mbl_ch4.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_mbl_ch4.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/noaaglobaltemp.py b/esmvaltool/cmorizers/data/downloaders/datasets/noaaglobaltemp.py index c63e104278..cf946df2cc 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/noaaglobaltemp.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/noaaglobaltemp.py @@ -15,7 +15,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_0116_nh.py b/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_0116_nh.py index f83ed1e221..c4bb4e3afe 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_0116_nh.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_0116_nh.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_0116_sh.py b/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_0116_sh.py index b9aa93acfc..adc8acd5ee 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_0116_sh.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_0116_sh.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_g02202_sh.py b/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_g02202_sh.py index b7c908119d..8c3c02c410 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_g02202_sh.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_g02202_sh.py @@ -16,7 +16,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/oceansoda_ethz.py b/esmvaltool/cmorizers/data/downloaders/datasets/oceansoda_ethz.py index f6fb85d505..2e099814dd 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/oceansoda_ethz.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/oceansoda_ethz.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/osi_450_nh.py b/esmvaltool/cmorizers/data/downloaders/datasets/osi_450_nh.py index 3c58e254c8..826548503f 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/osi_450_nh.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/osi_450_nh.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/osi_450_sh.py b/esmvaltool/cmorizers/data/downloaders/datasets/osi_450_sh.py index 030f76982d..5e3fd383f4 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/osi_450_sh.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/osi_450_sh.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/patmos_x.py b/esmvaltool/cmorizers/data/downloaders/datasets/patmos_x.py index f516a7d72d..20b70e7fb9 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/patmos_x.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/patmos_x.py @@ -15,7 +15,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/persiann_cdr.py b/esmvaltool/cmorizers/data/downloaders/datasets/persiann_cdr.py index 27f8900e54..5aa19ef257 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/persiann_cdr.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/persiann_cdr.py @@ -15,7 +15,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/phc.py b/esmvaltool/cmorizers/data/downloaders/datasets/phc.py index d8883a0959..3938735e83 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/phc.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/phc.py @@ -14,7 +14,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/regen.py b/esmvaltool/cmorizers/data/downloaders/datasets/regen.py index e585f981f7..4b4727c110 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/regen.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/regen.py @@ -15,7 +15,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/scripps_co2_kum.py b/esmvaltool/cmorizers/data/downloaders/datasets/scripps_co2_kum.py index 8fa0fc4559..e9b202ea0a 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/scripps_co2_kum.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/scripps_co2_kum.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/tcom_ch4.py b/esmvaltool/cmorizers/data/downloaders/datasets/tcom_ch4.py index ae5858c323..c317ec7f06 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/tcom_ch4.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/tcom_ch4.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/tcom_n2o.py b/esmvaltool/cmorizers/data/downloaders/datasets/tcom_n2o.py index 809ae73c71..38d4515f97 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/tcom_n2o.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/tcom_n2o.py @@ -13,7 +13,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/downloader.py b/esmvaltool/cmorizers/data/downloaders/downloader.py index 8b40b7d2d0..f9f36c8ea5 100644 --- a/esmvaltool/cmorizers/data/downloaders/downloader.py +++ b/esmvaltool/cmorizers/data/downloaders/downloader.py @@ -9,7 +9,7 @@ class BaseDownloader(): Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Dataset to download dataset_info : dict diff --git a/esmvaltool/cmorizers/data/downloaders/ftp.py b/esmvaltool/cmorizers/data/downloaders/ftp.py index 3c9dae0e61..9f0cd5e8f9 100644 --- a/esmvaltool/cmorizers/data/downloaders/ftp.py +++ b/esmvaltool/cmorizers/data/downloaders/ftp.py @@ -25,7 +25,7 @@ class FTPDownloader(BaseDownloader): Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration server : str FTP server URL dataset : str @@ -169,7 +169,7 @@ class CCIDownloader(FTPDownloader): Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Dataset to download dataset_info : dict From 0aea81d5dbcdd0996466ad6e331e605644d990b2 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 2 Oct 2024 18:38:55 +0200 Subject: [PATCH 11/29] Undo changes in individual CMORizers --- esmvaltool/cmorizers/data/downloaders/datasets/woa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/woa.py b/esmvaltool/cmorizers/data/downloaders/datasets/woa.py index f32de80f72..da8529a680 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/woa.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/woa.py @@ -15,7 +15,7 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date, Parameters ---------- config : dict - ESMValTool's configuration + ESMValTool's user configuration dataset : str Name of the dataset dataset_info : dict From 429c76794a2888d5c99fe0ec075d17070caf1787 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 2 Oct 2024 18:43:23 +0200 Subject: [PATCH 12/29] Typo --- doc/sphinx/source/recipes/recipe_ipccwg1ar6ch3.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/source/recipes/recipe_ipccwg1ar6ch3.rst b/doc/sphinx/source/recipes/recipe_ipccwg1ar6ch3.rst index ebfb4e5cfd..69bf078390 100644 --- a/doc/sphinx/source/recipes/recipe_ipccwg1ar6ch3.rst +++ b/doc/sphinx/source/recipes/recipe_ipccwg1ar6ch3.rst @@ -225,7 +225,7 @@ User settings in recipe * plot_legend: if true, plot legend will be plotted * plot_units: variable unit for plotting - * multi_model_mean: if true, multi-model mean and uncertaintiy will be + * multi_model_mean: if true, multi-model mean and uncertainty will be plotted *Optional settings for variables* From 2fc5a4643af7d2a0c27d67f9ee81a58252f9d9d4 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 7 Oct 2024 14:07:28 +0200 Subject: [PATCH 13/29] Fix NCL code style --- esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig7i.ncl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig7i.ncl b/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig7i.ncl index aefcf323d9..cf14857a7b 100644 --- a/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig7i.ncl +++ b/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig7i.ncl @@ -157,7 +157,7 @@ begin error_msg("f", "russell_fig-7i.ncl", " ", "areacello file for " + \ datasetnames(iii) + " not found in the metadata file," + \ + " not found in the metadata file, please specify " \ - + "'areacello' as supplementary variable in the recipe.") + + "'areacello' as supplementary variable in the recipe.") end if areacello_2d = fx_var delete(fx_var) From 56db54d2369afb3fa79d362de2f753be0c967e37 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 7 Oct 2024 14:39:12 +0200 Subject: [PATCH 14/29] Backwards-compatible CMORizer tests --- tests/integration/test_cmorizer.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_cmorizer.py b/tests/integration/test_cmorizer.py index 13d636cbd3..b5eb1435d5 100644 --- a/tests/integration/test_cmorizer.py +++ b/tests/integration/test_cmorizer.py @@ -4,6 +4,7 @@ import os import sys +import esmvalcore import iris import iris.coord_systems import iris.coords @@ -13,6 +14,7 @@ import pytest import yaml from cf_units import Unit +from packaging import version from esmvaltool import ESMValToolDeprecationWarning from esmvaltool.cmorizers.data.cmorizer import DataCommand @@ -144,11 +146,15 @@ def arguments(*args): sys.argv = backup -# Remove in v2.14.0 +@pytest.mark.skipif( + version.parse(esmvalcore.__version__) >= version.parse("2.14.0"), + reason='ESMValCore < v2.14.0', +) def test_cmorize_obs_woa_no_data_config_file(tmp_path): """Test for example run of cmorize_obs command.""" config_file = write_config_file(tmp_path) os.makedirs(os.path.join(tmp_path, 'raw_stuff', 'Tier2')) + os.makedirs(os.path.join(tmp_path, 'output_dir')) with keep_cwd(): with pytest.raises(Exception): with pytest.warns(ESMValToolDeprecationWarning): @@ -160,7 +166,10 @@ def test_cmorize_obs_woa_no_data_config_file(tmp_path): check_log_file(log_file, no_data=True) -# Remove in v2.14.0 +@pytest.mark.skipif( + version.parse(esmvalcore.__version__) >= version.parse("2.14.0"), + reason='ESMValCore < v2.14.0', +) def test_cmorize_obs_woa_data_config_file(tmp_path): """Test for example run of cmorize_obs command.""" config_file = write_config_file(tmp_path) @@ -179,8 +188,13 @@ def test_cmorize_obs_woa_data_config_file(tmp_path): check_conversion(output_path) +@pytest.mark.skipif( + version.parse(esmvalcore.__version__) < version.parse("2.12.0"), + reason='ESMValCore < v2.12.0', +) def test_cmorize_obs_woa_no_data(tmp_path): """Test for example run of cmorize_obs command.""" + print(version.parse(esmvalcore.__version__)) write_config_file(tmp_path) os.makedirs(os.path.join(tmp_path, 'raw_stuff', 'Tier2')) with keep_cwd(): @@ -193,6 +207,10 @@ def test_cmorize_obs_woa_no_data(tmp_path): check_log_file(log_file, no_data=True) +@pytest.mark.skipif( + version.parse(esmvalcore.__version__) < version.parse("2.12.0"), + reason='ESMValCore < v2.12.0', +) def test_cmorize_obs_woa_data(tmp_path): """Test for example run of cmorize_obs command.""" write_config_file(tmp_path) From f5a13a210c55c34cf0bba7f300340eff4570bd3e Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 7 Oct 2024 14:47:01 +0200 Subject: [PATCH 15/29] Backwards-compatible diagnostic run tests --- tests/integration/test_cmorizer.py | 4 ++-- tests/integration/test_diagnostic_run.py | 12 +++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_cmorizer.py b/tests/integration/test_cmorizer.py index b5eb1435d5..c68486a217 100644 --- a/tests/integration/test_cmorizer.py +++ b/tests/integration/test_cmorizer.py @@ -148,7 +148,7 @@ def arguments(*args): @pytest.mark.skipif( version.parse(esmvalcore.__version__) >= version.parse("2.14.0"), - reason='ESMValCore < v2.14.0', + reason='ESMValCore >= v2.14.0', ) def test_cmorize_obs_woa_no_data_config_file(tmp_path): """Test for example run of cmorize_obs command.""" @@ -168,7 +168,7 @@ def test_cmorize_obs_woa_no_data_config_file(tmp_path): @pytest.mark.skipif( version.parse(esmvalcore.__version__) >= version.parse("2.14.0"), - reason='ESMValCore < v2.14.0', + reason='ESMValCore >= v2.14.0', ) def test_cmorize_obs_woa_data_config_file(tmp_path): """Test for example run of cmorize_obs command.""" diff --git a/tests/integration/test_diagnostic_run.py b/tests/integration/test_diagnostic_run.py index a9b52a25d4..f51f15d698 100644 --- a/tests/integration/test_diagnostic_run.py +++ b/tests/integration/test_diagnostic_run.py @@ -10,6 +10,9 @@ from esmvalcore._main import run from esmvalcore.exceptions import ESMValCoreDeprecationWarning +from packaging import version +import esmvalcore + def write_config_file(dirname): config_file = dirname / 'config-user.yml' @@ -69,7 +72,10 @@ def check(result_file): ] -# Remove in v2.14.0 +@pytest.mark.skipif( + version.parse(esmvalcore.__version__) >= version.parse("2.14.0"), + reason='ESMValCore >= v2.14.0', +) @pytest.mark.installation @pytest.mark.parametrize('script_file', SCRIPTS) def test_diagnostic_run_config_file(tmp_path, script_file): @@ -111,6 +117,10 @@ def test_diagnostic_run_config_file(tmp_path, script_file): check(result_file) +@pytest.mark.skipif( + version.parse(esmvalcore.__version__) < version.parse("2.12.0"), + reason='ESMValCore < v2.12.0', +) @pytest.mark.installation @pytest.mark.parametrize('script_file', SCRIPTS) def test_diagnostic_run(tmp_path, script_file): From c49c6a8d0859815b5b5400d400a3db09c7b921e5 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 7 Oct 2024 14:48:10 +0200 Subject: [PATCH 16/29] isort --- tests/integration/test_diagnostic_run.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integration/test_diagnostic_run.py b/tests/integration/test_diagnostic_run.py index f51f15d698..7edd8a7e0d 100644 --- a/tests/integration/test_diagnostic_run.py +++ b/tests/integration/test_diagnostic_run.py @@ -5,13 +5,12 @@ from pathlib import Path from textwrap import dedent +import esmvalcore import pytest import yaml from esmvalcore._main import run from esmvalcore.exceptions import ESMValCoreDeprecationWarning - from packaging import version -import esmvalcore def write_config_file(dirname): From 8cae71308c56860fd5efb04b8e57934a0dfb92ae Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 7 Oct 2024 14:48:51 +0200 Subject: [PATCH 17/29] Remove print --- tests/integration/test_cmorizer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_cmorizer.py b/tests/integration/test_cmorizer.py index c68486a217..fbe28052dd 100644 --- a/tests/integration/test_cmorizer.py +++ b/tests/integration/test_cmorizer.py @@ -194,7 +194,6 @@ def test_cmorize_obs_woa_data_config_file(tmp_path): ) def test_cmorize_obs_woa_no_data(tmp_path): """Test for example run of cmorize_obs command.""" - print(version.parse(esmvalcore.__version__)) write_config_file(tmp_path) os.makedirs(os.path.join(tmp_path, 'raw_stuff', 'Tier2')) with keep_cwd(): From 0e9f378341b16b4326bbcdc1dc8911188f30a6f4 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 7 Oct 2024 14:58:19 +0200 Subject: [PATCH 18/29] Do not test deprecation warnings from ESMValCore --- tests/integration/test_diagnostic_run.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integration/test_diagnostic_run.py b/tests/integration/test_diagnostic_run.py index 7edd8a7e0d..670f7088dd 100644 --- a/tests/integration/test_diagnostic_run.py +++ b/tests/integration/test_diagnostic_run.py @@ -9,7 +9,6 @@ import pytest import yaml from esmvalcore._main import run -from esmvalcore.exceptions import ESMValCoreDeprecationWarning from packaging import version @@ -110,8 +109,7 @@ def test_diagnostic_run_config_file(tmp_path, script_file): config_file, str(recipe_file), ): - with pytest.warns(ESMValCoreDeprecationWarning): - run() + run() check(result_file) From dfcc1205232dcc48ac8d7919d677b29e658925c8 Mon Sep 17 00:00:00 2001 From: Manuel Schlund <32543114+schlunma@users.noreply.github.com> Date: Tue, 15 Oct 2024 12:52:19 +0200 Subject: [PATCH 19/29] Expand environment variables when reading config dirs for CMORizer Co-authored-by: Bouwe Andela --- esmvaltool/cmorizers/data/cmorizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvaltool/cmorizers/data/cmorizer.py b/esmvaltool/cmorizers/data/cmorizer.py index ef4fd31cbb..70b0e78be0 100755 --- a/esmvaltool/cmorizers/data/cmorizer.py +++ b/esmvaltool/cmorizers/data/cmorizer.py @@ -66,7 +66,7 @@ def start(self, command, datasets, config_file, config_dir, options): if config_file is not None: # remove in v2.14.0 CFG.load_from_file(config_file) elif config_dir is not None: - config_dir = Path(config_dir).expanduser().absolute() + config_dir = Path(os.path.expandvars(config_dir)).expanduser().absolute() if not config_dir.is_dir(): raise NotADirectoryError( f"Invalid --config_dir given: {config_dir} is not an " From 697e60ffdfd17c1fdd14312f16cb95b203796f68 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Tue, 15 Oct 2024 12:56:57 +0200 Subject: [PATCH 20/29] Do not use bare Exception --- esmvaltool/cmorizers/data/cmorizer.py | 9 ++++++--- .../diag_scripts/climate_metrics/feedback_parameters.py | 2 +- tests/integration/test_cmorizer.py | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/esmvaltool/cmorizers/data/cmorizer.py b/esmvaltool/cmorizers/data/cmorizer.py index 70b0e78be0..5e66b7a70f 100755 --- a/esmvaltool/cmorizers/data/cmorizer.py +++ b/esmvaltool/cmorizers/data/cmorizer.py @@ -66,7 +66,9 @@ def start(self, command, datasets, config_file, config_dir, options): if config_file is not None: # remove in v2.14.0 CFG.load_from_file(config_file) elif config_dir is not None: - config_dir = Path(os.path.expandvars(config_dir)).expanduser().absolute() + config_dir = Path( + os.path.expandvars(config_dir) + ).expanduser().absolute() if not config_dir.is_dir(): raise NotADirectoryError( f"Invalid --config_dir given: {config_dir} is not an " @@ -213,8 +215,9 @@ def format(self, start, end, install): failed_datasets.append(dataset) if failed_datasets: - raise Exception( - f'Format failed for datasets {" ".join(failed_datasets)}') + raise RuntimeError( + f'Format failed for datasets {" ".join(failed_datasets)}' + ) @staticmethod def has_downloader(dataset): diff --git a/esmvaltool/diag_scripts/climate_metrics/feedback_parameters.py b/esmvaltool/diag_scripts/climate_metrics/feedback_parameters.py index db350982a2..d6bd28b0fb 100644 --- a/esmvaltool/diag_scripts/climate_metrics/feedback_parameters.py +++ b/esmvaltool/diag_scripts/climate_metrics/feedback_parameters.py @@ -365,7 +365,7 @@ def _create_regression_plot(tas_cube, y_reg = reg.slope * x_reg + reg.intercept # Plot data - title = (f'{FEEDBACK_PARAMETERS.get(var,var)} TOA radiance for ' + title = (f'{FEEDBACK_PARAMETERS.get(var, var)} TOA radiance for ' f'{dataset_name}') filename = f'{var}_regression_{dataset_name}' if description is not None: diff --git a/tests/integration/test_cmorizer.py b/tests/integration/test_cmorizer.py index fbe28052dd..48f75b951a 100644 --- a/tests/integration/test_cmorizer.py +++ b/tests/integration/test_cmorizer.py @@ -156,7 +156,7 @@ def test_cmorize_obs_woa_no_data_config_file(tmp_path): os.makedirs(os.path.join(tmp_path, 'raw_stuff', 'Tier2')) os.makedirs(os.path.join(tmp_path, 'output_dir')) with keep_cwd(): - with pytest.raises(Exception): + with pytest.raises(RuntimeError): with pytest.warns(ESMValToolDeprecationWarning): DataCommand().format('WOA', config_file=config_file) @@ -197,7 +197,7 @@ def test_cmorize_obs_woa_no_data(tmp_path): write_config_file(tmp_path) os.makedirs(os.path.join(tmp_path, 'raw_stuff', 'Tier2')) with keep_cwd(): - with pytest.raises(Exception): + with pytest.raises(RuntimeError): DataCommand().format('WOA', config_dir=str(tmp_path)) log_dir = os.path.join(tmp_path, 'output_dir') From f31f9b77fd2b97b09072d67a31feea15c274508c Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Tue, 15 Oct 2024 14:00:46 +0200 Subject: [PATCH 21/29] Add FAQ how to read multiple config dirs --- doc/sphinx/source/faq.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/doc/sphinx/source/faq.rst b/doc/sphinx/source/faq.rst index 4b7e79b5f6..43251a801b 100644 --- a/doc/sphinx/source/faq.rst +++ b/doc/sphinx/source/faq.rst @@ -59,6 +59,18 @@ This is a useful functionality because it allows the user to `fix` things on-the quitting the Ipython console, code execution continues as per normal. +Using multiple configuration directories +======================================== + +By default, ESMValTool will read YAML configuration files from the user +configuration directory ``~/.config/esmvaltool``, which can be changed with the +``ESMVALTOOL_CONFIG_DIR`` environment variable. +If required, users can specify the command line option ``--config_dir`` to +select another configuration directory, which is read **in addition** to the +user configuration directory +See the section on configuration :ref:`config_yaml_files` for details on this. + + Create a symbolic link to the latest output directory ===================================================== From d67b4bc9c815aaea367d6d2628aa61b23f92f888 Mon Sep 17 00:00:00 2001 From: Manuel Schlund <32543114+schlunma@users.noreply.github.com> Date: Tue, 15 Oct 2024 14:04:13 +0200 Subject: [PATCH 22/29] Update esmvaltool/cmorizers/data/datasets.yml Co-authored-by: Bouwe Andela --- esmvaltool/cmorizers/data/datasets.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvaltool/cmorizers/data/datasets.yml b/esmvaltool/cmorizers/data/datasets.yml index 27240e4d61..6544a68169 100644 --- a/esmvaltool/cmorizers/data/datasets.yml +++ b/esmvaltool/cmorizers/data/datasets.yml @@ -301,7 +301,7 @@ datasets: last_access: 2020-03-23 info: | Create a new empty directory ``$RAWOBSPATH/Tier2/CT2019`` (where - ``$RAWOBSPATH`` is given by your user configuration) where the raw + ``$RAWOBSPATH`` is given by your configuration) where the raw data will be stored. The download of the data is automatically handled by this script. If data is already present in this directory, the download is skipped (to force a new download delete your old files). From cf85b57747fac2f210f0a448bef5806b9731341c Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Tue, 15 Oct 2024 14:05:25 +0200 Subject: [PATCH 23/29] remove "user" before configuration --- esmvaltool/cmorizers/data/datasets.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/esmvaltool/cmorizers/data/datasets.yml b/esmvaltool/cmorizers/data/datasets.yml index 6544a68169..0f34a738dc 100644 --- a/esmvaltool/cmorizers/data/datasets.yml +++ b/esmvaltool/cmorizers/data/datasets.yml @@ -903,11 +903,11 @@ datasets: Select "Data Access" -> "Subset/Get Data" -> "Get Data" and follow the "Instructions for downloading". All *.he5 files need to be saved in the $RAWOBS/Tier3/MLS-AURA directory, where $RAWOBS refers to the RAWOBS - directory defined in the user configuration. Apply this procedure to - both links provided above. The temperature fields are necessary for quality + directory defined in the configuration. Apply this procedure to both + links provided above. The temperature fields are necessary for quality control of the RHI data (see Data Quality Document for MLS-AURA for more information). - A registration is required + A registration is required. MOBO-DIC_MPIM: tier: 2 From 51ef776ea234fe9c48ac0f8a945459f14111394e Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Tue, 15 Oct 2024 17:36:39 +0200 Subject: [PATCH 24/29] Better line breaks in doc --- doc/sphinx/source/quickstart/configuration.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/sphinx/source/quickstart/configuration.rst b/doc/sphinx/source/quickstart/configuration.rst index 4b28f748c8..9cea6413b6 100644 --- a/doc/sphinx/source/quickstart/configuration.rst +++ b/doc/sphinx/source/quickstart/configuration.rst @@ -5,10 +5,10 @@ Configuration ************* The ``esmvaltool`` command is provided by the ESMValCore package, the -documentation on configuring ESMValCore can be found :ref:`here -`. -An overview of all configuration options can be found :ref:`here -`. +documentation on configuring ESMValCore can be found +:ref:`here `. +An overview of all configuration options can be found +:ref:`here `. In particular, it is recommended to read the section on how to :ref:`specify configuration options ` and the section on :ref:`Finding data `. From 918471c612548da9e5141a74ea24b78c55a21aac Mon Sep 17 00:00:00 2001 From: Manuel Schlund <32543114+schlunma@users.noreply.github.com> Date: Wed, 16 Oct 2024 16:40:32 +0200 Subject: [PATCH 25/29] Update doc/sphinx/source/recipes/recipe_oceans.rst Co-authored-by: Bouwe Andela --- doc/sphinx/source/recipes/recipe_oceans.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/source/recipes/recipe_oceans.rst b/doc/sphinx/source/recipes/recipe_oceans.rst index 2c96425794..17552b39fa 100644 --- a/doc/sphinx/source/recipes/recipe_oceans.rst +++ b/doc/sphinx/source/recipes/recipe_oceans.rst @@ -763,7 +763,7 @@ Some of these diagnostic scripts may not function on machines with no access to the internet, as cartopy may try to download the shape files. The solution to this issue is the put the relevant cartopy shapefiles in a directory which is visible to esmvaltool, then link that path to ESMValTool via the -:ref:`configuration option ` `auxiliary_data_dir`. +:ref:`configuration option ` ``auxiliary_data_dir``. The cartopy masking files can be downloaded from: https://www.naturalearthdata.com/downloads/ From 4db3491d35570bf56a56b352aa7a5548fcc14db6 Mon Sep 17 00:00:00 2001 From: Manuel Schlund <32543114+schlunma@users.noreply.github.com> Date: Wed, 16 Oct 2024 16:40:44 +0200 Subject: [PATCH 26/29] Update doc/sphinx/source/recipes/recipe_shapeselect.rst Co-authored-by: Bouwe Andela --- doc/sphinx/source/recipes/recipe_shapeselect.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/source/recipes/recipe_shapeselect.rst b/doc/sphinx/source/recipes/recipe_shapeselect.rst index b443059ddc..12da974c28 100644 --- a/doc/sphinx/source/recipes/recipe_shapeselect.rst +++ b/doc/sphinx/source/recipes/recipe_shapeselect.rst @@ -29,7 +29,7 @@ User settings in recipe *Required settings (scripts)* - * shapefile: path to the user provided shapefile. A relative path is relative to the :ref:`configuration option ` `auxiliary_data_dir``. + * shapefile: path to the user provided shapefile. A relative path is relative to the :ref:`configuration option ` ``auxiliary_data_dir``. * weighting_method: the preferred weighting method 'mean_inside' - mean of all grid points inside polygon; 'representative' - one point inside or close to the polygon is used to represent the complete area. From ce62d9f57c8b141f831f0c1f61738752b9f4e6c1 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 16 Oct 2024 16:43:08 +0200 Subject: [PATCH 27/29] `rootpath` -> ``rootpath`` --- doc/sphinx/source/community/dataset.rst | 8 ++++---- doc/sphinx/source/input.rst | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/sphinx/source/community/dataset.rst b/doc/sphinx/source/community/dataset.rst index 620d46a3c2..7a24e7c923 100644 --- a/doc/sphinx/source/community/dataset.rst +++ b/doc/sphinx/source/community/dataset.rst @@ -43,14 +43,14 @@ and run the recipe, to make sure the CMOR checks pass without warnings or errors To test a pull request for a new CMORizer script: #. Download the data following the instructions included in the script and - place it in the ``RAWOBS`` `rootpath` specified in your :ref:`configuration - ` + place it in the ``RAWOBS`` ``rootpath`` specified in your + :ref:`configuration ` #. If available, use the downloading script by running ``esmvaltool data download --config_file `` #. Run the cmorization by running ``esmvaltool data format `` #. Copy the resulting data to the ``OBS`` (for CMIP5 compliant data) or ``OBS6`` - (for CMIP6 compliant data) `rootpath` specified in your :ref:`configuration - ` + (for CMIP6 compliant data) ``rootpath`` specified in your + :ref:`configuration ` #. Run ``recipes/examples/recipe_check_obs.yml`` with the new dataset to check that the data can be used diff --git a/doc/sphinx/source/input.rst b/doc/sphinx/source/input.rst index bd66ef89b3..f972789315 100644 --- a/doc/sphinx/source/input.rst +++ b/doc/sphinx/source/input.rst @@ -193,7 +193,7 @@ To CMORize one or more datasets, run: esmvaltool data format --config_file [CONFIG_FILE] [DATASET_LIST] -The `rootpath` to the raw data to be CMORized must be specified in the +The ``rootpath`` to the raw data to be CMORized must be specified in the :ref:`configuration ` as ``RAWOBS``. Within this path, the data are expected to be organized in subdirectories corresponding to the data tier: Tier2 for freely-available datasets (other than From 75e2b3363d25f54602c65767a56cfc035a7630eb Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 16 Oct 2024 16:44:33 +0200 Subject: [PATCH 28/29] `auxiliary_data_dir` -> ``auxiliary_data_dir`` --- doc/sphinx/source/recipes/recipe_climwip.rst | 3 ++- doc/sphinx/source/recipes/recipe_ipccwg1ar6ch3.rst | 2 +- doc/sphinx/source/recipes/recipe_rainfarm.rst | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/sphinx/source/recipes/recipe_climwip.rst b/doc/sphinx/source/recipes/recipe_climwip.rst index 45e23fe685..900698b85a 100644 --- a/doc/sphinx/source/recipes/recipe_climwip.rst +++ b/doc/sphinx/source/recipes/recipe_climwip.rst @@ -44,7 +44,8 @@ Using shapefiles for cutting scientific regions To use shapefiles for selecting SREX or AR6 regions by name it is necessary to download them, e.g., from the sources below and reference the file using the `shapefile` parameter. This can either be a absolute or a relative path. In the example recipes they are stored in a subfolder `shapefiles` -in the :ref:`configuration option ` `auxiliary_data_dir`. +in the :ref:`configuration option ` +``auxiliary_data_dir``. SREX regions (AR5 reference regions): http://www.ipcc-data.org/guidelines/pages/ar5_regions.html diff --git a/doc/sphinx/source/recipes/recipe_ipccwg1ar6ch3.rst b/doc/sphinx/source/recipes/recipe_ipccwg1ar6ch3.rst index 69bf078390..718c345b19 100644 --- a/doc/sphinx/source/recipes/recipe_ipccwg1ar6ch3.rst +++ b/doc/sphinx/source/recipes/recipe_ipccwg1ar6ch3.rst @@ -45,7 +45,7 @@ To reproduce Fig. 3.9 you need the shapefile of the `AR6 reference regions Please download the file `IPCC-WGI-reference-regions-v4_shapefile.zip `_, unzip and store it in `/IPCC-regions/` (where -`auxiliary_data_dir` is given as :ref:`configuration option +``auxiliary_data_dir`` is given as :ref:`configuration option `). .. _`Eyring et al., 2021`: https://www.ipcc.ch/report/ar6/wg1/chapter/chapter-3/ diff --git a/doc/sphinx/source/recipes/recipe_rainfarm.rst b/doc/sphinx/source/recipes/recipe_rainfarm.rst index d6c06c6f7a..aeb7cd0638 100644 --- a/doc/sphinx/source/recipes/recipe_rainfarm.rst +++ b/doc/sphinx/source/recipes/recipe_rainfarm.rst @@ -32,7 +32,7 @@ User settings * nf: number of subdivisions for downscaling (e.g. 8 will produce output fields with linear resolution increased by a factor 8) * conserv_glob: logical, if to conserve precipitation over full domain * conserv_smooth: logical, if to conserve precipitation using convolution (if neither conserv_glob or conserv_smooth is chosen, box conservation is used) -* weights_climo: set to false or omit if no orographic weights are to be used, else set it to the path to a fine-scale precipitation climatology file. If a relative file path is used, `auxiliary_data_dir` will be searched for this file. The file is expected to be in NetCDF format and should contain at least one precipitation field. If several fields at different times are provided, a climatology is derived by time averaging. Suitable climatology files could be for example a fine-scale precipitation climatology from a high-resolution regional climate model (see e.g. Terzago et al. 2018), a local high-resolution gridded climatology from observations, or a reconstruction such as those which can be downloaded from the WORLDCLIM (http://www.worldclim.org) or CHELSA (http://chelsa-climate.org) websites. The latter data will need to be converted to NetCDF format before being used (see for example the GDAL tools (https://www.gdal.org). +* weights_climo: set to false or omit if no orographic weights are to be used, else set it to the path to a fine-scale precipitation climatology file. If a relative file path is used, ``auxiliary_data_dir`` will be searched for this file. The file is expected to be in NetCDF format and should contain at least one precipitation field. If several fields at different times are provided, a climatology is derived by time averaging. Suitable climatology files could be for example a fine-scale precipitation climatology from a high-resolution regional climate model (see e.g. Terzago et al. 2018), a local high-resolution gridded climatology from observations, or a reconstruction such as those which can be downloaded from the WORLDCLIM (http://www.worldclim.org) or CHELSA (http://chelsa-climate.org) websites. The latter data will need to be converted to NetCDF format before being used (see for example the GDAL tools (https://www.gdal.org). Variables @@ -60,4 +60,4 @@ Example plots .. figure:: /recipes/figures/rainfarm/rainfarm.png :width: 14cm - Example of daily cumulated precipitation from the CMIP5 EC-EARTH model on a specific day, downscaled using RainFARM from its original resolution (1.125°) (left panel), increasing spatial resolution by a factor of 8 to 0.14°; Two stochastic realizations are shown (central and right panel). A fixed spectral slope of s=1.7 was used. Notice how the downscaled fields introduce fine scale precipitation structures, while still maintaining on average the original coarse-resolution precipitation. Different stochastic realizations are shown to demonstrate how an ensemble of realizations can be used to reproduce unresolved subgrid variability. (N.B.: this plot was not produced by ESMValTool - the recipe output is netcdf only). + Example of daily cumulated precipitation from the CMIP5 EC-EARTH model on a specific day, downscaled using RainFARM from its original resolution (1.125°) (left panel), increasing spatial resolution by a factor of 8 to 0.14°; Two stochastic realizations are shown (central and right panel). A fixed spectral slope of s=1.7 was used. Notice how the downscaled fields introduce fine scale precipitation structures, while still maintaining on average the original coarse-resolution precipitation. Different stochastic realizations are shown to demonstrate how an ensemble of realizations can be used to reproduce unresolved subgrid variability. (N.B.: this plot was not produced by ESMValTool - the recipe output is netcdf only). From f0e75650e310f51ba7537f2ca0a28c4e3923d7b9 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 21 Oct 2024 15:16:32 +0200 Subject: [PATCH 29/29] Restored original recipe_filler --- esmvaltool/utils/recipe_filler.py | 47 +++++++++++++------------ tests/integration/test_recipe_filler.py | 23 ++++++------ 2 files changed, 36 insertions(+), 34 deletions(-) diff --git a/esmvaltool/utils/recipe_filler.py b/esmvaltool/utils/recipe_filler.py index 14350ae08d..40f637c6d5 100755 --- a/esmvaltool/utils/recipe_filler.py +++ b/esmvaltool/utils/recipe_filler.py @@ -30,8 +30,7 @@ - `start_year` and `end_year` are mandatory and are used to filter out the datasets that don't have data in the interval; if you want all possible years hence no filtering on years just use "*" for start and end years; -- `rootpath: CMIPX` in the configuration may be a list, rootpath lists are - supported; +- `config-user: rootpath: CMIPX` may be a list, rootpath lists are supported; Caveats: @@ -53,6 +52,7 @@ import esmvalcore import yaml + from esmvalcore import __version__ as core_ver from esmvalcore.cmor.table import CMOR_TABLES, read_cmor_tables from packaging import version as pkg_version @@ -183,13 +183,13 @@ def _normalize_path(path): return os.path.abspath(os.path.expanduser(os.path.expandvars(path))) -def read_config_file(config_file, folder_name, options=None): - """Read configuration file and store settings in a dictionary.""" +def read_config_user_file(config_file, folder_name, options=None): + """Read config user file and store settings in a dictionary.""" if not config_file: - config_file = '~/.config/esmvaltool/config-user.yml' + config_file = '~/.esmvaltool/config-user.yml' config_file = os.path.abspath( os.path.expandvars(os.path.expanduser(config_file))) - # Read config file + # Read user config file if not os.path.exists(config_file): print(f"ERROR: Config file {config_file} does not exist") @@ -292,14 +292,14 @@ def read_config_file(config_file, folder_name, options=None): def _get_download_dir(yamlconf, cmip_era): - """Get the Download Directory from configuration file.""" + """Get the Download Directory from user config file.""" if 'download_dir' in yamlconf: return os.path.join(yamlconf['download_dir'], cmip_era) return False def _get_site_rootpath(cmip_era): - """Get site (drs) from configuration.""" + """Get site (drs) from config-user.yml.""" config_yml = get_args().config_file with open(config_yml, 'r') as yamf: yamlconf = yaml.safe_load(yamf) @@ -541,7 +541,7 @@ def list_all_files(file_dict, cmip_era): if not new_path.startswith(os.sep): raise ValueError( "Could not expand ~ to user home dir " - "please expand it in the configuration file!") + "please expand it in the config user file!") logger.info("Expanding path to %s", new_path) # Globs all the wildcards into a list of files. @@ -656,20 +656,20 @@ def _check_recipe(recipe_dict): raise ValueError("Please fix the issues in recipe and rerun") -def _check_config_file(config_file): +def _check_config_file(user_config_file): """Perform a quick recipe check for mandatory fields.""" do_exit = False - if "rootpath" not in config_file: + if "rootpath" not in user_config_file: logger.error("Config file missing rootpath section.") do_exit = True - if "drs" not in config_file: + if "drs" not in user_config_file: logger.error("Config file missing drs section.") do_exit = True for proj in cmip_eras: - if proj not in config_file["rootpath"].keys(): + if proj not in user_config_file["rootpath"].keys(): logger.error("Config file missing rootpath for %s", proj) do_exit = True - if proj not in config_file["drs"].keys(): + if proj not in user_config_file["drs"].keys(): logger.error("Config file missing drs for %s", proj) do_exit = True if do_exit: @@ -790,9 +790,8 @@ def get_args(): parser.add_argument('recipe', help='Path/name of yaml pilot recipe file') parser.add_argument('-c', '--config-file', - default=os.path.expanduser( - '~/.config/esmvaltool/config-user.yml' - ), + default=os.path.join(os.environ["HOME"], '.esmvaltool', + 'config-user.yml'), help='User configuration file') parser.add_argument('-o', @@ -836,23 +835,25 @@ def run(): cmip_eras = ["CMIP5", "CMIP6"] # read the config file - cfg = read_config_file(args.config_file, 'recipe_filler', options={}) + config_user = read_config_user_file(args.config_file, + 'recipe_filler', + options={}) # configure logger - run_dir = os.path.join(cfg['output_dir'], 'recipe_filler') + run_dir = os.path.join(config_user['output_dir'], 'recipe_filler') if not os.path.isdir(run_dir): os.makedirs(run_dir) log_files = configure_logging(output_dir=run_dir, - console_log_level=cfg['log_level']) + console_log_level=config_user['log_level']) logger.info(HEADER) - logger.info("Using configuration file: %s", args.config_file) + logger.info("Using user configuration file: %s", args.config_file) logger.info("Using pilot recipe file: %s", input_recipe) logger.info("Writing filled out recipe to: %s", output_recipe) log_files = "\n".join(log_files) logger.info("Writing program log files to:\n%s", log_files) - # check configuration file - _check_config_file(cfg) + # check config user file + _check_config_file(config_user) # parse recipe with open(input_recipe, 'r') as yamlfile: diff --git a/tests/integration/test_recipe_filler.py b/tests/integration/test_recipe_filler.py index 8dfa900416..b78ac8c5f8 100644 --- a/tests/integration/test_recipe_filler.py +++ b/tests/integration/test_recipe_filler.py @@ -10,6 +10,7 @@ from esmvaltool.utils.recipe_filler import run + # Load test configuration with open(os.path.join(os.path.dirname(__file__), 'recipe_filler.yml')) as file: @@ -64,7 +65,7 @@ def create_tree(path, filenames=None, symlinks=None): os.symlink(symlink['target'], link_name) -def write_config_file(dirname, file_path, drs): +def write_config_user_file(dirname, file_path, drs): config_file = dirname / 'config-user.yml' cfg = { 'log_level': 'info', @@ -101,7 +102,7 @@ def root(): def setup_files(tmp_path, root, cfg): """Create config, recipe ,output recipe etc.""" - config_file = write_config_file(tmp_path, root, cfg['drs']) + user_config_file = write_config_user_file(tmp_path, root, cfg['drs']) diagnostics = {} diagnostics["test_diagnostic"] = {} diagnostics["test_diagnostic"]["variables"] = {} @@ -109,7 +110,7 @@ def setup_files(tmp_path, root, cfg): recipe = write_recipe(tmp_path, diagnostics) output_recipe = str(tmp_path / "recipe_auto.yml") - return config_file, recipe, output_recipe + return user_config_file, recipe, output_recipe @pytest.mark.parametrize('cfg', CONFIG['has_additional_datasets']) @@ -118,13 +119,13 @@ def test_adding_datasets(tmp_path, root, cfg): create_tree(root, cfg.get('available_files'), cfg.get('available_symlinks')) - config_file, recipe, output_recipe = setup_files(tmp_path, root, cfg) + user_config_file, recipe, output_recipe = setup_files(tmp_path, root, cfg) with arguments( 'recipe_filler', recipe, '-c', - config_file, + user_config_file, '-o', output_recipe, ): @@ -143,13 +144,13 @@ def test_not_adding_datasets(tmp_path, root, cfg): create_tree(root, cfg.get('available_files'), cfg.get('available_symlinks')) - config_file, recipe, output_recipe = setup_files(tmp_path, root, cfg) + user_config_file, recipe, output_recipe = setup_files(tmp_path, root, cfg) with arguments( 'recipe_filler', recipe, '-c', - config_file, + user_config_file, '-o', output_recipe, ): @@ -165,7 +166,7 @@ def test_not_adding_datasets(tmp_path, root, cfg): def test_bad_var(tmp_path, root): """Test a bad variable in the works.""" cfg = CONFIG['bad_variable'][0] - config_file, recipe, output_recipe = setup_files(tmp_path, root, cfg) + user_config_file, recipe, output_recipe = setup_files(tmp_path, root, cfg) # this doesn't fail and it shouldn't since it can go on # and look for data for other valid variables @@ -173,7 +174,7 @@ def test_bad_var(tmp_path, root): 'recipe_filler', recipe, '-c', - config_file, + user_config_file, '-o', output_recipe, ): @@ -189,7 +190,7 @@ def test_bad_var(tmp_path, root): def test_no_short_name(tmp_path, root): """Test a bad variable in the works.""" cfg = CONFIG['no_short_name'][0] - config_file, recipe, output_recipe = setup_files(tmp_path, root, cfg) + user_config_file, recipe, output_recipe = setup_files(tmp_path, root, cfg) # this doesn't fail and it shouldn't since it can go on # and look for data for other valid variables @@ -197,7 +198,7 @@ def test_no_short_name(tmp_path, root): 'recipe_filler', recipe, '-c', - config_file, + user_config_file, '-o', output_recipe, ):