diff --git a/.circleci/config.yml b/.circleci/config.yml index eb13a0ef08..82492e724f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -216,8 +216,8 @@ jobs: conda activate esmvaltool mkdir -p ~/climate_data esmvaltool config get_config_user - echo "search_esgf: when_missing" >> ~/.esmvaltool/config-user.yml - cat ~/.esmvaltool/config-user.yml + echo "search_esgf: when_missing" >> ~/.config/esmvaltool/config-user.yml + cat ~/.config/esmvaltool/config-user.yml for recipe in esmvaltool/recipes/testing/recipe_*.yml; do esmvaltool run "$recipe" done diff --git a/config-user-example.yml b/config-user-example.yml deleted file mode 100644 index c102928db9..0000000000 --- a/config-user-example.yml +++ /dev/null @@ -1,274 +0,0 @@ -############################################################################### -# Example user configuration file for ESMValTool -############################################################################### -# -# Note for users: -# -------------- -# Site-specific entries for different HPC centers are given at the bottom of -# this file. Comment out/replace as needed. This default version of the file -# can be used in combination with the command line argument -# ``search_esgf=when_missing``. If only certain values are allowed for an -# option, these are listed after ``---``. The option in square brackets is the -# default value, i.e., the one that is used if this option is omitted in the -# file. -# -############################################################################### -# -# Note for developers: -# ------------------- -# Two identical copies of this file (``ESMValTool/config-user-example.yml`` and -# ``ESMValCore/esmvalcore/config-user.yml``) exist. If you change one of it, -# make sure to apply the changes to the other. -# -############################################################################### ---- - -# Destination directory where all output will be written -# Includes log files and performance stats. -output_dir: ~/esmvaltool_output - -# Auxiliary data directory -# Used by some recipes to look for additional datasets. -auxiliary_data_dir: ~/auxiliary_data - -# Automatic data download from ESGF --- [never]/when_missing/always -# Use automatic download of missing CMIP3, CMIP5, CMIP6, CORDEX, and obs4MIPs -# data from ESGF. ``never`` disables this feature, which is useful if you are -# working on a computer without an internet connection, or if you have limited -# disk space. ``when_missing`` enables the automatic download for files that -# are not available locally. ``always`` will always check ESGF for the latest -# version of a file, and will only use local files if they correspond to that -# latest version. -search_esgf: never - -# Directory for storing downloaded climate data -# Make sure to use a directory where you can store multiple GBs of data. Your -# home directory on a HPC is usually not suited for this purpose, so please -# change the default value in this case! -download_dir: ~/climate_data - -# Run at most this many tasks in parallel --- [null]/1/2/3/4/... -# Set to ``null`` to use the number of available CPUs. If you run out of -# memory, try setting max_parallel_tasks to ``1`` and check the amount of -# memory you need for that by inspecting the file ``run/resource_usage.txt`` in -# the output directory. Using the number there you can increase the number of -# parallel tasks again to a reasonable number for the amount of memory -# available in your system. -max_parallel_tasks: null - -# Log level of the console --- debug/[info]/warning/error -# For much more information printed to screen set log_level to ``debug``. -log_level: info - -# Exit on warning --- true/[false] -# # Only used in NCL diagnostic scripts. -exit_on_warning: false - -# Plot file format --- [png]/pdf/ps/eps/epsi -output_file_type: png - -# Remove the ``preproc`` directory if the run was successful --- [true]/false -# By default this option is set to ``true``, so all preprocessor output files -# will be removed after a successful run. Set to ``false`` if you need those -# files. -remove_preproc_dir: true - -# Use netCDF compression --- true/[false] -compress_netcdf: false - -# Save intermediary cubes in the preprocessor --- true/[false] -# Setting this to ``true`` will save the output cube from each preprocessing -# step. These files are numbered according to the preprocessing order. -save_intermediary_cubes: false - -# Path to custom ``config-developer.yml`` file -# This can be used to customise project configurations. See -# ``config-developer.yml`` for an example. Set to ``null`` to use the default. -config_developer_file: null - -# Use a profiling tool for the diagnostic run --- [false]/true -# A profiler tells you which functions in your code take most time to run. -# Only available for Python diagnostics. -profile_diagnostic: false - -# Rootpaths to the data from different projects -# This default setting will work if files have been downloaded by ESMValTool -# via ``search_esgf``. Lists are also possible. For site-specific entries and -# more examples, see below. Comment out these when using a site-specific path. -rootpath: - default: ~/climate_data - -# Directory structure for input data --- [default]/ESGF/BADC/DKRZ/ETHZ/etc. -# This default setting will work if files have been downloaded by ESMValTool -# via ``search_esgf``. See ``config-developer.yml`` for definitions. Comment -# out/replace as per needed. -drs: - CMIP3: ESGF - CMIP5: ESGF - CMIP6: ESGF - CORDEX: ESGF - obs4MIPs: ESGF - -# Example rootpaths and directory structure that showcases the different -# projects and also the use of lists -# For site-specific entries, see below. -#rootpath: -# CMIP3: [~/cmip3_inputpath1, ~/cmip3_inputpath2] -# CMIP5: [~/cmip5_inputpath1, ~/cmip5_inputpath2] -# CMIP6: [~/cmip6_inputpath1, ~/cmip6_inputpath2] -# OBS: ~/obs_inputpath -# OBS6: ~/obs6_inputpath -# obs4MIPs: ~/obs4mips_inputpath -# ana4mips: ~/ana4mips_inputpath -# native6: ~/native6_inputpath -# RAWOBS: ~/rawobs_inputpath -# default: ~/default_inputpath -#drs: -# CMIP3: default -# CMIP5: default -# CMIP6: default -# CORDEX: default -# obs4MIPs: default - -# Directory tree created by automatically downloading from ESGF -# Uncomment the lines below to locate data that has been automatically -# downloaded from ESGF (using ``search_esgf``). -#rootpath: -# CMIP3: ~/climate_data -# CMIP5: ~/climate_data -# CMIP6: ~/climate_data -# CORDEX: ~/climate_data -# obs4MIPs: ~/climate_data -#drs: -# CMIP3: ESGF -# CMIP5: ESGF -# CMIP6: ESGF -# CORDEX: ESGF -# obs4MIPs: ESGF - -# Site-specific entries: JASMIN -# Uncomment the lines below to locate data on JASMIN. -#auxiliary_data_dir: /gws/nopw/j04/esmeval/aux_data/AUX -#rootpath: -# CMIP6: /badc/cmip6/data/CMIP6 -# CMIP5: /badc/cmip5/data/cmip5/output1 -# CMIP3: /badc/cmip3_drs/data/cmip3/output -# OBS: /gws/nopw/j04/esmeval/obsdata-v2 -# OBS6: /gws/nopw/j04/esmeval/obsdata-v2 -# obs4MIPs: /gws/nopw/j04/esmeval/obsdata-v2 -# ana4mips: /gws/nopw/j04/esmeval/obsdata-v2 -# CORDEX: /badc/cordex/data/CORDEX/output -#drs: -# CMIP6: BADC -# CMIP5: BADC -# CMIP3: BADC -# CORDEX: BADC -# OBS: default -# OBS6: default -# obs4MIPs: default -# ana4mips: default - -# Site-specific entries: DKRZ-Levante -# For bd0854 members a shared download directory is available -#search_esgf: when_missing -#download_dir: /work/bd0854/DATA/ESMValTool2/download -# Uncomment the lines below to locate data on Levante at DKRZ. -#auxiliary_data_dir: /work/bd0854/DATA/ESMValTool2/AUX -#rootpath: -# CMIP6: /work/bd0854/DATA/ESMValTool2/CMIP6_DKRZ -# CMIP5: /work/bd0854/DATA/ESMValTool2/CMIP5_DKRZ -# CMIP3: /work/bd0854/DATA/ESMValTool2/CMIP3 -# CORDEX: /work/ik1017/C3SCORDEX/data/c3s-cordex/output -# OBS: /work/bd0854/DATA/ESMValTool2/OBS -# OBS6: /work/bd0854/DATA/ESMValTool2/OBS -# obs4MIPs: /work/bd0854/DATA/ESMValTool2/OBS -# ana4mips: /work/bd0854/DATA/ESMValTool2/OBS -# native6: /work/bd0854/DATA/ESMValTool2/RAWOBS -# RAWOBS: /work/bd0854/DATA/ESMValTool2/RAWOBS -#drs: -# CMIP6: DKRZ -# CMIP5: DKRZ -# CMIP3: DKRZ -# CORDEX: BADC -# obs4MIPs: default -# ana4mips: default -# OBS: default -# OBS6: default -# native6: default - -# Site-specific entries: ETHZ -# Uncomment the lines below to locate data at ETHZ. -#rootpath: -# CMIP6: /net/atmos/data/cmip6 -# CMIP5: /net/atmos/data/cmip5 -# CMIP3: /net/atmos/data/cmip3 -# OBS: /net/exo/landclim/PROJECTS/C3S/datadir/obsdir/ -#drs: -# CMIP6: ETHZ -# CMIP5: ETHZ -# CMIP3: ETHZ - -# Site-specific entries: IPSL -# Uncomment the lines below to locate data on Ciclad at IPSL. -#rootpath: -# IPSLCM: / -# CMIP5: /bdd/CMIP5/output -# CMIP6: /bdd/CMIP6 -# CMIP3: /bdd/CMIP3 -# CORDEX: /bdd/CORDEX/output -# obs4MIPs: /bdd/obs4MIPS/obs-CFMIP/observations -# ana4mips: /not_yet -# OBS: /not_yet -# OBS6: /not_yet -# RAWOBS: /not_yet -#drs: -# CMIP6: DKRZ -# CMIP5: DKRZ -# CMIP3: IPSL -# CORDEX: BADC -# obs4MIPs: IPSL -# ana4mips: default -# OBS: not_yet -# OBS6: not_yet - -# Site-specific entries: Met Office -# Uncomment the lines below to locate data at the Met Office. -#rootpath: -# CMIP5: /project/champ/data/cmip5/output1 -# CMIP6: /project/champ/data/CMIP6 -# CORDEX: /project/champ/data/cordex/output -# OBS: /data/users/esmval/ESMValTool/obs -# OBS6: /data/users/esmval/ESMValTool/obs -# obs4MIPs: /data/users/esmval/ESMValTool/obs -# ana4mips: /project/champ/data/ana4MIPs -# native6: /data/users/esmval/ESMValTool/rawobs -# RAWOBS: /data/users/esmval/ESMValTool/rawobs -#drs: -# CMIP5: BADC -# CMIP6: BADC -# CORDEX: BADC -# OBS: default -# OBS6: default -# obs4MIPs: default -# ana4mips: BADC -# native6: default - -# Site-specific entries: NCI -# Uncomment the lines below to locate data at NCI. -#rootpath: -# CMIP6: [/g/data/oi10/replicas/CMIP6, /g/data/fs38/publications/CMIP6, /g/data/xp65/public/apps/esmvaltool/replicas/CMIP6] -# CMIP5: [/g/data/r87/DRSv3/CMIP5, /g/data/al33/replicas/CMIP5/combined, /g/data/rr3/publications/CMIP5/output1, /g/data/xp65/public/apps/esmvaltool/replicas/cmip5/output1] -# CMIP3: /g/data/r87/DRSv3/CMIP3 -# OBS: /g/data/ct11/access-nri/replicas/esmvaltool/obsdata-v2 -# OBS6: /g/data/ct11/access-nri/replicas/esmvaltool/obsdata-v2 -# obs4MIPs: /g/data/ct11/access-nri/replicas/esmvaltool/obsdata-v2 -# ana4mips: /g/data/ct11/access-nri/replicas/esmvaltool/obsdata-v2 -# native6: /g/data/xp65/public/apps/esmvaltool/native6 -# -#drs: -# CMIP6: NCI -# CMIP5: NCI -# CMIP3: NCI -# CORDEX: ESGF -# obs4MIPs: default -# ana4mips: default diff --git a/doc/sphinx/source/community/dataset.rst b/doc/sphinx/source/community/dataset.rst index 424d4d4694..7a24e7c923 100644 --- a/doc/sphinx/source/community/dataset.rst +++ b/doc/sphinx/source/community/dataset.rst @@ -42,14 +42,15 @@ and run the recipe, to make sure the CMOR checks pass without warnings or errors To test a pull request for a new CMORizer script: -#. Download the data following the instructions included in the script and place - it in the ``RAWOBS`` path specified in your ``config-user.yml`` +#. Download the data following the instructions included in the script and + place it in the ``RAWOBS`` ``rootpath`` specified in your + :ref:`configuration ` #. If available, use the downloading script by running ``esmvaltool data download --config_file `` #. Run the cmorization by running ``esmvaltool data format `` #. Copy the resulting data to the ``OBS`` (for CMIP5 compliant data) or ``OBS6`` - (for CMIP6 compliant data) path specified in your - ``config-user.yml`` + (for CMIP6 compliant data) ``rootpath`` specified in your + :ref:`configuration ` #. Run ``recipes/examples/recipe_check_obs.yml`` with the new dataset to check that the data can be used diff --git a/doc/sphinx/source/community/diagnostic.rst b/doc/sphinx/source/community/diagnostic.rst index 285815f7cf..1be820f7b8 100644 --- a/doc/sphinx/source/community/diagnostic.rst +++ b/doc/sphinx/source/community/diagnostic.rst @@ -64,7 +64,7 @@ If it is just a few simple scripts or packaging is not possible (i.e. for NCL) y and paste the source code into the ``esmvaltool/diag_scripts`` directory. If you have existing code in a compiled language like -C, C++, or Fortran that you want to re-use, the recommended way to proceed is to add Python bindings and publish +C, C++, or Fortran that you want to reuse, the recommended way to proceed is to add Python bindings and publish the package on PyPI so it can be installed as a Python dependency. You can then call the functions it provides using a Python diagnostic. @@ -134,9 +134,8 @@ Diagnostic output Typically, diagnostic scripts create plots, but any other output such as e.g. text files or tables is also possible. Figures should be saved in the ``plot_dir``, either in both ``.pdf`` and -``.png`` format (preferred), or -respect the ``output_file_type`` specified in the -:ref:`esmvalcore:user configuration file`. +``.png`` format (preferred), or respect the :ref:`configuration option +` ``output_file_type`` . Data should be saved in the ``work_dir``, preferably as a ``.nc`` (`NetCDF `__) file, following the `CF-Conventions `__ as much as possible. @@ -181,7 +180,7 @@ human inspection. In addition to provenance information, a caption is also added to the plots. Provenance information from the recipe is automatically recorded by ESMValCore, whereas -diagnostic scripts must include code specifically to record provenance. See below for +diagnostic scripts must include code specifically to record provenance. See below for documentation of provenance attributes that can be included in a recipe. When contributing a diagnostic, please make sure it records the provenance, and that no warnings related to provenance are generated when running the recipe. @@ -252,7 +251,7 @@ for example plot_types: errorbar: error bar plot -To use these items, include them in the provenance record dictionary in the form +To use these items, include them in the provenance record dictionary in the form :code:`key: [value]` i.e. for the example above as :code:`'plot_types': ['errorbar']`. @@ -275,8 +274,8 @@ Always use :func:`esmvaltool.diag_scripts.shared.run_diagnostic` at the end of y with run_diagnostic() as config: main(config) -Create a ``provenance_record`` for each diagnostic file (i.e. image or data -file) that the diagnostic script outputs. The ``provenance_record`` is a +Create a ``provenance_record`` for each diagnostic file (i.e. image or data +file) that the diagnostic script outputs. The ``provenance_record`` is a dictionary of provenance items, for example: .. code-block:: python @@ -296,15 +295,15 @@ dictionary of provenance items, for example: 'statistics': ['mean'], } -To save a matplotlib figure, use the convenience function -:func:`esmvaltool.diag_scripts.shared.save_figure`. Similarly, to save Iris cubes use +To save a matplotlib figure, use the convenience function +:func:`esmvaltool.diag_scripts.shared.save_figure`. Similarly, to save Iris cubes use :func:`esmvaltool.diag_scripts.shared.save_data`. Both of these functions take ``provenance_record`` as an argument and log the provenance accordingly. Have a look at the example Python diagnostic in `esmvaltool/diag_scripts/examples/diagnostic.py `_ for a complete example. -For any other files created, you will need to make use of a +For any other files created, you will need to make use of a :class:`esmvaltool.diag_scripts.shared.ProvenanceLogger` to log provenance. Include the following code directly after the file is saved: @@ -489,7 +488,7 @@ This includes the following items: * In-code documentation (comments, docstrings) * Code quality (e.g. no hardcoded pathnames) * No Codacy errors reported -* Re-use of existing functions whenever possible +* Reuse of existing functions whenever possible * Provenance implemented Run recipe diff --git a/doc/sphinx/source/community/release_strategy/detailed_release_procedure.rst b/doc/sphinx/source/community/release_strategy/detailed_release_procedure.rst index a73643f454..d0d7f74672 100644 --- a/doc/sphinx/source/community/release_strategy/detailed_release_procedure.rst +++ b/doc/sphinx/source/community/release_strategy/detailed_release_procedure.rst @@ -49,7 +49,7 @@ and attach it in the release testing issue; to record the environment in a yaml Modifications to configuration files need to be documented as well. To test recipes, it is recommended to only use the default options and DKRZ data directories, simply by uncommenting -the DKRZ-Levante block of a newly generated ``config-user.yml`` file. +the DKRZ-Levante block of a :ref:`newly generated configuration file `. Submit run scripts - test recipe runs ------------------------------------- @@ -61,7 +61,7 @@ You will have to set the name of your environment, your email address (if you wa More information on running jobs with SLURM on DKRZ/Levante can be found in the DKRZ `documentation `_. -You can also specify the path to your ``config-user.yml`` file where ``max_parallel_tasks`` can be set. The script was found to work well with ``max_parallel_tasks=8``. Some recipes need to be run with ``max_parallel_tasks=1`` (large memory requirements, CMIP3 data, diagnostic issues, ...). These recipes are listed in `ONE_TASK_RECIPES`. +You can also specify the path to your configuration directory where ``max_parallel_tasks`` can be set in a YAML file. The script was found to work well with ``max_parallel_tasks=8``. Some recipes need to be run with ``max_parallel_tasks=1`` (large memory requirements, CMIP3 data, diagnostic issues, ...). These recipes are listed in `ONE_TASK_RECIPES`. Some recipes need other job requirements, you can add their headers in the `SPECIAL_RECIPES` dictionary. Otherwise the header will be written following the template that is written in the lines below. If you want to exclude recipes, you can do so by uncommenting the `exclude` lines. diff --git a/doc/sphinx/source/community/upgrading.rst b/doc/sphinx/source/community/upgrading.rst index 9ed7f8b5b1..9a9b37f178 100644 --- a/doc/sphinx/source/community/upgrading.rst +++ b/doc/sphinx/source/community/upgrading.rst @@ -145,7 +145,7 @@ Many operations previously performed by the diagnostic scripts, are now included The backend operations are fully controlled by the ``preprocessors`` section in the recipe. Here, a number of preprocessor sets can be defined, with different options for each of the operations. The sets defined in this section are applied in the ``diagnostics`` section to preprocess a given variable. -It is recommended to proceed step by step, porting and testing each operation separately before proceeding with the next one. A useful setting in the user configuration file (``config-private.yml``) called ``write_intermediary_cube`` allows writing out the variable field after each preprocessing step, thus facilitating the comparison with the old version (e.g., after CMORization, level selection, after regridding, etc.). The CMORization step of the new backend exactly corresponds to the operation performed by the old backend (and stored in the ``climo`` directory, now called ``preprec``): this is the very first step to be checked, by simply comparing the intermediary file produced by the new backend after CMORization with the output of the old backend in the ``climo`` directorsy (see "Testing" below for instructions). +It is recommended to proceed step by step, porting and testing each operation separately before proceeding with the next one. A useful setting in the configuration called ``write_intermediary_cube`` allows writing out the variable field after each preprocessing step, thus facilitating the comparison with the old version (e.g., after CMORization, level selection, after regridding, etc.). The CMORization step of the new backend exactly corresponds to the operation performed by the old backend (and stored in the ``climo`` directory, now called ``preprec``): this is the very first step to be checked, by simply comparing the intermediary file produced by the new backend after CMORization with the output of the old backend in the ``climo`` directorsy (see "Testing" below for instructions). The new backend also performs variable derivation, replacing the ``calculate`` function in the ``variable_defs`` scripts. If the recipe which is being ported makes use of derived variables, the corresponding calculation must be ported from the ``./variable_defs/.ncl`` file to ``./esmvaltool/preprocessor/_derive.py``. @@ -159,7 +159,7 @@ In the new version, all settings are centralized in the recipe, completely repla Make sure the diagnostic script writes NetCDF output ====================================================== -Each diagnostic script is required to write the output of the anaylsis in one or more NetCDF files. This is to give the user the possibility to further look into the results, besides the plots, but (most importantly) for tagging purposes when publishing the data in a report and/or on a website. +Each diagnostic script is required to write the output of the analysis in one or more NetCDF files. This is to give the user the possibility to further look into the results, besides the plots, but (most importantly) for tagging purposes when publishing the data in a report and/or on a website. For each of the plot produced by the diagnostic script a single NetCDF file has to be generated. The variable saved in this file should also contain all the necessary metadata that documents the plot (dataset names, units, statistical methods, etc.). The files have to be saved in the work directory (defined in `cfg['work_dir']` and `config_user_info@work_dir`, for the python and NCL diagnostics, respectively). @@ -209,7 +209,7 @@ Before submitting a pull request, the code should be cleaned to adhere to the co Update the documentation ======================== -If necessary, add or update the documentation for your recipes in the corrsponding rst file, which is now in ``doc\sphinx\source\recipes``. Do not forget to also add the documentation file to the list in ``doc\sphinx\source\annex_c`` to make sure it actually appears in the documentation. +If necessary, add or update the documentation for your recipes in the corresponding rst file, which is now in ``doc\sphinx\source\recipes``. Do not forget to also add the documentation file to the list in ``doc\sphinx\source\annex_c`` to make sure it actually appears in the documentation. Open a pull request =================== diff --git a/doc/sphinx/source/develop/dataset.rst b/doc/sphinx/source/develop/dataset.rst index f3c168a17c..f624a44feb 100644 --- a/doc/sphinx/source/develop/dataset.rst +++ b/doc/sphinx/source/develop/dataset.rst @@ -76,7 +76,7 @@ for downloading (e.g. providing contact information, licence agreements) and using the observations. The unformatted (raw) observations should then be stored in the appropriate of these three folders. -For each additional dataset, an entry needs to be made to the file +For each additional dataset, an entry needs to be made to the file `datasets.yml `_. The dataset entry should contain: @@ -92,10 +92,10 @@ of the cmorizing script (see Section `4. Create a cmorizer for the dataset`_). 3.1 Downloader script (optional) -------------------------------- -A Python script can be written to download raw observations +A Python script can be written to download raw observations from source and store the data in the appropriate tier subdirectory of the folder ``RAWOBS`` automatically. -There are many downloading scripts available in +There are many downloading scripts available in `/esmvaltool/cmorizers/data/downloaders/datasets/ `_ where several data download mechanisms are provided: @@ -108,18 +108,18 @@ Note that the name of this downloading script has to be identical to the name of the dataset. Depending on the source server, the downloading script needs to contain paths to -raw observations, filename patterns and various necessary fields to retrieve +raw observations, filename patterns and various necessary fields to retrieve the data. -Default ``start_date`` and ``end_date`` can be provided in cases where raw data +Default ``start_date`` and ``end_date`` can be provided in cases where raw data are stored in daily, monthly, and yearly files. The downloading script for the given dataset can be run with: .. code-block:: console - esmvaltool data download --config_file + esmvaltool data download --config_dir -The options ``--start`` and ``--end`` can be added to the command above to +The options ``--start`` and ``--end`` can be added to the command above to restrict the download of raw data to a time range. They will be ignored if a specific dataset does not support it (i.e. because it is provided as a single file). Valid formats are ``YYYY``, ``YYYYMM`` and ``YYYYMMDD``. By default, already downloaded data are not overwritten @@ -128,7 +128,7 @@ unless the option ``--overwrite=True`` is used. 4. Create a cmorizer for the dataset ==================================== -There are many cmorizing scripts available in +There are many cmorizing scripts available in `/esmvaltool/cmorizers/data/formatters/datasets/ `_ where solutions to many kinds of format issues with observational data are @@ -158,7 +158,7 @@ configuration file: `MTE.yml `_ in the directory ``ESMValTool/esmvaltool/cmorizers/data/cmor_config/``. Note that both the name of this configuration file and the cmorizing script have to be -identical to the name of your dataset. +identical to the name of your dataset. It is recommended that you set ``project`` to ``OBS6`` in the configuration file. That way, the variables defined in the CMIP6 CMOR table, augmented with the custom variables described above, are available to your script. @@ -188,7 +188,8 @@ The main body of the CMORizer script must contain a function called with this exact call signature. Here, ``in_dir`` corresponds to the input directory of the raw files, ``out_dir`` to the output directory of final reformatted data set, ``cfg`` to the dataset-specific configuration file, -``cfg_user`` to the user configuration file, ``start_date`` to the start +``cfg_user`` to the configuration object (which behaves basically like a +dictionary), ``start_date`` to the start of the period to format, and ``end_date`` to the end of the period to format. If not needed, the last three arguments can be ignored using underscores. The return value of this function is ignored. All @@ -256,9 +257,9 @@ The cmorizing script for the given dataset can be run with: .. code-block:: console - esmvaltool data format --config_file + esmvaltool data format --config_dir -The options ``--start`` and ``--end`` can be added to the command above to +The options ``--start`` and ``--end`` can be added to the command above to restrict the formatting of raw data to a time range. They will be ignored if a specific dataset does not support it (i.e. because it is provided as a single file). Valid formats are ``YYYY``, ``YYYYMM`` and ``YYYYMMDD``. @@ -267,12 +268,12 @@ does not support it (i.e. because it is provided as a single file). Valid format The output path given in the configuration file is the path where your cmorized dataset will be stored. The ESMValTool will create a folder - with the correct tier information + with the correct tier information (see Section `2. Edit your configuration file`_) if that tier folder is not - already available, and then a folder named after the dataset. + already available, and then a folder named after the dataset. In this folder the cmorized data set will be stored as a NetCDF file. The cmorized dataset will be automatically moved to the correct tier - subfolder of your OBS or OBS6 directory if the option + subfolder of your OBS or OBS6 directory if the option ``--install=True`` is used in the command above and no such directory was already created. @@ -284,9 +285,9 @@ the cmorizing scripts can be run in a single command with: .. code-block:: console - esmvaltool data prepare --config_file + esmvaltool data prepare --config_dir -Note that options from the ```esmvaltool data download`` and +Note that options from the ```esmvaltool data download`` and ``esmvaltool data format`` commands can be passed to the above command. 6. Naming convention of the observational data files diff --git a/doc/sphinx/source/faq.rst b/doc/sphinx/source/faq.rst index 10c72bd2cb..43251a801b 100644 --- a/doc/sphinx/source/faq.rst +++ b/doc/sphinx/source/faq.rst @@ -59,12 +59,17 @@ This is a useful functionality because it allows the user to `fix` things on-the quitting the Ipython console, code execution continues as per normal. -Use multiple config-user.yml files -================================== +Using multiple configuration directories +======================================== + +By default, ESMValTool will read YAML configuration files from the user +configuration directory ``~/.config/esmvaltool``, which can be changed with the +``ESMVALTOOL_CONFIG_DIR`` environment variable. +If required, users can specify the command line option ``--config_dir`` to +select another configuration directory, which is read **in addition** to the +user configuration directory +See the section on configuration :ref:`config_yaml_files` for details on this. -The user selects the configuration yaml file at run time. It's possible to -have several configurations files. For instance, it may be practical to have one -config file for debugging runs and another for production runs. Create a symbolic link to the latest output directory ===================================================== diff --git a/doc/sphinx/source/functionalities.rst b/doc/sphinx/source/functionalities.rst index 5b49c118a2..0098d95ded 100644 --- a/doc/sphinx/source/functionalities.rst +++ b/doc/sphinx/source/functionalities.rst @@ -12,9 +12,9 @@ that it can: - execute the workflow; and - output the desired collective data and media. -To facilitate these four steps, the user has control over the tool via -two main input files: the :ref:`user configuration file ` -and the :ref:`recipe `. The configuration file sets +To facilitate these four steps, the user has control over the tool via the +:ref:`configuration ` and the :ref:`recipe +`. The configuration sets user and site-specific parameters (like input and output paths, desired output graphical formats, logging level, etc.), whereas the recipe file sets data, preprocessing and diagnostic-specific parameters (data @@ -27,7 +27,7 @@ recyclable; the recipe file can be used for a large number of applications, since it may include as many datasets, preprocessors and diagnostics sections as the user deems useful. -Once the user configuration files and the recipe are at hand, the user +Once the configuration files and the recipe are at hand, the user can start the tool. A schematic overview of the ESMValTool workflow is depicted in the figure below. diff --git a/doc/sphinx/source/input.rst b/doc/sphinx/source/input.rst index 65aef57cd8..f972789315 100644 --- a/doc/sphinx/source/input.rst +++ b/doc/sphinx/source/input.rst @@ -76,7 +76,7 @@ For example, run to run the default example recipe and automatically download the required data to the directory ``~/climate_data``. -The data only needs to be downloaded once, every following run will re-use +The data only needs to be downloaded once, every following run will reuse previously downloaded data stored in this directory. See :ref:`esmvalcore:config-esgf` for a more in depth explanation and the available configuration options. @@ -117,7 +117,7 @@ OBS and OBS6 data is stored in the `esmeval` Group Workspace (GWS), and to be gr GWS, one must apply at https://accounts.jasmin.ac.uk/services/group_workspaces/esmeval/ ; after permission has been granted, the user is encouraged to use the data locally, and not move it elsewhere, to minimize both data transfers and stale disk usage; to note that Tier 3 data is subject to data protection restrictions; for further inquiries, -the GWS is adminstered by [Valeriu Predoi](mailto:valeriu.predoi@ncas.ac.uk). +the GWS is administered by [Valeriu Predoi](mailto:valeriu.predoi@ncas.ac.uk). Using a CMORizer script ----------------------- @@ -193,8 +193,8 @@ To CMORize one or more datasets, run: esmvaltool data format --config_file [CONFIG_FILE] [DATASET_LIST] -The path to the raw data to be CMORized must be specified in the :ref:`user -configuration file` as RAWOBS. +The ``rootpath`` to the raw data to be CMORized must be specified in the +:ref:`configuration ` as ``RAWOBS``. Within this path, the data are expected to be organized in subdirectories corresponding to the data tier: Tier2 for freely-available datasets (other than obs4MIPs and ana4mips) and Tier3 for restricted datasets (i.e., dataset which @@ -492,8 +492,8 @@ A list of all currently supported native datasets is :ref:`provided here A detailed description of how to include new native datasets is given :ref:`here `. -To use this functionality, users need to provide a path in the -:ref:`esmvalcore:user configuration file` for the ``native6`` project data +To use this functionality, users need to provide a ``rootpath`` in the +:ref:`configuration ` for the ``native6`` project data and/or the dedicated project used for the native dataset, e.g., ``ICON``. Then, in the recipe, they can refer to those projects. For example: diff --git a/doc/sphinx/source/quickstart/configuration.rst b/doc/sphinx/source/quickstart/configuration.rst index 34c29aac5c..9cea6413b6 100644 --- a/doc/sphinx/source/quickstart/configuration.rst +++ b/doc/sphinx/source/quickstart/configuration.rst @@ -1,4 +1,4 @@ -.. _config-user: +.. _config: ************* Configuration @@ -7,22 +7,23 @@ Configuration The ``esmvaltool`` command is provided by the ESMValCore package, the documentation on configuring ESMValCore can be found :ref:`here `. -In particular, it is recommended to read the section on the -:ref:`User configuration file ` -and the section on +An overview of all configuration options can be found +:ref:`here `. +In particular, it is recommended to read the section on how to :ref:`specify +configuration options ` and the section on :ref:`Finding data `. -To install the default configuration file in the default location, run +To install the default configuration in the default location, run .. code:: bash esmvaltool config get_config_user -Note that this file needs to be customized using the instructions above, so +Note that this needs to be customized using the instructions above, so the ``esmvaltool`` command can find the data on your system, before it can run a recipe. There is a lesson available in the `ESMValTool tutorial `_ -that describes how to personalize the configuration file. It can be found +that describes how to personalize the configuration. It can be found `at this site `_. diff --git a/doc/sphinx/source/quickstart/output.rst b/doc/sphinx/source/quickstart/output.rst index 4a33e8ca42..33836f1c9a 100644 --- a/doc/sphinx/source/quickstart/output.rst +++ b/doc/sphinx/source/quickstart/output.rst @@ -5,8 +5,9 @@ Output ****** ESMValTool automatically generates a new output directory with every run. The -location is determined by the output_dir option in the config-user.yml file, -the recipe name, and the date and time, using the the format: YYYYMMDD_HHMMSS. +location is determined by the :ref:`configuration option +` ``output_dir``, the recipe name, and the date and +time, using the the format: YYYYMMDD_HHMMSS. For instance, a typical output location would be: output_directory/recipe_ocean_amoc_20190118_1027/ @@ -33,13 +34,15 @@ The preprocessed datasets will be stored to the preproc/ directory. Each variable in each diagnostic will have its own the `metadata.yml`_ interface files saved in the preproc directory. -If the option ``save_intermediary_cubes`` is set to ``true`` in the -config-user.yml file, then the intermediary cubes will also be saved here. -This option is set to false in the default ``config-user.yml`` file. +If the :ref:`configuration option ` +``save_intermediary_cubes`` is set to ``true`` , then the intermediary cubes +will also be saved here. +This option is set to ``false`` by default. -If the option ``remove_preproc_dir`` is set to ``true`` in the config-user.yml -file, then the preproc directory will be deleted after the run completes. This -option is set to true in the default ``config-user.yml`` file. +If the :ref:`configuration option ` +``remove_preproc_dir`` is set to ``true`` , then the preproc directory will be +deleted after the run completes. +This option is set to ``true`` by default. Run @@ -70,8 +73,8 @@ Plots ===== The plots directory is where diagnostics save their output figures. These -plots are saved in the format requested by the option `output_file_type` in the -config-user.yml file. +plots are saved in the format requested by the :ref:`configuration option +` ``output_file_type``. Settings.yml @@ -81,10 +84,10 @@ The settings.yml file is automatically generated by ESMValCore. For each diagnos a unique settings.yml file will be produced. The settings.yml file passes several global level keys to diagnostic scripts. -This includes several flags from the config-user.yml file (such as -'write_netcdf', 'write_plots', etc...), several paths which are specific to the -diagnostic being run (such as 'plot_dir' and 'run_dir') and the location on -disk of the metadata.yml file (described below). +This includes several flags from the configuration (such as +``write_netcdf``, ``write_plots``, etc...), several paths which are specific to +the diagnostic being run (such as ``plot_dir`` and ``run_dir``) and the +location on disk of the metadata.yml file (described below). .. code-block:: yaml @@ -147,5 +150,5 @@ As you can see, this is effectively a dictionary with several items including data paths, metadata and other information. There are several tools available in python which are built to read and parse -these files. The tools are avaialbe in the shared directory in the diagnostics +these files. The tools are available in the shared directory in the diagnostics directory. diff --git a/doc/sphinx/source/quickstart/running.rst b/doc/sphinx/source/quickstart/running.rst index 7f9cadbaa1..20cb8620b0 100644 --- a/doc/sphinx/source/quickstart/running.rst +++ b/doc/sphinx/source/quickstart/running.rst @@ -39,20 +39,20 @@ from ESGF to the local directory ``~/climate_data``, run The ``--search_esgf=when_missing`` option tells ESMValTool to search for and download the necessary climate data files, if they cannot be found locally. -The data only needs to be downloaded once, every following run will re-use +The data only needs to be downloaded once, every following run will reuse previously downloaded data. If you have all required data available locally, you can run the tool with ``--search_esgf=never`` argument (the default). Note that in that case the required data should be located in the directories -specified in your user configuration file. +specified in the configuration (see :ref:`esmvalcore:config_option_rootpath`). A third option ``--search_esgf=always`` is available. With this option, the tool will first check the ESGF for the needed data, regardless of any local data availability; if the data found on ESGF is newer than the local data (if any) or the user specifies a version of the data that is available only from the ESGF, then that data will be downloaded; otherwise, local data will be used. -Recall that the chapter :ref:`Configuring ESMValTool ` -provides an explanation of how to create your own config-user.yml file. +Recall that the chapter on :ref:`configuring ESMValTool ` +provides an explanation of how to set up the configuration. See :ref:`running esmvaltool ` in the ESMValCore documentation for a more complete introduction to the ``esmvaltool`` command. diff --git a/doc/sphinx/source/recipes/recipe_carvalhais14nat.rst b/doc/sphinx/source/recipes/recipe_carvalhais14nat.rst index dc26a745e2..b551bbbdc5 100644 --- a/doc/sphinx/source/recipes/recipe_carvalhais14nat.rst +++ b/doc/sphinx/source/recipes/recipe_carvalhais14nat.rst @@ -73,7 +73,7 @@ The settings needed for loading the observational dataset in all diagnostics are provided in the recipe through `obs_info` within `obs_details` section. * ``obs_data_subdir``: subdirectory of auxiliary_data_dir (set in - config-user file) where observation data are stored {e.g., + configuration) where observation data are stored {e.g., data_ESMValTool_Carvalhais2014}. * ``source_label``: source data label {'Carvalhais2014'}. * ``variant_label``: variant of the observation {'BE'} for best estimate. @@ -112,7 +112,7 @@ Script land_carbon_cycle/diag_global_turnover.py * ``y0``: {``float``, 1.0} Y - coordinate of the upper edge of the figure. * ``wp``: {``float``, 1 / number of models} - width of each map. * ``hp``: {``float``, = wp} - height of each map. - * ``xsp``: {``float``, 0} - spacing betweeen maps in X - direction. + * ``xsp``: {``float``, 0} - spacing between maps in X - direction. * ``ysp``: {``float``, -0.03} - spacing between maps in Y -direction. Negative to reduce the spacing below default. * ``aspect_map``: {``float``, 0.5} - aspect of the maps. @@ -217,10 +217,10 @@ Due to inherent dependence of the diagnostic on uncertainty estimates in observation, the data needed for each diagnostic script are processed at different spatial resolutions (as in Carvalhais et al., 2014), and provided in 11 different resolutions (see Table 1). Note that the uncertainties were -estimated at the resolution of the selected models, and, thus, only the -pre-processed observed data can be used with the recipe. -It is not possible to use regridding functionalities of ESMValTool to regrid -the observational data to other spatial resolutions, as the uncertainty +estimated at the resolution of the selected models, and, thus, only the +pre-processed observed data can be used with the recipe. +It is not possible to use regridding functionalities of ESMValTool to regrid +the observational data to other spatial resolutions, as the uncertainty estimates cannot be regridded. Table 1. A summary of the observation datasets at different resolutions. @@ -309,7 +309,7 @@ Example plots Comparison of latitudinal (zonal) variations of pearson correlation between turnover time and climate: turnover time and precipitation, controlled for - temperature (left) and vice-versa (right). Reproduces figures 2c and 2d in + temperature (left) and vice-versa (right). Reproduces figures 2c and 2d in `Carvalhais et al. (2014)`_. .. _fig_carvalhais14nat_2: @@ -320,7 +320,7 @@ Example plots Comparison of observation-based and modelled ecosystem carbon turnover time. Along the diagnonal, tau_ctotal are plotted, above the bias, and below - density plots. The inset text in density plots indicate the correlation. + density plots. The inset text in density plots indicate the correlation. .. _fig_carvalhais14nat_3: @@ -328,11 +328,11 @@ Example plots :align: center :width: 80% - Global distributions of multimodel bias and model agreement. Multimodel bias - is calculated as the ratio of multimodel median turnover time and that from - observation. Stippling indicates the regions where only less than one - quarter of the models fall within the range of observational uncertainties - (`5^{th}` and `95^{th}` percentiles). Reproduces figure 3 in `Carvalhais et + Global distributions of multimodel bias and model agreement. Multimodel bias + is calculated as the ratio of multimodel median turnover time and that from + observation. Stippling indicates the regions where only less than one + quarter of the models fall within the range of observational uncertainties + (`5^{th}` and `95^{th}` percentiles). Reproduces figure 3 in `Carvalhais et al. (2014)`_. .. _fig_carvalhais14nat_4: @@ -341,7 +341,7 @@ Example plots :align: center :width: 80% - Comparison of latitudinal (zonal) variations of observation-based and - modelled ecosystem carbon turnover time. The zonal turnover time is - calculated as the ratio of zonal `ctotal` and `gpp`. Reproduces figures 2a + Comparison of latitudinal (zonal) variations of observation-based and + modelled ecosystem carbon turnover time. The zonal turnover time is + calculated as the ratio of zonal `ctotal` and `gpp`. Reproduces figures 2a and 2b in `Carvalhais et al. (2014)`_. diff --git a/doc/sphinx/source/recipes/recipe_climwip.rst b/doc/sphinx/source/recipes/recipe_climwip.rst index 0928ba939f..900698b85a 100644 --- a/doc/sphinx/source/recipes/recipe_climwip.rst +++ b/doc/sphinx/source/recipes/recipe_climwip.rst @@ -43,9 +43,9 @@ Using shapefiles for cutting scientific regions To use shapefiles for selecting SREX or AR6 regions by name it is necessary to download them, e.g., from the sources below and reference the file using the `shapefile` parameter. This can either be a -absolute or a relative path. In the example recipes they are stored in a subfolder `shapefiles` -in the `auxiliary_data_dir` (with is specified in the -`config-user.yml `_). +absolute or a relative path. In the example recipes they are stored in a subfolder `shapefiles` +in the :ref:`configuration option ` +``auxiliary_data_dir``. SREX regions (AR5 reference regions): http://www.ipcc-data.org/guidelines/pages/ar5_regions.html @@ -249,7 +249,7 @@ Brunner et al. (2020) recipe and example independence weighting The recipe uses an additional step between pre-processor and weight calculation to calculate anomalies relative to the global mean (e.g., tas_ANOM = tas_CLIM - global_mean(tas_CLIM)). This means we do not use the absolute temperatures of a model as performance criterion but rather the horizontal temperature distribution (see `Brunner et al. 2020 `_ for a discussion). -This recipe also implements a somewhat general independence weighting for CMIP6. In contrast to model performance (which should be case specific) model independence can largely be seen as only dependet on the multi-model ensemble in use but not the target variable or region. This means that the configuration used should be valid for similar subsets of CMIP6 as used in this recipe: +This recipe also implements a somewhat general independence weighting for CMIP6. In contrast to model performance (which should be case specific) model independence can largely be seen as only dependent on the multi-model ensemble in use but not the target variable or region. This means that the configuration used should be valid for similar subsets of CMIP6 as used in this recipe: .. code-block:: yaml diff --git a/doc/sphinx/source/recipes/recipe_gier20bg.rst b/doc/sphinx/source/recipes/recipe_gier20bg.rst index bb11770a24..b8f8fb9b8e 100644 --- a/doc/sphinx/source/recipes/recipe_gier20bg.rst +++ b/doc/sphinx/source/recipes/recipe_gier20bg.rst @@ -53,7 +53,7 @@ User settings in recipe * Optional diag_script_info attributes: * ``styleset``: styleset for color coding panels - * ``output_file_type``: output file type for plots, default: config_user -> png + * ``output_file_type``: output file type for plots, default: png * ``var_plotname``: NCL string formatting how variable should be named in plots defaults to short_name if not assigned. @@ -64,7 +64,7 @@ User settings in recipe amplitude contour plot * Optional diag_script_info attributes: - * ``output_file_type``: output file type for plots, default: config_user -> png + * ``output_file_type``: output file type for plots, default: png #. Script xco2_analysis/main.ncl: @@ -77,7 +77,7 @@ User settings in recipe accounting for the ensemble member named in "ensemble_refs" * Optional diag_script_info attributes: - * ``output_file_type``: output file type for plots, default: config_user -> png + * ``output_file_type``: output file type for plots, default: png * ``ensemble_refs``: list of model-ensemble pairs to denote which ensemble member to use for calculating multi-model mean. required if ensemble_mean = true @@ -97,17 +97,17 @@ User settings in recipe * ``plot_var2_mean``: If True adds mean of seasonal cycle to panel as string. * Optional diag_script_info attributes: - * ``output_file_type``: output file type for plots, default: config_user -> png + * ``output_file_type``: output file type for plots, default: png * ``var_plotname``: String formatting how variable should be named in plots defaults to short_name if not assigned #. Script xco2_analysis/sat_masks.ncl: * Optional diag_script_info attributes: - * ``output_file_type``: output file type for plots, default: config_user -> png + * ``output_file_type``: output file type for plots, default: png * ``var_plotname``: String formatting how variable should be named in plots defaults to short_name if not assigned - * ``c3s_plots``: Missing value plots seperated by timeseries of c3s satellites + * ``c3s_plots``: Missing value plots separated by timeseries of c3s satellites #. Script xco2_analysis/station_comparison.ncl: @@ -116,7 +116,7 @@ User settings in recipe first, then 2D variable, followed by surface stations * Optional diag_script_info attributes: - * ``output_file_type``: output file type for plots, default: config_user -> png + * ``output_file_type``: output file type for plots, default: png * ``var_plotnames``: String formatting how variables should be named in plots defaults to short_name if not assigned * ``overwrite_altitudes``: Give other altitude values than the ones attached in diff --git a/doc/sphinx/source/recipes/recipe_hydrology.rst b/doc/sphinx/source/recipes/recipe_hydrology.rst index d0e2e0bcb3..995a70b3ae 100644 --- a/doc/sphinx/source/recipes/recipe_hydrology.rst +++ b/doc/sphinx/source/recipes/recipe_hydrology.rst @@ -62,13 +62,13 @@ Diagnostics are stored in esmvaltool/diag_scripts/hydrology * wflow.py * lisflood.py * hype.py - * globwat.py + * globwat.py User settings in recipe ----------------------- -All hydrological recipes require a shapefile as an input to produce forcing data. This shapefile determines the shape of the basin for which the data will be cut out and processed. All recipes are tested with `the shapefiles `_ that are used for the eWaterCycle project. In principle any shapefile can be used, for example, the freely available basin shapefiles from the `HydroSHEDS project `_. +All hydrological recipes require a shapefile as an input to produce forcing data. This shapefile determines the shape of the basin for which the data will be cut out and processed. All recipes are tested with `the shapefiles `_ that are used for the eWaterCycle project. In principle any shapefile can be used, for example, the freely available basin shapefiles from the `HydroSHEDS project `_. #. recipe_pcrglobwb.yml @@ -87,7 +87,7 @@ All hydrological recipes require a shapefile as an input to produce forcing data *extract_shape:* - * shapefile: Meuse.shp (MARRMoT is a hydrological Lumped model that needs catchment-aggregated forcing data. The catchment is provided as a shapefile, the path can be relative to ``auxiliary_data_dir`` as defined in config-user.yml.). + * shapefile: Meuse.shp (MARRMoT is a hydrological Lumped model that needs catchment-aggregated forcing data. The catchment is provided as a shapefile, the path can be relative to :ref:`configuration option ` ``auxiliary_data_dir``). * method: contains * crop: true @@ -107,7 +107,7 @@ All hydrological recipes require a shapefile as an input to produce forcing data * dem_file: netcdf file containing a digital elevation model with elevation in meters and coordinates latitude and longitude. A wflow example dataset is available at: https://github.com/openstreams/wflow/tree/master/examples/wflow_rhine_sbm - The example dem_file can be obtained from https://github.com/openstreams/wflow/blob/master/examples/wflow_rhine_sbm/staticmaps/wflow_dem.map + The example dem_file can be obtained from https://github.com/openstreams/wflow/blob/master/examples/wflow_rhine_sbm/staticmaps/wflow_dem.map * regrid: the regridding scheme for regridding to the digital elevation model. Choose ``area_weighted`` (slow) or ``linear``. #. recipe_lisflood.yml diff --git a/doc/sphinx/source/recipes/recipe_ipccwg1ar6ch3.rst b/doc/sphinx/source/recipes/recipe_ipccwg1ar6ch3.rst index 42bedcec09..718c345b19 100644 --- a/doc/sphinx/source/recipes/recipe_ipccwg1ar6ch3.rst +++ b/doc/sphinx/source/recipes/recipe_ipccwg1ar6ch3.rst @@ -6,7 +6,7 @@ IPCC AR6 Chapter 3 (selected figures) Overview -------- -This recipe collects selected diagnostics used in IPCC AR6 WGI Chapter 3: +This recipe collects selected diagnostics used in IPCC AR6 WGI Chapter 3: Human influence on the climate system (`Eyring et al., 2021`_). Plots from IPCC AR6 can be readily reproduced and compared to previous versions. The aim is to be able to start with what was available now the next time allowing us to focus @@ -15,7 +15,8 @@ on developing more innovative analysis methods rather than constantly having to Processing of CMIP3 models currently works only in serial mode, due to an issue in the input data still under investigation. To run the recipe for Fig 3.42a -and Fig. 3.43 set "max_parallel_tasks: 1" in the config-user.yml file. +and Fig. 3.43 set the :ref:`configuration option ` +``max_parallel_tasks: 1``. The plots are produced collecting the diagnostics from individual recipes. The following figures from `Eyring et al. (2021)`_ can currently be reproduced: @@ -43,10 +44,9 @@ To reproduce Fig. 3.9 you need the shapefile of the `AR6 reference regions (`Iturbide et al., 2020 `_). Please download the file `IPCC-WGI-reference-regions-v4_shapefile.zip `_, -unzip and store it in `/IPCC-regions/` (the `auxiliary_data_dir` -is defined in the `config-user.yml -`_ -file). +unzip and store it in `/IPCC-regions/` (where +``auxiliary_data_dir`` is given as :ref:`configuration option +`). .. _`Eyring et al., 2021`: https://www.ipcc.ch/report/ar6/wg1/chapter/chapter-3/ .. _`Eyring et al. (2021)`: https://www.ipcc.ch/report/ar6/wg1/chapter/chapter-3/ @@ -179,7 +179,7 @@ User settings in recipe * start_year: start year in figure * end_year: end year in figure - * panels: list of variable blocks for each panel + * panels: list of variable blocks for each panel *Optional settings for script* @@ -205,7 +205,7 @@ User settings in recipe * plot_units: variable unit for plotting * y-min: set min of y-axis * y-max: set max of y-axis - * order: order in which experiments should be plotted + * order: order in which experiments should be plotted * stat_shading: if true: shading of statistic range * ref_shading: if true: shading of reference period @@ -225,7 +225,7 @@ User settings in recipe * plot_legend: if true, plot legend will be plotted * plot_units: variable unit for plotting - * multi_model_mean: if true, multi-model mean and uncertaintiy will be + * multi_model_mean: if true, multi-model mean and uncertainty will be plotted *Optional settings for variables* @@ -304,7 +304,7 @@ User settings in recipe * labels: List of labels for each variable on the x-axis * model_spread: if True, model spread is shaded * plot_median: if True, median is plotted - * project_order: give order of projects + * project_order: give order of projects Variables @@ -452,7 +452,7 @@ Example plots 2013). For line colours see the legend of Figure 3.4. Additionally, the multi-model mean (red) and standard deviation (grey shading) are shown. Observational and model datasets were detrended by removing the - least-squares quadratic trend. + least-squares quadratic trend. .. figure:: /recipes/figures/ipccwg1ar6ch3/tas_anom_damip_global_1850-2020.png :align: center @@ -467,7 +467,7 @@ Example plots anomalies are shown relative to 1950-2010 for Antarctica and relative to 1850-1900 for other continents. CMIP6 historical simulations are expanded by the SSP2-4.5 scenario simulations. All available ensemble members were used. - Regions are defined by Iturbide et al. (2020). + Regions are defined by Iturbide et al. (2020). .. figure:: /recipes/figures/ipccwg1ar6ch3/model_bias_pr_annualclim_CMIP6.png :align: center @@ -487,7 +487,7 @@ Example plots show a change greater than the variability threshold; crossed lines indicate regions with conflicting signal, where >=66% of models show change greater than the variability threshold and <80% of all models agree on the sign of - change. + change. .. figure:: /recipes/figures/ipccwg1ar6ch3/precip_anom_1950-2014.png :align: center @@ -511,7 +511,7 @@ Example plots forcings (brown) and natural forcings only (blue). Observed trends for each observational product are shown as horizontal lines. Panel (b) shows annual mean precipitation rate (mm day-1) of GHCN version 2 for the years 1950-2014 - over land areas used to compute the plots. + over land areas used to compute the plots. .. figure:: /recipes/figures/ipccwg1ar6ch3/zonal_westerly_winds.png :align: center diff --git a/doc/sphinx/source/recipes/recipe_kcs.rst b/doc/sphinx/source/recipes/recipe_kcs.rst index fa07f0a167..1ed117ecb6 100644 --- a/doc/sphinx/source/recipes/recipe_kcs.rst +++ b/doc/sphinx/source/recipes/recipe_kcs.rst @@ -30,7 +30,7 @@ In the second diagnostic, for both the control and future periods, the N target 2. Further constrain the selection by picking samples that represent either high or low changes in summer precipitation and summer and winter temperature, by limiting the remaining samples to certain percentile ranges: relatively wet/cold in the control and dry/warm in the future, or vice versa. The percentile ranges are listed in table 1 of Lenderink 2014's supplement. This should result is approximately 50 remaining samples for each scenario, for both control and future. 3. Use a Monte-Carlo method to make a final selection of 8 resamples with minimal reuse of the same ensemble member/segment. -Datasets have been split in two parts: the CMIP datasets and the target model datasets. An example use case for this recipe is to compare between CMIP5 and CMIP6, for example. The recipe can work with a target model that is not part of CMIP, provided that the data are CMOR compatible, and using the same data referece syntax as the CMIP data. Note that you can specify :ref:`multiple data paths` in the user configuration file. +Datasets have been split in two parts: the CMIP datasets and the target model datasets. An example use case for this recipe is to compare between CMIP5 and CMIP6, for example. The recipe can work with a target model that is not part of CMIP, provided that the data are CMOR compatible, and using the same data reference syntax as the CMIP data. Note that you can specify :ref:`multiple data paths` in the configuration. Available recipes and diagnostics @@ -128,7 +128,7 @@ AND highlighting the selected steering parameters and resampling periods: .. figure:: /recipes/figures/kcs/global_matching.png :align: center -The diagnostic ``local_resampling`` procudes a number of output files: +The diagnostic ``local_resampling`` produces a number of output files: * ``season_means_.nc``: intermediate results, containing the season means for each segment of the original target model ensemble. * ``top1000_.csv``: intermediate results, containing the 1000 combinations that have been selected based on winter mean precipitation. diff --git a/doc/sphinx/source/recipes/recipe_model_evaluation.rst b/doc/sphinx/source/recipes/recipe_model_evaluation.rst index 9e199815e0..c61f34aa62 100644 --- a/doc/sphinx/source/recipes/recipe_model_evaluation.rst +++ b/doc/sphinx/source/recipes/recipe_model_evaluation.rst @@ -35,9 +35,9 @@ User settings ------------- It is recommended to use a vector graphic file type (e.g., SVG) for the output -format when running this recipe, i.e., run the recipe with the command line -option ``--output_file_type=svg`` or use ``output_file_type: svg`` in your -:ref:`esmvalcore:user configuration file`. +format when running this recipe, i.e., run the recipe with the +:ref:`configuration options ` ``output_file_type: +svg``. Note that map and profile plots are rasterized by default. Use ``rasterize: false`` in the recipe to disable this. diff --git a/doc/sphinx/source/recipes/recipe_monitor.rst b/doc/sphinx/source/recipes/recipe_monitor.rst index ee3b9b44fa..8f4893fc12 100644 --- a/doc/sphinx/source/recipes/recipe_monitor.rst +++ b/doc/sphinx/source/recipes/recipe_monitor.rst @@ -36,9 +36,9 @@ User settings ------------- It is recommended to use a vector graphic file type (e.g., SVG) for the output -files when running this recipe, i.e., run the recipe with the command line -option ``--output_file_type=svg`` or use ``output_file_type: svg`` in your -:ref:`esmvalcore:user configuration file`. +format when running this recipe, i.e., run the recipe with the +:ref:`configuration options ` ``output_file_type: +svg``. Note that map and profile plots are rasterized by default. Use ``rasterize_maps: false`` or ``rasterize: false`` (see `Recipe settings`_) in the recipe to disable this. diff --git a/doc/sphinx/source/recipes/recipe_oceans.rst b/doc/sphinx/source/recipes/recipe_oceans.rst index d8bf3143e1..17552b39fa 100644 --- a/doc/sphinx/source/recipes/recipe_oceans.rst +++ b/doc/sphinx/source/recipes/recipe_oceans.rst @@ -458,7 +458,7 @@ and a latitude and longitude coordinates. This diagnostic also includes the optional arguments, `maps_range` and `diff_range` to manually define plot ranges. Both arguments are a list of two floats -to set plot range minimun and maximum values respectively for Model and Observations +to set plot range minimum and maximum values respectively for Model and Observations maps (Top panels) and for the Model minus Observations panel (bottom left). Note that if input data have negative values the Model over Observations map (bottom right) is not produced. @@ -491,14 +491,14 @@ diagnostic_maps_multimodel.py The diagnostic_maps_multimodel.py_ diagnostic makes model(s) vs observations maps and if data are not provided it draws only model field. -It is always nessary to define the overall layout trough the argument `layout_rowcol`, +It is always necessary to define the overall layout through the argument `layout_rowcol`, which is a list of two integers indicating respectively the number of rows and columns to organize the plot. Observations has not be accounted in here as they are automatically added at the top of the figure. This diagnostic also includes the optional arguments, `maps_range` and `diff_range` to manually define plot ranges. Both arguments are a list of two floats -to set plot range minimun and maximum values respectively for variable data and +to set plot range minimum and maximum values respectively for variable data and the Model minus Observations range. Note that this diagnostic assumes that the preprocessors do the bulk of the @@ -748,7 +748,7 @@ These tools are: - bgc_units: converts to sensible units where appropriate (ie Celsius, mmol/m3) - timecoord_to_float: Converts time series to decimal time ie: Midnight on January 1st 1970 is 1970.0 - add_legend_outside_right: a plotting tool, which adds a legend outside the axes. -- get_image_format: loads the image format, as defined in the global user config.yml. +- get_image_format: loads the image format, as defined in the global configuration. - get_image_path: creates a path for an image output. - make_cube_layer_dict: makes a dictionary for several layers of a cube. @@ -762,8 +762,8 @@ A note on the auxiliary data directory Some of these diagnostic scripts may not function on machines with no access to the internet, as cartopy may try to download the shape files. The solution to this issue is the put the relevant cartopy shapefiles in a directory which -is visible to esmvaltool, then link that path to ESMValTool via -the `auxiliary_data_dir` variable in your config-user.yml file. +is visible to esmvaltool, then link that path to ESMValTool via the +:ref:`configuration option ` ``auxiliary_data_dir``. The cartopy masking files can be downloaded from: https://www.naturalearthdata.com/downloads/ diff --git a/doc/sphinx/source/recipes/recipe_rainfarm.rst b/doc/sphinx/source/recipes/recipe_rainfarm.rst index d6c06c6f7a..aeb7cd0638 100644 --- a/doc/sphinx/source/recipes/recipe_rainfarm.rst +++ b/doc/sphinx/source/recipes/recipe_rainfarm.rst @@ -32,7 +32,7 @@ User settings * nf: number of subdivisions for downscaling (e.g. 8 will produce output fields with linear resolution increased by a factor 8) * conserv_glob: logical, if to conserve precipitation over full domain * conserv_smooth: logical, if to conserve precipitation using convolution (if neither conserv_glob or conserv_smooth is chosen, box conservation is used) -* weights_climo: set to false or omit if no orographic weights are to be used, else set it to the path to a fine-scale precipitation climatology file. If a relative file path is used, `auxiliary_data_dir` will be searched for this file. The file is expected to be in NetCDF format and should contain at least one precipitation field. If several fields at different times are provided, a climatology is derived by time averaging. Suitable climatology files could be for example a fine-scale precipitation climatology from a high-resolution regional climate model (see e.g. Terzago et al. 2018), a local high-resolution gridded climatology from observations, or a reconstruction such as those which can be downloaded from the WORLDCLIM (http://www.worldclim.org) or CHELSA (http://chelsa-climate.org) websites. The latter data will need to be converted to NetCDF format before being used (see for example the GDAL tools (https://www.gdal.org). +* weights_climo: set to false or omit if no orographic weights are to be used, else set it to the path to a fine-scale precipitation climatology file. If a relative file path is used, ``auxiliary_data_dir`` will be searched for this file. The file is expected to be in NetCDF format and should contain at least one precipitation field. If several fields at different times are provided, a climatology is derived by time averaging. Suitable climatology files could be for example a fine-scale precipitation climatology from a high-resolution regional climate model (see e.g. Terzago et al. 2018), a local high-resolution gridded climatology from observations, or a reconstruction such as those which can be downloaded from the WORLDCLIM (http://www.worldclim.org) or CHELSA (http://chelsa-climate.org) websites. The latter data will need to be converted to NetCDF format before being used (see for example the GDAL tools (https://www.gdal.org). Variables @@ -60,4 +60,4 @@ Example plots .. figure:: /recipes/figures/rainfarm/rainfarm.png :width: 14cm - Example of daily cumulated precipitation from the CMIP5 EC-EARTH model on a specific day, downscaled using RainFARM from its original resolution (1.125°) (left panel), increasing spatial resolution by a factor of 8 to 0.14°; Two stochastic realizations are shown (central and right panel). A fixed spectral slope of s=1.7 was used. Notice how the downscaled fields introduce fine scale precipitation structures, while still maintaining on average the original coarse-resolution precipitation. Different stochastic realizations are shown to demonstrate how an ensemble of realizations can be used to reproduce unresolved subgrid variability. (N.B.: this plot was not produced by ESMValTool - the recipe output is netcdf only). + Example of daily cumulated precipitation from the CMIP5 EC-EARTH model on a specific day, downscaled using RainFARM from its original resolution (1.125°) (left panel), increasing spatial resolution by a factor of 8 to 0.14°; Two stochastic realizations are shown (central and right panel). A fixed spectral slope of s=1.7 was used. Notice how the downscaled fields introduce fine scale precipitation structures, while still maintaining on average the original coarse-resolution precipitation. Different stochastic realizations are shown to demonstrate how an ensemble of realizations can be used to reproduce unresolved subgrid variability. (N.B.: this plot was not produced by ESMValTool - the recipe output is netcdf only). diff --git a/doc/sphinx/source/recipes/recipe_shapeselect.rst b/doc/sphinx/source/recipes/recipe_shapeselect.rst index 63afbcae6c..12da974c28 100644 --- a/doc/sphinx/source/recipes/recipe_shapeselect.rst +++ b/doc/sphinx/source/recipes/recipe_shapeselect.rst @@ -29,7 +29,7 @@ User settings in recipe *Required settings (scripts)* - * shapefile: path to the user provided shapefile. A relative path is relative to the auxiliary_data_dir as configured in config-user.yml. + * shapefile: path to the user provided shapefile. A relative path is relative to the :ref:`configuration option ` ``auxiliary_data_dir``. * weighting_method: the preferred weighting method 'mean_inside' - mean of all grid points inside polygon; 'representative' - one point inside or close to the polygon is used to represent the complete area. diff --git a/doc/sphinx/source/recipes/recipe_wenzel14jgr.rst b/doc/sphinx/source/recipes/recipe_wenzel14jgr.rst index 3c7fa86a3a..4faa05c2a9 100644 --- a/doc/sphinx/source/recipes/recipe_wenzel14jgr.rst +++ b/doc/sphinx/source/recipes/recipe_wenzel14jgr.rst @@ -28,8 +28,8 @@ User settings .. note:: - Make sure to run this recipe setting ``max_parallel_tasks: 1`` in the ``config_user.yml`` - file or using the CLI flag ``--max_parallel_tasks=1``. + Make sure to run this recipe with the :ref:`configuration option + ` ``max_parallel_tasks: 1``. User setting files (cfg files) are stored in nml/cfg_carbon/ diff --git a/doc/sphinx/source/recipes/recipe_wenzel16nat.rst b/doc/sphinx/source/recipes/recipe_wenzel16nat.rst index 03bb822545..a661844e70 100644 --- a/doc/sphinx/source/recipes/recipe_wenzel16nat.rst +++ b/doc/sphinx/source/recipes/recipe_wenzel16nat.rst @@ -35,9 +35,8 @@ User settings .. note:: - Make sure to run this recipe setting ``output_file_type: pdf`` in the ``config_user.yml`` - file or using the CLI flag ``--output_file_type=pdf``. - + Make sure to run this recipe with the :ref:`configuration option + ` ``max_parallel_tasks: 1``. #. Script carbon_beta.ncl @@ -58,7 +57,7 @@ User settings none -#. Script carbon_co2_cycle.ncl +#. Script carbon_co2_cycle.ncl *Required Settings (scripts)* @@ -72,7 +71,7 @@ User settings *Required settings (variables)* - * reference_dataset: name of reference datatset (observations) + * reference_dataset: name of reference dataset (observations) *Optional settings (variables)* @@ -102,15 +101,15 @@ Example plots ------------- .. figure:: /recipes/figures/wenzel16nat/fig_1.png - :width: 12 cm + :width: 12 cm :align: center - + Comparison of CO\ :sub:`2` seasonal amplitudes for CMIP5 historical simulations and observations showing annual mean atmospheric CO\ :sub:`2` versus the amplitudes of the CO\ :sub:`2` seasonal cycle at Pt. Barrow, Alaska (produced with carbon_co2_cycle.ncl, similar to Fig. 1a from Wenzel et al. (2016)). - + .. figure:: /recipes/figures/wenzel16nat/fig_2.png - :width: 12 cm + :width: 12 cm :align: center - + Barchart showing the gradient of the linear correlations for the comparison of CO\ :sub:`2` seasonal amplitudes for CMIP5 historical for at Pt. Barrow, Alaska (produced with carbon_co2_cycle.ncl, similar to Fig. 1b from Wenzel et al. (2016)). .. figure:: /recipes/figures/wenzel16nat/fig_3.png diff --git a/doc/sphinx/source/utils.rst b/doc/sphinx/source/utils.rst index 49c3df7aef..536b78ebee 100644 --- a/doc/sphinx/source/utils.rst +++ b/doc/sphinx/source/utils.rst @@ -135,10 +135,11 @@ This suite is configured to work with versions of cylc older than 8.0.0 . To prepare for using this tool: #. Log in to a system that uses `slurm `_ -#. Make sure the required CMIP and observational datasets are available and configured in config-user.yml +#. Make sure the required CMIP and observational datasets are available and + their ``rootpath`` and ``drs`` is properly set up in the :ref:`configuration + ` #. Make sure the required auxiliary data is available (see :ref:`recipe documentation `) #. Install ESMValTool -#. Update config-user.yml so it points to the right data locations Next, get started with `cylc `_: @@ -181,7 +182,7 @@ The following parameters have to be set in the script in order to make it run: Optionally, the following parameters can be edited: -* ``config_file``, *str*: Path to ``config-user.yml`` if default ``~/.esmvaltool/config-user.yml`` not used. +* ``config_dir``, *str*: Path to :ref:`configuration directory `, by default ``~/.config/esmvaltool/``. * ``partition``, *str*: Name of the DKRZ partition used to run jobs. Default is ``interactive`` to minimize computing cost compared to ``compute`` for which nodes cannot be shared. * ``memory``, *str*: Amount of memory requested for each run. Default is ``64G`` to allow to run 4 recipes on the same node in parallel. * ``time``, *str*: Time limit. Default is ``04:00:00`` to increase the job priority. Jobs can run for up to 8 hours and 12 hours on the compute and interactive partitions, respectively. @@ -230,7 +231,7 @@ script as well as a list of all available recipes. To generate the list, run the for recipe in $(esmvaltool recipes list | grep '\.yml$'); do echo $(basename "$recipe"); done > all_recipes.txt -To keep the script execution fast, it is recommended to use ``log_level: info`` in your config-user.yml file so that SLURM +To keep the script execution fast, it is recommended to use ``log_level: info`` in the configuration so that SLURM output files are rather small. .. _overview_page: diff --git a/esmvaltool/cmorizers/data/cmorizer.py b/esmvaltool/cmorizers/data/cmorizer.py index 16b7666350..5e66b7a70f 100755 --- a/esmvaltool/cmorizers/data/cmorizer.py +++ b/esmvaltool/cmorizers/data/cmorizer.py @@ -10,6 +10,7 @@ import os import shutil import subprocess +import warnings from pathlib import Path import esmvalcore @@ -18,13 +19,14 @@ from esmvalcore.config import CFG from esmvalcore.config._logging import configure_logging +from esmvaltool import ESMValToolDeprecationWarning from esmvaltool.cmorizers.data.utilities import read_cmor_config logger = logging.getLogger(__name__) datasets_file = os.path.join(os.path.dirname(__file__), 'datasets.yml') -class Formatter(): +class _Formatter(): """ Class to manage the download and formatting of datasets. @@ -39,26 +41,40 @@ def __init__(self, info): self.datasets_info = info self.config = '' - def start(self, command, datasets, config_file, options): + def start(self, command, datasets, config_file, config_dir, options): """Read configuration and set up formatter for data processing. Parameters ---------- command: str - Name of the command to execute + Name of the command to execute. datasets: str - List of datasets to process, comma separated + List of datasets to process, comma separated. config_file: str - Config file to use + Config file to use. Option will be removed in v2.14.0. + config_dir: str + Config directory to use. options: dict() - Extra options to overwrite config user file + Extra options to overwrite configuration. + """ if isinstance(datasets, str): self.datasets = datasets.split(',') else: self.datasets = datasets - CFG.load_from_file(config_file) + if config_file is not None: # remove in v2.14.0 + CFG.load_from_file(config_file) + elif config_dir is not None: + config_dir = Path( + os.path.expandvars(config_dir) + ).expanduser().absolute() + if not config_dir.is_dir(): + raise NotADirectoryError( + f"Invalid --config_dir given: {config_dir} is not an " + f"existing directory" + ) + CFG.update_from_dirs([config_dir]) CFG.update(options) self.config = CFG.start_session(f'data_{command}') @@ -199,8 +215,9 @@ def format(self, start, end, install): failed_datasets.append(dataset) if failed_datasets: - raise Exception( - f'Format failed for datasets {" ".join(failed_datasets)}') + raise RuntimeError( + f'Format failed for datasets {" ".join(failed_datasets)}' + ) @staticmethod def has_downloader(dataset): @@ -400,7 +417,7 @@ class DataCommand(): def __init__(self): with open(datasets_file, 'r', encoding='utf8') as data: self._info = yaml.safe_load(data) - self.formatter = Formatter(self._info) + self.formatter = _Formatter(self._info) def _has_downloader(self, dataset): return 'Yes' if self.formatter.has_downloader(dataset) else "No" @@ -441,28 +458,48 @@ def download(self, start=None, end=None, overwrite=False, + config_dir=None, **kwargs): """Download datasets. Parameters ---------- - datasets : list(str) + datasets: list(str) List of datasets to format - config_file : str, optional - Path to ESMValTool's config user file, by default None - start : str, optional + config_file: str, optional + Path to ESMValTool's config user file, by default None. + + .. deprecated:: 2.12.0 + This option has been deprecated in ESMValTool version 2.12.0 + and is scheduled for removal in version 2.14.0. Please use the + option `config_dir` instead. + start: str, optional Start of the interval to process, by default None. Valid formats are YYYY, YYYYMM and YYYYMMDD. - end : str, optional + end: str, optional End of the interval to process, by default None. Valid formats are YYYY, YYYYMM and YYYYMMDD. - overwrite : bool, optional + overwrite: bool, optional If true, download already present data again + config_dir: str, optional + Path to additional ESMValTool configuration directory. See + :ref:`esmvalcore:config_yaml_files` for details. + """ + if config_file is not None: + msg = ( + "The option `config_file` has been deprecated in ESMValTool " + "version 2.12.0 and is scheduled for removal in version " + "2.14.0. Please use the option ``config_dir`` instead." + ) + warnings.warn(msg, ESMValToolDeprecationWarning) + start = self._parse_date(start) end = self._parse_date(end) - self.formatter.start('download', datasets, config_file, kwargs) + self.formatter.start( + 'download', datasets, config_file, config_dir, kwargs + ) self.formatter.download(start, end, overwrite) def format(self, @@ -471,6 +508,7 @@ def format(self, start=None, end=None, install=False, + config_dir=None, **kwargs): """Format datasets. @@ -480,6 +518,11 @@ def format(self, List of datasets to format config_file : str, optional Path to ESMValTool's config user file, by default None + + .. deprecated:: 2.12.0 + This option has been deprecated in ESMValTool version 2.12.0 + and is scheduled for removal in version 2.14.0. Please use the + option `config_dir` instead. start : str, optional Start of the interval to process, by default None. Valid formats are YYYY, YYYYMM and YYYYMMDD. @@ -488,11 +531,25 @@ def format(self, are YYYY, YYYYMM and YYYYMMDD. install : bool, optional If true, move processed data to the folder, by default False + config_dir: str, optional + Path to additional ESMValTool configuration directory. See + :ref:`esmvalcore:config_yaml_files` for details. + """ + if config_file is not None: + msg = ( + "The option `config_file` has been deprecated in ESMValTool " + "version 2.12.0 and is scheduled for removal in version " + "2.14.0. Please use the option ``config_dir`` instead." + ) + warnings.warn(msg, ESMValToolDeprecationWarning) + start = self._parse_date(start) end = self._parse_date(end) - self.formatter.start('formatting', datasets, config_file, kwargs) + self.formatter.start( + 'formatting', datasets, config_file, config_dir, kwargs + ) self.formatter.format(start, end, install) def prepare(self, @@ -502,6 +559,7 @@ def prepare(self, end=None, overwrite=False, install=False, + config_dir=None, **kwargs): """Download and format a set of datasets. @@ -511,6 +569,11 @@ def prepare(self, List of datasets to format config_file : str, optional Path to ESMValTool's config user file, by default None + + .. deprecated:: 2.12.0 + This option has been deprecated in ESMValTool version 2.12.0 + and is scheduled for removal in version 2.14.0. Please use the + option `config_dir` instead. start : str, optional Start of the interval to process, by default None. Valid formats are YYYY, YYYYMM and YYYYMMDD. @@ -521,11 +584,25 @@ def prepare(self, If true, move processed data to the folder, by default False overwrite : bool, optional If true, download already present data again + config_dir: str, optional + Path to additional ESMValTool configuration directory. See + :ref:`esmvalcore:config_yaml_files` for details. + """ + if config_file is not None: + msg = ( + "The option `config_file` has been deprecated in ESMValTool " + "version 2.12.0 and is scheduled for removal in version " + "2.14.0. Please use the option ``config_dir`` instead." + ) + warnings.warn(msg, ESMValToolDeprecationWarning) + start = self._parse_date(start) end = self._parse_date(end) - self.formatter.start('preparation', datasets, config_file, kwargs) + self.formatter.start( + 'preparation', datasets, config_file, config_dir, kwargs + ) if self.formatter.download(start, end, overwrite): self.formatter.format(start, end, install) else: diff --git a/esmvaltool/cmorizers/data/datasets.yml b/esmvaltool/cmorizers/data/datasets.yml index 8fcb6adc21..0f34a738dc 100644 --- a/esmvaltool/cmorizers/data/datasets.yml +++ b/esmvaltool/cmorizers/data/datasets.yml @@ -17,16 +17,16 @@ datasets: analyses covering analysis of monthly rainfall. The dataset provides consistent temporal and spatial analyses across Australia for each observed data variable. This accounts for spatial and temporal gaps in observations. Where possible, the gridded analysis techniques provide useful estimates in data-sparse regions - such as central Australia. - + such as central Australia. + Time coverage: Site-based data are used to provide gridded climate data at the monthly timescale for rainfall (1900+). Reference: Evans, A., Jones, D.A., Smalley, R., and Lellyett, S. 2020. An enhanced gridded rainfall analysis scheme for Australia. Bureau of Meteorology Research Report. No. 41. National Computational Infrastructure (NCI) - Catalogue Record: http://dx.doi.org/10.25914/6009600786063. - Data from NCI (National Computing Infrastructure Australia https://nci.org.au/), + Data from NCI (National Computing Infrastructure Australia https://nci.org.au/), requires an NCI account and access to Gadi(Supercomputer in Canberra) and the project found in catalogue record. Access can be requested through NCI. NCI is an ESGF node (https://esgf.nci.org.au/projects/esgf-nci/) - + ANUClimate: tier: 3 source: "https://dx.doi.org/10.25914/60a10aa56dd1b" @@ -35,7 +35,7 @@ datasets: Data from NCI project requiring an NCI account and access to GADI ANUClimate 2.0 consists of gridded daily and monthly climate variables across the terrestrial landmass of Australia - from at least 1970 to the present. Rainfall grids are generated from 1900 to the present. The underpinning spatial + from at least 1970 to the present. Rainfall grids are generated from 1900 to the present. The underpinning spatial models have been developed at the Fenner School of Environment and Society of the Australian National University. APHRO-MA: @@ -301,7 +301,7 @@ datasets: last_access: 2020-03-23 info: | Create a new empty directory ``$RAWOBSPATH/Tier2/CT2019`` (where - ``$RAWOBSPATH`` is given by your user configuration file) where the raw + ``$RAWOBSPATH`` is given by your configuration) where the raw data will be stored. The download of the data is automatically handled by this script. If data is already present in this directory, the download is skipped (to force a new download delete your old files). @@ -479,11 +479,11 @@ datasets: Download and processing instructions: Use the following CLI to download all the files: esmvaltool data download ESACCI-LANDCOVER - The underlying downloader is located here: + The underlying downloader is located here: /ESMValTool/esmvaltool/cmorizers/data/downloaders/datasets/esacci_landcover.py - and it will download all the files currently available on CEDA (1992-2020) + and it will download all the files currently available on CEDA (1992-2020) under a single directory as follow: ${RAWOBS}/Tier2/ESACCI-LANDCOVER - + ESACCI-LST: tier: 2 source: On CEDA-JASMIN, /gws/nopw/j04/esacci_lst/public @@ -554,7 +554,7 @@ datasets: source: https://wui.cmsaf.eu/safira/action/viewDoiDetails?acronym=COMBI_V001 last_access: 2024-02-21 info: | - CDR2 requires registration at EUMETSAT CM SAF, the information on how to + CDR2 requires registration at EUMETSAT CM SAF, the information on how to download the order will be emailed once the order is ready. All files need to be in one directory, not in yearly subdirectories. @@ -903,11 +903,11 @@ datasets: Select "Data Access" -> "Subset/Get Data" -> "Get Data" and follow the "Instructions for downloading". All *.he5 files need to be saved in the $RAWOBS/Tier3/MLS-AURA directory, where $RAWOBS refers to the RAWOBS - directory defined in the user configuration file. Apply this procedure to - both links provided above. The temperature fields are necessary for quality + directory defined in the configuration. Apply this procedure to both + links provided above. The temperature fields are necessary for quality control of the RHI data (see Data Quality Document for MLS-AURA for more information). - A registration is required + A registration is required. MOBO-DIC_MPIM: tier: 2 @@ -1076,7 +1076,7 @@ datasets: last_access: 2023-12-04 info: | Download the following files: - ersst.yyyymm.nc + ersst.yyyymm.nc for years 1854 to 2020 NOAA-ERSSTv5: @@ -1085,7 +1085,7 @@ datasets: last_access: 2023-12-04 info: | Download the following files: - ersst.v5.yyyymm.nc + ersst.v5.yyyymm.nc for years 1854 onwards NOAAGlobalTemp: @@ -1112,13 +1112,13 @@ datasets: Download daily data from: https://nsidc.org/data/NSIDC-0116 Login required for download, and also requires citation only to use - + NSIDC-G02202-sh: tier: 3 source: https://polarwatch.noaa.gov/erddap/griddap/nsidcG02202v4shmday last_access: 2023-05-13 info: | - Download monthly data. + Download monthly data. Login required for download, and also requires citation only to use OceanSODA-ETHZ: diff --git a/esmvaltool/cmorizers/data/download_scripts/download_era_interim.py b/esmvaltool/cmorizers/data/download_scripts/download_era_interim.py index 72cf8d98af..374c750ef6 100644 --- a/esmvaltool/cmorizers/data/download_scripts/download_era_interim.py +++ b/esmvaltool/cmorizers/data/download_scripts/download_era_interim.py @@ -12,8 +12,13 @@ 4. Copy/paste the text in https://api.ecmwf.int/v1/key/ into a blank text file and save it as $HOME/.ecmwfapirc -5. Use ESMValCore/esmvalcore/config-user.yml as an template -and set the rootpath of the output directory in RAWOBS +5. Copy the default configuration file with + +```bash +esmvaltool config get_config_user --path=config-user.yml +``` + +and set the ``rootpath`` for the RAWOBS project. 6. Check the description of the variables at https://apps.ecmwf.int/codes/grib/param-db diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/jra_55.py b/esmvaltool/cmorizers/data/downloaders/datasets/jra_55.py index a5dc5b851c..7a9e374136 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/jra_55.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/jra_55.py @@ -1,14 +1,12 @@ """Script to download JRA-55 from RDA.""" import logging import os - from datetime import datetime from dateutil import relativedelta from esmvaltool.cmorizers.data.downloaders.wget import WGetDownloader - logger = logging.getLogger(__name__) diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv3b.py b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv3b.py index 0ac6a3e012..5a54080be4 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv3b.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv3b.py @@ -1,6 +1,7 @@ """Script to download NOAA-ERSST-v3b.""" import logging from datetime import datetime + from dateutil import relativedelta from esmvaltool.cmorizers.data.downloaders.wget import WGetDownloader diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv5.py b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv5.py index f995f9d2c7..7dbeccfe12 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv5.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/noaa_ersstv5.py @@ -1,6 +1,7 @@ """Script to download NOAA-ERSST-V5.""" import logging from datetime import datetime + from dateutil import relativedelta from esmvaltool.cmorizers.data.downloaders.wget import WGetDownloader diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_g02202_sh.py b/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_g02202_sh.py index 798decda96..8c3c02c410 100644 --- a/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_g02202_sh.py +++ b/esmvaltool/cmorizers/data/downloaders/datasets/nsidc_g02202_sh.py @@ -1,6 +1,7 @@ """Script to download NSIDC-G02202-sh.""" import logging from datetime import datetime + from dateutil import relativedelta from esmvaltool.cmorizers.data.downloaders.wget import WGetDownloader diff --git a/esmvaltool/cmorizers/data/formatters/datasets/ct2019.py b/esmvaltool/cmorizers/data/formatters/datasets/ct2019.py index 33f56f234d..64f64f4e82 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/ct2019.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/ct2019.py @@ -11,7 +11,7 @@ Download and processing instructions Create a new empty directory ``$RAWOBSPATH/Tier2/CT2019`` (where - ``$RAWOBSPATH`` is given by your user configuration file) where the raw + ``$RAWOBSPATH`` is given in the configuration) where the raw data will be stored. The download of the data is automatically handled by this script. If data is already present in this directory, the download is skipped (to force a new download delete your old files). diff --git a/esmvaltool/cmorizers/data/formatters/datasets/merra.ncl b/esmvaltool/cmorizers/data/formatters/datasets/merra.ncl index b57bca6a09..d9fbf761df 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/merra.ncl +++ b/esmvaltool/cmorizers/data/formatters/datasets/merra.ncl @@ -14,7 +14,7 @@ ; Download and processing instructions ; (requires EarthData login; see https://urs.earthdata.nasa.gov/) ; Use ESMValTool automatic download: -; esmvaltool data download --config_file MERRA +; esmvaltool data download MERRA ; ; Modification history ; 20230818-lauer_axel: added output of clwvi (iwp + lwp) @@ -209,7 +209,7 @@ begin delete(tmp) - ; calcuation of outgoing fluxes: out = in - net + ; calculation of outgoing fluxes: out = in - net if ((VAR(vv) .eq. "rsut") .or. (VAR(vv) .eq. "rsutcs")) then tmp = f->SWTDN if (isatt(tmp, "scale_factor") .or. isatt(tmp, "add_offset")) then @@ -220,7 +220,8 @@ begin delete(tmp) end if - ; calcuation of total precipitation flux = large-scale+convective+anvil + ; calculation of total precipitation flux = + ; large-scale+convective+anvil if (VAR(vv) .eq. "pr") then tmp = f->PRECCON ; surface precipitation flux from convection if (isatt(tmp, "scale_factor") .or. isatt(tmp, "add_offset")) then diff --git a/esmvaltool/cmorizers/data/formatters/datasets/mls_aura.py b/esmvaltool/cmorizers/data/formatters/datasets/mls_aura.py index 5b500e9087..0a5031b243 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/mls_aura.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/mls_aura.py @@ -14,7 +14,7 @@ Select "Data Access" -> "Subset/Get Data" -> "Get Data" and follow the "Instructions for downloading". All *.he5 files need to be saved in the $RAWOBS/Tier3/MLS-AURA directory, where $RAWOBS refers to the RAWOBS - directory defined in the user configuration file. Apply this procedure to + directory defined in the configuration. Apply this procedure to both links provided above. The temperature fields are necessary for quality control of the RHI data (see Data Quality Document for MLS-AURA for more information). diff --git a/esmvaltool/diag_scripts/kcs/local_resampling.py b/esmvaltool/diag_scripts/kcs/local_resampling.py index 9eb2ea28ed..0bf6260d65 100644 --- a/esmvaltool/diag_scripts/kcs/local_resampling.py +++ b/esmvaltool/diag_scripts/kcs/local_resampling.py @@ -292,7 +292,7 @@ def select_final_subset(cfg, subsets, prov=None): Final set of eight samples should have with minimal reuse of the same ensemble member for the same period. From 10.000 randomly - selected sets of 8 samples, count and penalize re-used segments (1 + selected sets of 8 samples, count and penalize reused segments (1 for 3*reuse, 5 for 4*reuse). Choose the set with the lowest penalty. """ n_samples = cfg['n_samples'] @@ -387,7 +387,7 @@ def _get_climatology(cfg, scenario_name, table, prov=None): resampled_control = _recombine(segments_control, table['control']) resampled_future = _recombine(segments_future, table['future']) - # Store the resampled contol climates + # Store the resampled control climates filename = get_diagnostic_filename(f'resampled_control_{scenario_name}', cfg, extension='nc') diff --git a/esmvaltool/diag_scripts/monitor/compute_eofs.py b/esmvaltool/diag_scripts/monitor/compute_eofs.py index dea5d63b9a..a07ca835c0 100644 --- a/esmvaltool/diag_scripts/monitor/compute_eofs.py +++ b/esmvaltool/diag_scripts/monitor/compute_eofs.py @@ -24,10 +24,10 @@ Path to the folder to store figures. Defaults to ``{plot_dir}/../../{dataset}/{exp}/{modeling_realm}/{real_name}``. All tags (i.e., the entries in curly brackets, e.g., ``{dataset}``, are - replaced with the corresponding tags). ``{plot_dir}`` is replaced with the + replaced with the corresponding tags). ``{plot_dir}`` is replaced with the default ESMValTool plot directory (i.e., ``output_dir/plots/diagnostic_name/script_name/``, see - :ref:`esmvalcore:user configuration file`). + :ref:`esmvalcore:outputdata`). rasterize_maps: bool, optional (default: True) If ``True``, use `rasterization `_ for diff --git a/esmvaltool/diag_scripts/monitor/monitor.py b/esmvaltool/diag_scripts/monitor/monitor.py index 59e37b9842..dda5aa4f3d 100644 --- a/esmvaltool/diag_scripts/monitor/monitor.py +++ b/esmvaltool/diag_scripts/monitor/monitor.py @@ -52,10 +52,10 @@ Path to the folder to store figures. Defaults to ``{plot_dir}/../../{dataset}/{exp}/{modeling_realm}/{real_name}``. All tags (i.e., the entries in curly brackets, e.g., ``{dataset}``, are - replaced with the corresponding tags). ``{plot_dir}`` is replaced with the + replaced with the corresponding tags). ``{plot_dir}`` is replaced with the default ESMValTool plot directory (i.e., ``output_dir/plots/diagnostic_name/script_name/``, see - :ref:`esmvalcore:user configuration file`). + :ref:`esmvalcore:outputdata`). rasterize_maps: bool, optional (default: True) If ``True``, use `rasterization `_ for diff --git a/esmvaltool/diag_scripts/monitor/multi_datasets.py b/esmvaltool/diag_scripts/monitor/multi_datasets.py index 879346954c..32f654b3b6 100644 --- a/esmvaltool/diag_scripts/monitor/multi_datasets.py +++ b/esmvaltool/diag_scripts/monitor/multi_datasets.py @@ -100,10 +100,10 @@ Path to the folder to store figures. Defaults to ``{plot_dir}/../../{dataset}/{exp}/{modeling_realm}/{real_name}``. All tags (i.e., the entries in curly brackets, e.g., ``{dataset}``, are - replaced with the corresponding tags). ``{plot_dir}`` is replaced with the + replaced with the corresponding tags). ``{plot_dir}`` is replaced with the default ESMValTool plot directory (i.e., ``output_dir/plots/diagnostic_name/script_name/``, see - :ref:`esmvalcore:user configuration file`). + :ref:`esmvalcore:outputdata`). savefig_kwargs: dict, optional Optional keyword arguments for :func:`matplotlib.pyplot.savefig`. By default, uses ``bbox_inches: tight, dpi: 300, orientation: landscape``. diff --git a/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig6a.ncl b/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig6a.ncl index bd672ed3cf..0f1b49c224 100644 --- a/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig6a.ncl +++ b/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig6a.ncl @@ -151,10 +151,8 @@ begin fx_variable = "volcello" error_msg("f", "russell18jgr-fig6.ncl", " ", "volcello file for " \ + vo_datasets(iii) \ - + " not found in the metadata file, please add "\ - + "'fx_files: [volcello]' to the variable dictionary in the " \ - + "recipe or add the location of file to input directory " \ - + "in config-user.yml ") + + " not found in the metadata file, please specify " \ + + "'volcello' as supplementary variable in the recipe.") end if dataset_so_time = read_data(so_items[iii]) diff --git a/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig6b.ncl b/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig6b.ncl index 6b019625f0..71323f411d 100644 --- a/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig6b.ncl +++ b/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig6b.ncl @@ -45,10 +45,10 @@ ; ; Caveats ; -; - MIROC-ESM and BNU-ESM doesnot work as depth variable is not called lev. -; - MRI_ESM1 doesnot work as the data is ofset by 80 degrees in longitude +; - MIROC-ESM and BNU-ESM does not work as depth variable is not called lev. +; - MRI_ESM1 does not work as the data is offset by 80 degrees in longitude ; and causes problem in interpolation. -; - CCSM4 ans CESM1-CAM5 dont work as the units for so is 1, not accepted +; - CCSM4 and CESM1-CAM5 dont work as the units for so is 1, not accepted ; by ESMValTool. ; - Transport is very small in case of NorESM1-M and ME as volcello ; values look incorrect(very small). @@ -153,11 +153,10 @@ begin if (all(ismissing(fx_var))) then fx_variable = "volcello" - error_msg("f", "russell_fig-7i.ncl", " ", "areacello file for " + \ + error_msg("f", "russell_fig-7i.ncl", " ", "volcello file for " + \ vo_datasets(iii) \ - + " not found in the metadata file, please " + \ - "add 'fx_files: [volcello]' to the variable dictionary in" + \ - " the recipe or add the location of file to config-user.yml") + + " not found in the metadata file, please specify " \ + + "'volcello' as supplementary variable in the recipe.") end if dataset_so_time = read_data(so_items[iii]) diff --git a/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig7i.ncl b/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig7i.ncl index 86ce4bee70..cf14857a7b 100644 --- a/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig7i.ncl +++ b/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig7i.ncl @@ -156,9 +156,8 @@ begin fx_variable = "areacello" error_msg("f", "russell_fig-7i.ncl", " ", "areacello file for " + \ datasetnames(iii) + " not found in the metadata file," + \ - " please add 'fx_files: [areacello]' to the variable " + \ - "dictionary in the recipe or add the location of " + \ - " file to config-user.yml") + + " not found in the metadata file, please specify " \ + + "'areacello' as supplementary variable in the recipe.") end if areacello_2d = fx_var delete(fx_var) @@ -212,9 +211,9 @@ begin "lgPerimOn" : False ; no perimeter "lgItemCount" : dimsizes(annots) ; how many "lgLineLabelStrings" : annots ; labels - "lgLabelsOn" : False ; no default lables + "lgLabelsOn" : False ; no default labsels "lgLineLabelFontHeightF" : 0.0085 ; font height - "lgDashIndexes" : dashes ; line paterns + "lgDashIndexes" : dashes ; line patterns "lgLineColors" : colors "lgMonoLineLabelFontColor" : True ; one label color end create diff --git a/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig9c.ncl b/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig9c.ncl index 2fe0cc3e4a..017b70103a 100644 --- a/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig9c.ncl +++ b/esmvaltool/diag_scripts/russell18jgr/russell18jgr-fig9c.ncl @@ -227,9 +227,8 @@ begin if (all(ismissing(fx_var))) then error_msg("f", "russell18jgr-fig9c.ncl", " ", "areacello file for " + \ datasetnames(iii) + " not found in the metadata file, " + \ - "please add 'fx_files: [areacello]' to the variable " + \ - "dictionary in the recipe or add the location of " + \ - " file to config-user.yml ") + + " not found in the metadata file, please specify " \ + + "'areacello' as supplementary variable in the recipe.") end if areacello_2d = fx_var @@ -304,9 +303,9 @@ begin "lgPerimOn" : False ; no perimeter "lgItemCount" : dimsizes(annots) ; how many "lgLabelStrings" : annots ; labels - "lgLabelsOn" : True ; no default lables + "lgLabelsOn" : True ; no default labels "lgLabelFontHeightF" : 0.001 ; font height - "lgItemType" : "markers" ; line paterns + "lgItemType" : "markers" ; line patterns "lgMarkerColors" : colors "lgMarkerIndexes" : markers ; one label color end create diff --git a/esmvaltool/interface_scripts/logging.ncl b/esmvaltool/interface_scripts/logging.ncl index 6333479f96..35c3167341 100644 --- a/esmvaltool/interface_scripts/logging.ncl +++ b/esmvaltool/interface_scripts/logging.ncl @@ -61,9 +61,9 @@ procedure log_debug(output_string[*]:string) ; output_string: the text to be output as message on screen ; ; Description -; Write a debug message to the log file (only if log_level = debug in -; config-user.yml). If the input is an array, each element will be -; written on different lines. +; Write a debug message to the log file (only if log_level = debug in the +; configuration). If the input is an array, each element will be written on +; different lines. ; ; Caveats ; diff --git a/esmvaltool/recipes/examples/recipe_extract_shape.yml b/esmvaltool/recipes/examples/recipe_extract_shape.yml index 79f04371b5..08d1bab490 100644 --- a/esmvaltool/recipes/examples/recipe_extract_shape.yml +++ b/esmvaltool/recipes/examples/recipe_extract_shape.yml @@ -7,7 +7,7 @@ documentation: The example shapefile(s) can be copied from esmvaltool/diag_scripts/shapeselect/testdata/Elbe.* and - placed in the auxiliary_data_dir defined in config-user.yml. + placed in the auxiliary_data_dir defined in the configuration. title: Example recipe extracting precipitation in the Elbe catchment. diff --git a/esmvaltool/recipes/hydrology/recipe_hydro_forcing.yml b/esmvaltool/recipes/hydrology/recipe_hydro_forcing.yml index f68a597733..925d9bd420 100644 --- a/esmvaltool/recipes/hydrology/recipe_hydro_forcing.yml +++ b/esmvaltool/recipes/hydrology/recipe_hydro_forcing.yml @@ -9,7 +9,7 @@ documentation: used to: 1. Plot a timeseries of the raw daily data - 2. Plot monthly aggregrated data over a certain period + 2. Plot monthly aggregated data over a certain period 3. Plot the monthly climate statistics over a certain period authors: @@ -33,7 +33,7 @@ datasets: preprocessors: daily: extract_shape: &extract_shape - # In aux (config-user.yml) + # Relative to auxiliary_data_dir defined in configuration shapefile: Lorentz_Basin_Shapefiles/Meuse/Meuse.shp method: contains crop: true diff --git a/esmvaltool/recipes/hydrology/recipe_lisflood.yml b/esmvaltool/recipes/hydrology/recipe_lisflood.yml index ffecbc37be..3acb4be481 100644 --- a/esmvaltool/recipes/hydrology/recipe_lisflood.yml +++ b/esmvaltool/recipes/hydrology/recipe_lisflood.yml @@ -37,7 +37,8 @@ preprocessors: scheme: linear extract_shape: # Perhaps a single shapefile needs to be created covering multiple basins - shapefile: Lorentz_Basin_Shapefiles/Meuse/Meuse.shp # (config-user, aux) + # Relative to auxiliary_data_dir defined in configuration + shapefile: Lorentz_Basin_Shapefiles/Meuse/Meuse.shp method: contains crop: true # set to false to keep the entire globe (memory intensive!) daily_water: diff --git a/esmvaltool/recipes/hydrology/recipe_marrmot.yml b/esmvaltool/recipes/hydrology/recipe_marrmot.yml index dd6eef0a49..e85a66d9b9 100644 --- a/esmvaltool/recipes/hydrology/recipe_marrmot.yml +++ b/esmvaltool/recipes/hydrology/recipe_marrmot.yml @@ -28,7 +28,8 @@ preprocessors: daily: &daily extract_shape: # Lumped model: needs catchment-aggregated input data - shapefile: Meuse/Meuse.shp # In aux (config-user.yml) + # Relative to auxiliary_data_dir defined in configuration + shapefile: Meuse/Meuse.shp method: contains crop: true diff --git a/esmvaltool/recipes/ipccwg1ar6ch3/recipe_ipccwg1ar6ch3_fig_3_42_a.yml b/esmvaltool/recipes/ipccwg1ar6ch3/recipe_ipccwg1ar6ch3_fig_3_42_a.yml index 20b0402a23..55c53147ec 100644 --- a/esmvaltool/recipes/ipccwg1ar6ch3/recipe_ipccwg1ar6ch3_fig_3_42_a.yml +++ b/esmvaltool/recipes/ipccwg1ar6ch3/recipe_ipccwg1ar6ch3_fig_3_42_a.yml @@ -10,7 +10,7 @@ documentation: Contribution to the Sixth Assessment Report: Chapter 3 Processing of CMIP3 models currently works only in serial mode, due to an issue in the input data still under investigation. To run the recipe - set: max_parallel_tasks: 1 in the config-user.yml file. + set the configuration option ``max_parallel_tasks: 1``. authors: - bock_lisa diff --git a/esmvaltool/recipes/recipe_carvalhais14nat.yml b/esmvaltool/recipes/recipe_carvalhais14nat.yml index 9ec0811c00..63bfbb1edd 100644 --- a/esmvaltool/recipes/recipe_carvalhais14nat.yml +++ b/esmvaltool/recipes/recipe_carvalhais14nat.yml @@ -8,7 +8,7 @@ documentation: Carvalhais et al., 2014, Nature. The data required in the obs_details section can be obtained at http://www.bgc-jena.mpg.de/geodb/BGI/tau4ESMValTool.php - and have to be stored in the auxiliary_data_dir defined i config-user.yml, + and have to be stored in the auxiliary_data_dir defined in the configuration in a subdirectory obs_data_subdir specified in the obs_details section below. diff --git a/esmvaltool/recipes/recipe_runoff_et.yml b/esmvaltool/recipes/recipe_runoff_et.yml index 6924321c7c..0a83213caa 100644 --- a/esmvaltool/recipes/recipe_runoff_et.yml +++ b/esmvaltool/recipes/recipe_runoff_et.yml @@ -8,7 +8,7 @@ documentation: water balance components for different catchments and compares the results against observations. Currently, the required catchment mask needs to be downloaded manually at https://doi.org/10.5281/zenodo.2025776 and saved in - the auxiliary_data_dir defined in config-user.yml. + the auxiliary_data_dir defined in configuration. authors: - hagemann_stefan diff --git a/esmvaltool/recipes/recipe_sea_surface_salinity.yml b/esmvaltool/recipes/recipe_sea_surface_salinity.yml index 4e670eec7f..43ec0e6b5e 100644 --- a/esmvaltool/recipes/recipe_sea_surface_salinity.yml +++ b/esmvaltool/recipes/recipe_sea_surface_salinity.yml @@ -20,8 +20,7 @@ documentation: preprocessors: timeseries: extract_shape: - # Relative paths are relative to 'auxiliary_data_dir' as configured in - # the config-user.yml file. + # Relative paths are relative to the configuration option 'auxiliary_data_dir'. # The example shapefile can be downloaded from # https://marineregions.org/download_file.php?name=World_Seas_IHO_v3.zip # but any shapefile can be used @@ -50,7 +49,7 @@ datasets: - {<<: *cmip6, dataset: MPI-ESM1-2-HR, alias: MPI-ESM1-2-HR} - {<<: *cmip6, dataset: NorESM2-MM, alias: NorESM2-MM} - {<<: *cmip6, dataset: GISS-E2-2-H, alias: GISS-E2-2-H, institute: NASA-GISS} - + diagnostics: compare_salinity: diff --git a/esmvaltool/recipes/recipe_shapeselect.yml b/esmvaltool/recipes/recipe_shapeselect.yml index 0fb22c0d5d..ee56810f03 100644 --- a/esmvaltool/recipes/recipe_shapeselect.yml +++ b/esmvaltool/recipes/recipe_shapeselect.yml @@ -36,8 +36,7 @@ diagnostics: script: shapeselect/diag_shapeselect.py # Example shapefiles can be found in: # esmvaltool/diag_scripts/shapeselect/testdata/ - # Relative paths are relative to 'auxiliary_data_dir' as configured in - # the config-user.yml file. + # Relative paths are relative to configuration option 'auxiliary_data_dir'. shapefile: 'Thames.shp' weighting_method: 'mean_inside' write_xlsx: true diff --git a/esmvaltool/utils/batch-jobs/generate.py b/esmvaltool/utils/batch-jobs/generate.py index d1ceeffaa0..428229b6eb 100644 --- a/esmvaltool/utils/batch-jobs/generate.py +++ b/esmvaltool/utils/batch-jobs/generate.py @@ -9,7 +9,7 @@ - conda_path 2) If needed, edit optional parameters: - outputs -- config_file +- config_dir 3) SLURM settings This script is configured to optimize the computing footprint of the recipe testing. It is not necessary to edit @@ -49,11 +49,11 @@ # Full path to the miniforge3/etc/profile.d/conda.sh executable # Set the path to conda conda_path = 'PATH_TO/miniforge3/etc/profile.d/conda.sh' -# Full path to config_file -# If none, ~/.esmvaltool/config-user.yml is used -config_file = '' +# Full path to configuration directory +# If none, ~/.config/esmvaltool/ +config_dir = '' # Set max_parallel_tasks -# If none, read from config_file +# If none, read from configuration default_max_parallel_tasks = 8 # List of recipes that require non-default SLURM options set above @@ -315,11 +315,11 @@ def generate_submit(): file.write(f'. {conda_path}\n') file.write(f'conda activate {env}\n') file.write('\n') - if not config_file: + if not config_dir: file.write(f'esmvaltool run {str(recipe)}') else: - file.write(f'esmvaltool run --config_file ' - f'{str(config_file)} {str(recipe)}') + file.write(f'esmvaltool run --config_dir ' + f'{str(config_dir)} {str(recipe)}') # set max_parallel_tasks max_parallel_tasks = MAX_PARALLEL_TASKS.get( recipe.stem, diff --git a/tests/integration/test_cmorizer.py b/tests/integration/test_cmorizer.py index 11bade4190..48f75b951a 100644 --- a/tests/integration/test_cmorizer.py +++ b/tests/integration/test_cmorizer.py @@ -4,6 +4,7 @@ import os import sys +import esmvalcore import iris import iris.coord_systems import iris.coords @@ -13,7 +14,9 @@ import pytest import yaml from cf_units import Unit +from packaging import version +from esmvaltool import ESMValToolDeprecationWarning from esmvaltool.cmorizers.data.cmorizer import DataCommand @@ -28,8 +31,8 @@ def keep_cwd(): os.chdir(curr_path) -def write_config_user_file(dirname): - """Replace config_user file values for testing.""" +def write_config_file(dirname): + """Replace configuration values for testing.""" config_file = dirname / 'config-user.yml' cfg = { 'output_dir': str(dirname / 'output_dir'), @@ -143,14 +146,59 @@ def arguments(*args): sys.argv = backup -def test_cmorize_obs_woa_no_data(tmp_path): +@pytest.mark.skipif( + version.parse(esmvalcore.__version__) >= version.parse("2.14.0"), + reason='ESMValCore >= v2.14.0', +) +def test_cmorize_obs_woa_no_data_config_file(tmp_path): """Test for example run of cmorize_obs command.""" + config_file = write_config_file(tmp_path) + os.makedirs(os.path.join(tmp_path, 'raw_stuff', 'Tier2')) + os.makedirs(os.path.join(tmp_path, 'output_dir')) + with keep_cwd(): + with pytest.raises(RuntimeError): + with pytest.warns(ESMValToolDeprecationWarning): + DataCommand().format('WOA', config_file=config_file) + + log_dir = os.path.join(tmp_path, 'output_dir') + log_file = os.path.join(log_dir, + os.listdir(log_dir)[0], 'run', 'main_log.txt') + check_log_file(log_file, no_data=True) + + +@pytest.mark.skipif( + version.parse(esmvalcore.__version__) >= version.parse("2.14.0"), + reason='ESMValCore >= v2.14.0', +) +def test_cmorize_obs_woa_data_config_file(tmp_path): + """Test for example run of cmorize_obs command.""" + config_file = write_config_file(tmp_path) + data_path = os.path.join(tmp_path, 'raw_stuff', 'Tier2', 'WOA') + put_dummy_data(data_path) + with keep_cwd(): + with pytest.warns(ESMValToolDeprecationWarning): + DataCommand().format('WOA', config_file=config_file) - config_user_file = write_config_user_file(tmp_path) + log_dir = os.path.join(tmp_path, 'output_dir') + log_file = os.path.join(log_dir, + os.listdir(log_dir)[0], 'run', 'main_log.txt') + check_log_file(log_file, no_data=False) + output_path = os.path.join(log_dir, os.listdir(log_dir)[0], 'Tier2', 'WOA') + check_output_exists(output_path) + check_conversion(output_path) + + +@pytest.mark.skipif( + version.parse(esmvalcore.__version__) < version.parse("2.12.0"), + reason='ESMValCore < v2.12.0', +) +def test_cmorize_obs_woa_no_data(tmp_path): + """Test for example run of cmorize_obs command.""" + write_config_file(tmp_path) os.makedirs(os.path.join(tmp_path, 'raw_stuff', 'Tier2')) with keep_cwd(): - with pytest.raises(Exception): - DataCommand().format('WOA', config_user_file) + with pytest.raises(RuntimeError): + DataCommand().format('WOA', config_dir=str(tmp_path)) log_dir = os.path.join(tmp_path, 'output_dir') log_file = os.path.join(log_dir, @@ -158,14 +206,17 @@ def test_cmorize_obs_woa_no_data(tmp_path): check_log_file(log_file, no_data=True) +@pytest.mark.skipif( + version.parse(esmvalcore.__version__) < version.parse("2.12.0"), + reason='ESMValCore < v2.12.0', +) def test_cmorize_obs_woa_data(tmp_path): """Test for example run of cmorize_obs command.""" - - config_user_file = write_config_user_file(tmp_path) + write_config_file(tmp_path) data_path = os.path.join(tmp_path, 'raw_stuff', 'Tier2', 'WOA') put_dummy_data(data_path) with keep_cwd(): - DataCommand().format('WOA', config_user_file) + DataCommand().format('WOA', config_dir=str(tmp_path)) log_dir = os.path.join(tmp_path, 'output_dir') log_file = os.path.join(log_dir, diff --git a/tests/integration/test_diagnostic_run.py b/tests/integration/test_diagnostic_run.py index b0c606f4ee..670f7088dd 100644 --- a/tests/integration/test_diagnostic_run.py +++ b/tests/integration/test_diagnostic_run.py @@ -5,12 +5,14 @@ from pathlib import Path from textwrap import dedent +import esmvalcore import pytest import yaml from esmvalcore._main import run +from packaging import version -def write_config_user_file(dirname): +def write_config_file(dirname): config_file = dirname / 'config-user.yml' cfg = { 'output_dir': str(dirname / 'output_dir'), @@ -68,10 +70,13 @@ def check(result_file): ] +@pytest.mark.skipif( + version.parse(esmvalcore.__version__) >= version.parse("2.14.0"), + reason='ESMValCore >= v2.14.0', +) @pytest.mark.installation @pytest.mark.parametrize('script_file', SCRIPTS) -def test_diagnostic_run(tmp_path, script_file): - +def test_diagnostic_run_config_file(tmp_path, script_file): local_script_file = Path(__file__).parent / script_file recipe_file = tmp_path / 'recipe_test.yml' @@ -96,12 +101,58 @@ def test_diagnostic_run(tmp_path, script_file): """.format(script_file, result_file)) recipe_file.write_text(str(recipe)) - config_user_file = write_config_user_file(tmp_path) + config_file = write_config_file(tmp_path) with arguments( 'esmvaltool', 'run', '--config_file', - config_user_file, + config_file, + str(recipe_file), + ): + run() + + check(result_file) + + +@pytest.mark.skipif( + version.parse(esmvalcore.__version__) < version.parse("2.12.0"), + reason='ESMValCore < v2.12.0', +) +@pytest.mark.installation +@pytest.mark.parametrize('script_file', SCRIPTS) +def test_diagnostic_run(tmp_path, script_file): + local_script_file = Path(__file__).parent / script_file + + recipe_file = tmp_path / 'recipe_test.yml' + script_file = tmp_path / script_file + result_file = tmp_path / 'result.yml' + config_dir = tmp_path / 'config' + config_dir.mkdir(exist_ok=True, parents=True) + + shutil.copy(local_script_file, script_file) + + # Create recipe + recipe = dedent(""" + documentation: + title: Test recipe + description: Recipe with no data. + authors: [andela_bouwe] + + diagnostics: + diagnostic_name: + scripts: + script_name: + script: {} + setting_name: {} + """.format(script_file, result_file)) + recipe_file.write_text(str(recipe)) + + write_config_file(config_dir) + with arguments( + 'esmvaltool', + 'run', + '--config_dir', + str(config_dir), str(recipe_file), ): run()