From 91b5dadf8b81b1624d1128834cc48248a726c656 Mon Sep 17 00:00:00 2001 From: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> Date: Tue, 18 Jun 2024 17:07:48 -0600 Subject: [PATCH] UW-391 Add IODA driver (#511) Adds IODA driver, docs, and schema --- docs/Makefile | 2 +- docs/index.rst | 6 + docs/sections/user_guide/api/index.rst | 1 + docs/sections/user_guide/api/ioda.rst | 5 + .../sections/user_guide/cli/drivers/index.rst | 1 + docs/sections/user_guide/cli/drivers/ioda.rst | 52 ++++++++ .../user_guide/cli/drivers/ioda/Makefile | 1 + .../user_guide/cli/drivers/ioda/help.cmd | 1 + .../user_guide/cli/drivers/ioda/help.out | 26 ++++ .../user_guide/cli/drivers/ioda/run-help.cmd | 1 + .../user_guide/cli/drivers/ioda/run-help.out | 30 +++++ .../user_guide/cli/drivers/jedi/help.out | 2 +- .../user_guide/yaml/components/index.rst | 1 + .../user_guide/yaml/components/ioda.rst | 41 +++++++ docs/shared/ioda.yaml | 25 ++++ docs/shared/jedi.yaml | 2 +- docs/shared/sfc_climo_gen.yaml | 2 +- docs/shared/upp.yaml | 2 +- src/uwtools/api/ioda.py | 12 ++ src/uwtools/cli.py | 2 + src/uwtools/drivers/ioda.py | 52 ++++++++ src/uwtools/drivers/jedi.py | 80 +----------- src/uwtools/drivers/jedi_base.py | 107 ++++++++++++++++ src/uwtools/drivers/mpas_base.py | 7 -- .../resources/jsonschema/ioda.jsonschema | 53 ++++++++ src/uwtools/strings.py | 1 + src/uwtools/tests/drivers/test_ioda.py | 115 ++++++++++++++++++ src/uwtools/tests/drivers/test_jedi.py | 10 +- src/uwtools/tests/test_schemas.py | 49 ++++++++ 29 files changed, 596 insertions(+), 93 deletions(-) create mode 100644 docs/sections/user_guide/api/ioda.rst create mode 100644 docs/sections/user_guide/cli/drivers/ioda.rst create mode 120000 docs/sections/user_guide/cli/drivers/ioda/Makefile create mode 100644 docs/sections/user_guide/cli/drivers/ioda/help.cmd create mode 100644 docs/sections/user_guide/cli/drivers/ioda/help.out create mode 100644 docs/sections/user_guide/cli/drivers/ioda/run-help.cmd create mode 100644 docs/sections/user_guide/cli/drivers/ioda/run-help.out create mode 100644 docs/sections/user_guide/yaml/components/ioda.rst create mode 100644 docs/shared/ioda.yaml create mode 100644 src/uwtools/api/ioda.py create mode 100644 src/uwtools/drivers/ioda.py create mode 100644 src/uwtools/drivers/jedi_base.py create mode 100644 src/uwtools/resources/jsonschema/ioda.jsonschema create mode 100644 src/uwtools/tests/drivers/test_ioda.py diff --git a/docs/Makefile b/docs/Makefile index fcbe5c56f..5f68e1b41 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -17,7 +17,7 @@ docs: $(MAKE) html examples: - $(MAKE) -C sections + COLUMNS=80 $(MAKE) -C sections linkcheck: $(SPHINXBUILD) -b linkcheck $(SPHINXOPTS) -c $(CURDIR) $(CURDIR) build/linkcheck diff --git a/docs/index.rst b/docs/index.rst index 5dfc63e79..3cbb280e9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -178,6 +178,12 @@ UPP Driver for JEDI ^^^^^^^^^^^^^^^ +IODA +"""" + +| **CLI**: ``uw ioda -h`` +| **API**: ``import uwtools.api.ioda`` + JEDI """" diff --git a/docs/sections/user_guide/api/index.rst b/docs/sections/user_guide/api/index.rst index f4666de9c..15bbbd764 100644 --- a/docs/sections/user_guide/api/index.rst +++ b/docs/sections/user_guide/api/index.rst @@ -9,6 +9,7 @@ API filter_topo fv3 global_equiv_resol + ioda jedi logging make_hgrid diff --git a/docs/sections/user_guide/api/ioda.rst b/docs/sections/user_guide/api/ioda.rst new file mode 100644 index 000000000..a5e5ae2dd --- /dev/null +++ b/docs/sections/user_guide/api/ioda.rst @@ -0,0 +1,5 @@ +``uwtools.api.ioda`` +==================== + +.. automodule:: uwtools.api.ioda + :members: diff --git a/docs/sections/user_guide/cli/drivers/index.rst b/docs/sections/user_guide/cli/drivers/index.rst index 5a32da3a0..45e97ffb6 100644 --- a/docs/sections/user_guide/cli/drivers/index.rst +++ b/docs/sections/user_guide/cli/drivers/index.rst @@ -9,6 +9,7 @@ Drivers filter_topo fv3 global_equiv_resol + ioda jedi make_hgrid make_solo_mosaic diff --git a/docs/sections/user_guide/cli/drivers/ioda.rst b/docs/sections/user_guide/cli/drivers/ioda.rst new file mode 100644 index 000000000..160d15ab6 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ioda.rst @@ -0,0 +1,52 @@ +``ioda`` +======== + +The ``uw`` mode for configuring and running the IODA components of the JEDI framework. + +.. literalinclude:: ioda/help.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: ioda/help.out + :language: text + +All tasks take the same arguments. For example: + +.. literalinclude:: ioda/run-help.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: ioda/run-help.out + :language: text + +Examples +^^^^^^^^ + +The examples use a configuration file named ``config.yaml`` with contents similar to: + +.. highlight:: yaml +.. literalinclude:: /shared/ioda.yaml + +Its contents are described in section :ref:`ioda_yaml`. + +* Run ``ioda`` on an interactive node + + .. code-block:: text + + $ uw ioda run --config-file config.yaml --cycle 2024-05-22T12 + +The driver creates a ``runscript.ioda`` file in the directory specified by ``run_dir:`` in the config and runs it, executing ``ioda``. + +* Run ``ioda`` via a batch job + + .. code-block:: text + + $ uw ioda run --config-file config.yaml --cycle 2024-05-22T12 --batch + +The driver creates a ``runscript.ioda`` file in the directory specified by ``run_dir:`` in the config and submits it to the batch system. Running with ``--batch`` requires a correctly configured ``platform:`` block in ``config.yaml``, as well as appropriate settings in the ``execution:`` block under ``ioda:``. + +* Specifying the ``--dry-run`` flag results in the driver logging messages about actions it would have taken, without actually taking any. + + .. code-block:: text + + $ uw ioda run --config-file config.yaml --cycle 2024-05-22T12 --batch --dry-run + +.. include:: /shared/key_path.rst diff --git a/docs/sections/user_guide/cli/drivers/ioda/Makefile b/docs/sections/user_guide/cli/drivers/ioda/Makefile new file mode 120000 index 000000000..2486334a6 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ioda/Makefile @@ -0,0 +1 @@ +../../Makefile.outputs \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/ioda/help.cmd b/docs/sections/user_guide/cli/drivers/ioda/help.cmd new file mode 100644 index 000000000..6950774cf --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ioda/help.cmd @@ -0,0 +1 @@ +uw ioda --help diff --git a/docs/sections/user_guide/cli/drivers/ioda/help.out b/docs/sections/user_guide/cli/drivers/ioda/help.out new file mode 100644 index 000000000..6ab9994ba --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ioda/help.out @@ -0,0 +1,26 @@ +usage: uw ioda [-h] [--version] TASK ... + +Execute ioda tasks + +Optional arguments: + -h, --help + Show help and exit + --version + Show version info and exit + +Positional arguments: + TASK + configuration_file + The executable's YAML configuration file + files_copied + Files copied for run + files_linked + Files linked for run + provisioned_run_directory + Run directory provisioned with all required content + run + A run + runscript + The runscript + validate + Validate the UW driver config diff --git a/docs/sections/user_guide/cli/drivers/ioda/run-help.cmd b/docs/sections/user_guide/cli/drivers/ioda/run-help.cmd new file mode 100644 index 000000000..94ff27e02 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ioda/run-help.cmd @@ -0,0 +1 @@ +uw ioda run --help diff --git a/docs/sections/user_guide/cli/drivers/ioda/run-help.out b/docs/sections/user_guide/cli/drivers/ioda/run-help.out new file mode 100644 index 000000000..fcbe6dd72 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ioda/run-help.out @@ -0,0 +1,30 @@ +usage: uw ioda run --cycle CYCLE [-h] [--version] [--config-file PATH] + [--batch] [--dry-run] [--graph-file PATH] + [--key-path KEY[.KEY...]] [--quiet] [--verbose] + +A run + +Required arguments: + --cycle CYCLE + The cycle in ISO8601 format (e.g. 2024-06-17T18) + +Optional arguments: + -h, --help + Show help and exit + --version + Show version info and exit + --config-file PATH, -c PATH + Path to UW YAML config file (default: read from stdin) + --batch + Submit run to batch scheduler + --dry-run + Only log info, making no changes + --graph-file PATH + Path to Graphviz DOT output [experimental] + --key-path KEY[.KEY...] + Dot-separated path of keys leading through the config to the driver's + configuration block + --quiet, -q + Print no logging messages + --verbose, -v + Print all logging messages diff --git a/docs/sections/user_guide/cli/drivers/jedi/help.out b/docs/sections/user_guide/cli/drivers/jedi/help.out index ba762a87c..847c99fb1 100644 --- a/docs/sections/user_guide/cli/drivers/jedi/help.out +++ b/docs/sections/user_guide/cli/drivers/jedi/help.out @@ -11,7 +11,7 @@ Optional arguments: Positional arguments: TASK configuration_file - The JEDI YAML configuration file + The executable's YAML configuration file files_copied Files copied for run files_linked diff --git a/docs/sections/user_guide/yaml/components/index.rst b/docs/sections/user_guide/yaml/components/index.rst index 7c53512cc..5ea9f9050 100644 --- a/docs/sections/user_guide/yaml/components/index.rst +++ b/docs/sections/user_guide/yaml/components/index.rst @@ -9,6 +9,7 @@ UW YAML for Components filter_topo fv3 global_equiv_resol + ioda jedi make_hgrid make_solo_mosaic diff --git a/docs/sections/user_guide/yaml/components/ioda.rst b/docs/sections/user_guide/yaml/components/ioda.rst new file mode 100644 index 000000000..2cfa1c32b --- /dev/null +++ b/docs/sections/user_guide/yaml/components/ioda.rst @@ -0,0 +1,41 @@ +.. _ioda_yaml: + +ioda +==== + +Structured YAML to run IODA is validated by JSON Schema and requires the ``ioda:`` block, described below. If ``ioda`` is to be run via a batch system, the ``platform:`` block, described :ref:`here `, is also required. + +.. include:: /shared/injected_cycle.rst + +Here is a prototype UW YAML ``ioda:`` block, explained in detail below: + +.. highlight:: yaml +.. literalinclude:: /shared/ioda.yaml + +UW YAML for the ``ioda:`` Block +------------------------------- + +execution: +^^^^^^^^^^ + +See :ref:`this page ` for details. + +configuration_file: +^^^^^^^^^^^^^^^^^^^ + +Supports ``base_file:`` and ``update_values:`` blocks (see :ref:`updating_values` for details). + +files_to_copy: +^^^^^^^^^^^^^^ + +See :ref:`this page ` for details. + +files_to_link: +^^^^^^^^^^^^^^ + +Identical to ``files_to_copy:`` except that symbolic links will be created in the run directory instead of copies. + +run_dir: +^^^^^^^^ + +The path to the run directory. diff --git a/docs/shared/ioda.yaml b/docs/shared/ioda.yaml new file mode 100644 index 000000000..ac4f4c351 --- /dev/null +++ b/docs/shared/ioda.yaml @@ -0,0 +1,25 @@ +ioda: + configuration_file: + base_file: path/to/config.yaml + update_values: + baz: qux + execution: + batchargs: + nodes: 1 + stdout: path/to/runscript.out + walltime: "00:05:00" + envcmds: + - module load some-module + - module load ioda-module + executable: /path/to/a/ioda/exe + mpicmd: time + files_to_copy: + d/f2: /path/to/f2 + f1: /path/to/f1 + files_to_link: + f3: /path/to/f3 + f4: d/f4 + run_dir: /path/to/run/dir +platform: + account: me + scheduler: slurm diff --git a/docs/shared/jedi.yaml b/docs/shared/jedi.yaml index 19215ef73..088281934 100644 --- a/docs/shared/jedi.yaml +++ b/docs/shared/jedi.yaml @@ -22,7 +22,7 @@ jedi: files_to_link: f3: /path/to/f3 f4: d/f4 - run_dir: /path/to/run + run_dir: /path/to/run/dir platform: account: me scheduler: slurm diff --git a/docs/shared/sfc_climo_gen.yaml b/docs/shared/sfc_climo_gen.yaml index 4d231ba55..8ba912c26 100644 --- a/docs/shared/sfc_climo_gen.yaml +++ b/docs/shared/sfc_climo_gen.yaml @@ -33,7 +33,7 @@ sfc_climo_gen: snowfree_albedo_method: bilinear vegetation_greenness_method: bilinear validate: true - run_dir: /path/to/run + run_dir: /path/to/run/dir platform: account: me scheduler: slurm diff --git a/docs/shared/upp.yaml b/docs/shared/upp.yaml index 7bb92efa3..a28f60e4f 100644 --- a/docs/shared/upp.yaml +++ b/docs/shared/upp.yaml @@ -35,7 +35,7 @@ upp: - 100 - 1 validate: true - run_dir: /path/to/run + run_dir: /path/to/run/dir platform: account: me scheduler: slurm diff --git a/src/uwtools/api/ioda.py b/src/uwtools/api/ioda.py new file mode 100644 index 000000000..cc3a5174e --- /dev/null +++ b/src/uwtools/api/ioda.py @@ -0,0 +1,12 @@ +""" +API access to the ``uwtools`` ``ioda`` driver. +""" + +from uwtools.drivers.ioda import IODA as _Driver +from uwtools.drivers.support import graph +from uwtools.utils.api import make_execute as _make_execute +from uwtools.utils.api import make_tasks as _make_tasks + +execute = _make_execute(_Driver, with_cycle=True) +tasks = _make_tasks(_Driver) +__all__ = ["execute", "graph", "tasks"] diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index a9f82308e..8298f3ce6 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -69,6 +69,7 @@ def main() -> None: STR.filtertopo, STR.fv3, STR.globalequivresol, + STR.ioda, STR.jedi, STR.makehgrid, STR.makesolomosaic, @@ -1046,6 +1047,7 @@ def _parse_args(raw_args: List[str]) -> Tuple[Args, Checks]: for component in [ STR.chgrescube, STR.fv3, + STR.ioda, STR.jedi, STR.mpas, STR.mpasinit, diff --git a/src/uwtools/drivers/ioda.py b/src/uwtools/drivers/ioda.py new file mode 100644 index 000000000..24dff3c41 --- /dev/null +++ b/src/uwtools/drivers/ioda.py @@ -0,0 +1,52 @@ +""" +A driver for the ioda component. +""" + +from iotaa import tasks + +from uwtools.drivers.jedi_base import JEDIBase +from uwtools.strings import STR + + +class IODA(JEDIBase): + """ + A driver for the IODA component. + """ + + @tasks + def provisioned_run_directory(self): + """ + Run directory provisioned with all required content. + """ + yield self._taskname("provisioned run directory") + yield [ + self.configuration_file(), + self.files_copied(), + self.files_linked(), + self.runscript(), + ] + + # Private helper methods + + @property + def _config_fn(self) -> str: + """ + Returns the name of the config file used in execution. + """ + return "ioda.yaml" + + @property + def _driver_name(self) -> str: + """ + Returns the name of this driver. + """ + return STR.ioda + + @property + def _runcmd(self) -> str: + """ + Returns the full command-line component invocation. + """ + executable = self._driver_config["execution"]["executable"] + jedi_config = str(self._rundir / self._config_fn) + return " ".join([executable, jedi_config]) diff --git a/src/uwtools/drivers/jedi.py b/src/uwtools/drivers/jedi.py index b44a2ed10..7e83c5941 100644 --- a/src/uwtools/drivers/jedi.py +++ b/src/uwtools/drivers/jedi.py @@ -3,86 +3,22 @@ """ import logging -from datetime import datetime from pathlib import Path -from typing import List, Optional from iotaa import asset, refs, run, task, tasks -from uwtools.config.formats.yaml import YAMLConfig -from uwtools.drivers.driver import Driver +from uwtools.drivers.jedi_base import JEDIBase from uwtools.strings import STR -from uwtools.utils.tasks import file, filecopy, symlink +from uwtools.utils.tasks import file -class JEDI(Driver): +class JEDI(JEDIBase): """ A driver for the JEDI component. """ - def __init__( - self, - cycle: datetime, - config: Optional[Path] = None, - dry_run: bool = False, - batch: bool = False, - key_path: Optional[List[str]] = None, - ): - """ - The driver. - - :param cycle: The forecast cycle. - :param config: Path to config file. - :param dry_run: Run in dry-run mode? - :param batch: Run component via the batch system? - :param key_path: Keys leading through the config to the driver's configuration block. - """ - super().__init__( - config=config, dry_run=dry_run, batch=batch, cycle=cycle, key_path=key_path - ) - self._cycle = cycle - # Workflow tasks - @task - def configuration_file(self): - """ - The JEDI YAML configuration file. - """ - fn = self._config_fn - yield self._taskname(fn) - path = self._rundir / fn - yield asset(path, path.is_file) - base_file = self._driver_config["configuration_file"].get("base_file") - yield file(Path(base_file)) if base_file else None - self._create_user_updated_config( - config_class=YAMLConfig, - config_values=self._driver_config["configuration_file"], - path=path, - ) - - @tasks - def files_copied(self): - """ - Files copied for run. - """ - yield self._taskname("files copied") - yield [ - filecopy(src=Path(src), dst=self._rundir / dst) - for dst, src in self._driver_config.get("files_to_copy", {}).items() - ] - - @tasks - def files_linked(self): - """ - Files linked for run. - """ - yield self._taskname("files linked") - yield [ - symlink(target=Path(target), linkname=self._rundir / linkname) - for linkname, target in self._driver_config.get("files_to_link", {}).items() - ] - @tasks def provisioned_run_directory(self): """ @@ -138,7 +74,7 @@ def _runcmd(self) -> str: """ Returns the full command-line component invocation. """ - execution = self._driver_config.get("execution", {}) + execution = self._driver_config["execution"] jedi_config = self._rundir / self._config_fn mpiargs = execution.get("mpiargs", []) components = [ @@ -148,11 +84,3 @@ def _runcmd(self) -> str: str(jedi_config), # JEDI config file ] return " ".join(filter(None, components)) - - def _taskname(self, suffix: str) -> str: - """ - Returns a common tag for graph-task log messages. - - :param suffix: Log-string suffix. - """ - return self._taskname_with_cycle(self._cycle, suffix) diff --git a/src/uwtools/drivers/jedi_base.py b/src/uwtools/drivers/jedi_base.py new file mode 100644 index 000000000..9418ec482 --- /dev/null +++ b/src/uwtools/drivers/jedi_base.py @@ -0,0 +1,107 @@ +""" +A base class for jedi-based drivers. +""" + +from abc import abstractmethod +from datetime import datetime +from pathlib import Path +from typing import List, Optional + +from iotaa import asset, task, tasks + +from uwtools.config.formats.yaml import YAMLConfig +from uwtools.drivers.driver import Driver +from uwtools.utils.tasks import file, filecopy, symlink + + +class JEDIBase(Driver): + """ + A base class for the JEDI-like drivers. + """ + + def __init__( + self, + cycle: datetime, + config: Optional[Path] = None, + dry_run: bool = False, + batch: bool = False, + key_path: Optional[List[str]] = None, + ): + """ + The driver. + + :param cycle: The forecast cycle. + :param config: Path to config file. + :param dry_run: Run in dry-run mode? + :param batch: Run component via the batch system? + :param key_path: Keys leading through the config to the driver's configuration block. + """ + super().__init__( + config=config, dry_run=dry_run, batch=batch, cycle=cycle, key_path=key_path + ) + self._cycle = cycle + + # Workflow tasks + + @task + def configuration_file(self): + """ + The executable's YAML configuration file. + """ + fn = self._config_fn + yield self._taskname(fn) + path = self._rundir / fn + yield asset(path, path.is_file) + base_file = self._driver_config["configuration_file"].get("base_file") + yield file(Path(base_file)) if base_file else None + self._create_user_updated_config( + config_class=YAMLConfig, + config_values=self._driver_config["configuration_file"], + path=path, + ) + + @tasks + def files_copied(self): + """ + Files copied for run. + """ + yield self._taskname("files copied") + yield [ + filecopy(src=Path(src), dst=self._rundir / dst) + for dst, src in self._driver_config.get("files_to_copy", {}).items() + ] + + @tasks + def files_linked(self): + """ + Files linked for run. + """ + yield self._taskname("files linked") + yield [ + symlink(target=Path(target), linkname=self._rundir / linkname) + for linkname, target in self._driver_config.get("files_to_link", {}).items() + ] + + @tasks + @abstractmethod + def provisioned_run_directory(self): + """ + Run directory provisioned with all required content. + """ + + # Private helper methods + + @property + @abstractmethod + def _config_fn(self) -> str: + """ + Returns the name of the config file used in execution. + """ + + def _taskname(self, suffix: str) -> str: + """ + Returns a common tag for graph-task log messages. + + :param suffix: Log-string suffix. + """ + return self._taskname_with_cycle(self._cycle, suffix) diff --git a/src/uwtools/drivers/mpas_base.py b/src/uwtools/drivers/mpas_base.py index 253635218..a72be4dd7 100644 --- a/src/uwtools/drivers/mpas_base.py +++ b/src/uwtools/drivers/mpas_base.py @@ -134,13 +134,6 @@ def streams_file(self): # Private helper methods - @property - @abstractmethod - def _driver_name(self) -> str: - """ - Returns the name of this driver. - """ - @property @abstractmethod def _streams_fn(self) -> str: diff --git a/src/uwtools/resources/jsonschema/ioda.jsonschema b/src/uwtools/resources/jsonschema/ioda.jsonschema new file mode 100644 index 000000000..be74a2635 --- /dev/null +++ b/src/uwtools/resources/jsonschema/ioda.jsonschema @@ -0,0 +1,53 @@ +{ + "properties": { + "ioda": { + "additionalProperties": false, + "properties": { + "configuration_file": { + "additionalProperties": false, + "anyOf": [ + { + "required": [ + "base_file" + ] + }, + { + "required": [ + "update_values" + ] + } + ], + "properties": { + "base_file": { + "type": "string" + }, + "update_values": { + "minProperties": 1, + "type": "object" + } + }, + "type": "object" + }, + "execution": { + "$ref": "urn:uwtools:execution-serial" + }, + "files_to_copy": { + "$ref": "urn:uwtools:files-to-stage" + }, + "files_to_link": { + "$ref": "urn:uwtools:files-to-stage" + }, + "run_dir": { + "type": "string" + } + }, + "required": [ + "configuration_file", + "execution", + "run_dir" + ], + "type": "object" + } + }, + "type": "object" +} diff --git a/src/uwtools/strings.py b/src/uwtools/strings.py index ac76ce501..ffd14bb67 100644 --- a/src/uwtools/strings.py +++ b/src/uwtools/strings.py @@ -84,6 +84,7 @@ class STR: help: str = "help" infile: str = "input_file" infmt: str = "input_format" + ioda: str = "ioda" jedi: str = "jedi" keypath: str = "key_path" keys: str = "keys" diff --git a/src/uwtools/tests/drivers/test_ioda.py b/src/uwtools/tests/drivers/test_ioda.py new file mode 100644 index 000000000..1394322c5 --- /dev/null +++ b/src/uwtools/tests/drivers/test_ioda.py @@ -0,0 +1,115 @@ +# pylint: disable=missing-function-docstring,protected-access,redefined-outer-name +""" +IODA driver tests. +""" +import datetime as dt +from unittest.mock import DEFAULT as D +from unittest.mock import patch + +from pytest import fixture + +from uwtools.drivers.ioda import IODA +from uwtools.drivers.jedi_base import JEDIBase + +# Fixtures + + +@fixture +def config(tmp_path): + base_file = tmp_path / "base.yaml" + base_file.write_text("foo: bar") + return { + "ioda": { + "execution": { + "batchargs": { + "export": "NONE", + "cores": 1, + "stdout": "/path/to/file", + "walltime": "00:02:00", + }, + "envcmds": [ + "module load some-module", + "module load jedi-module", + ], + "executable": "/path/to/bufr2ioda.x", + }, + "configuration_file": { + "base_file": str(base_file), + "update_values": {"baz": "qux"}, + }, + "files_to_copy": { + "foo": "/path/to/foo", + "bar/baz": "/path/to/baz", + }, + "files_to_link": { + "foo": "/path/to/foo", + "bar/baz": "/path/to/baz", + }, + "run_dir": str(tmp_path), + }, + "platform": { + "account": "me", + "scheduler": "slurm", + }, + } + + +@fixture +def cycle(): + return dt.datetime(2024, 5, 1, 6) + + +@fixture +def driverobj(config, cycle): + return IODA(config=config, cycle=cycle, batch=True) + + +# Tests + + +def test_IODA(): + for method in [ + "_driver_config", + "_resources", + "_run_via_batch_submission", + "_run_via_local_execution", + "_runscript", + "_runscript_done_file", + "_runscript_path", + "_scheduler", + "_validate", + "_write_runscript", + "run", + "runscript", + ]: + assert getattr(IODA, method) is getattr(JEDIBase, method) + + +def test_IODA_provisioned_run_directory(driverobj): + with patch.multiple( + driverobj, + configuration_file=D, + files_copied=D, + files_linked=D, + runscript=D, + ) as mocks: + driverobj.provisioned_run_directory() + for m in mocks: + mocks[m].assert_called_once_with() + + +def test_IODA__config_fn(driverobj): + assert driverobj._config_fn == "ioda.yaml" + + +def test_IODA__driver_name(driverobj): + assert driverobj._driver_name == "ioda" + + +def test_IODA__runcmd(driverobj): + config = str(driverobj._rundir / driverobj._config_fn) + assert driverobj._runcmd == f"/path/to/bufr2ioda.x {config}" + + +def test_IODA__taskname(driverobj): + assert driverobj._taskname("foo") == "20240501 06Z ioda foo" diff --git a/src/uwtools/tests/drivers/test_jedi.py b/src/uwtools/tests/drivers/test_jedi.py index 3a0853225..26a79929f 100644 --- a/src/uwtools/tests/drivers/test_jedi.py +++ b/src/uwtools/tests/drivers/test_jedi.py @@ -13,9 +13,9 @@ from pytest import fixture from uwtools.config.formats.yaml import YAMLConfig -from uwtools.drivers import jedi -from uwtools.drivers.driver import Driver +from uwtools.drivers import jedi, jedi_base from uwtools.drivers.jedi import JEDI +from uwtools.drivers.jedi_base import JEDIBase from uwtools.logging import log from uwtools.tests.support import regex_logged @@ -92,7 +92,7 @@ def test_JEDI(): "run", "runscript", ]: - assert getattr(JEDI, method) is getattr(Driver, method) + assert getattr(JEDI, method) is getattr(JEDIBase, method) def test_JEDI_configuration_file(driverobj): @@ -120,7 +120,7 @@ def test_JEDI_configuration_file_missing_base_file(caplog, driverobj): def test_JEDI_files_copied(driverobj): - with patch.object(jedi, "filecopy") as filecopy: + with patch.object(jedi_base, "filecopy") as filecopy: driverobj._driver_config["run_dir"] = "/path/to/run" driverobj.files_copied() assert filecopy.call_count == 2 @@ -134,7 +134,7 @@ def test_JEDI_files_copied(driverobj): def test_JEDI_files_linked(driverobj): - with patch.object(jedi, "symlink") as symlink: + with patch.object(jedi_base, "symlink") as symlink: driverobj._driver_config["run_dir"] = "/path/to/run" driverobj.files_linked() assert symlink.call_count == 2 diff --git a/src/uwtools/tests/test_schemas.py b/src/uwtools/tests/test_schemas.py index 0b187ff55..a99e8d152 100644 --- a/src/uwtools/tests/test_schemas.py +++ b/src/uwtools/tests/test_schemas.py @@ -54,6 +54,11 @@ def global_equiv_resol_prop(): ) +@fixture +def ioda_prop(): + return partial(schema_validator, "ioda", "properties", "ioda", "properties") + + @fixture def jedi_prop(): return partial(schema_validator, "jedi", "properties", "jedi", "properties") @@ -642,6 +647,50 @@ def test_schema_global_equiv_resol_paths(global_equiv_resol_prop, schema_entry): assert "88 is not of type 'string'" in errors(88) +# ioda + + +def test_schema_ioda(): + config = { + "configuration_file": { + "base_file": "/path/to/ioda.yaml", + "update_values": {"foo": "bar", "baz": "qux"}, + }, + "execution": {"executable": "/tmp/ioda.exe"}, + "files_to_copy": {"file1": "src1", "file2": "src2"}, + "files_to_link": {"link1": "src3", "link2": "src4"}, + "run_dir": "/tmp", + } + errors = schema_validator("ioda", "properties", "ioda") + # Basic correctness: + assert not errors(config) + # All top-level keys are required: + for key in ("configuration_file", "execution", "run_dir"): + assert f"'{key}' is a required property" in errors(with_del(config, key)) + # Additional top-level keys are not allowed: + assert "Additional properties are not allowed" in errors({**config, "foo": "bar"}) + + +def test_schema_ioda_configuration_file(ioda_prop): + bf = {"base_file": "/path/to/ioda.yaml"} + uv = {"update_values": {"foo": "bar", "baz": "qux"}} + errors = ioda_prop("configuration_file") + # base_file and update_values are ok together: + assert not errors({**bf, **uv}) + # And either is ok alone: + assert not errors(bf) + assert not errors(uv) + # update_values cannot be empty: + assert "should be non-empty" in errors({"update_values": {}}) + + +def test_schema_ioda_run_dir(ioda_prop): + errors = ioda_prop("run_dir") + # Must be a string: + assert not errors("/some/path") + assert "88 is not of type 'string'" in errors(88) + + # jedi