UW-559 Add argument for directing drivers to an arbitrary block of an…

… experiment YAML. (ufs-community#490)

* Initial WIP, add key_path to CLI

* Update keypath type

* WIP

* Add key_path arg to drivers

* Reformat files

* Correcting the type of key_path arg

* WIP

* Add key_path arg to drivers

* WIP

* Add key_path arg to api

* Add key_path arg to api

* Add test coverage

* Update key_path arg docstring

* Reformat and test coverage 100%

* Update documentation

* Update docstring

* Update docs

* WIP

* Update docs

* WIP

* WIP

* Update docs/shared/key_path.rst

Co-authored-by: Paul Madden <[email protected]>

* Update docs/shared/key_path.rst

Co-authored-by: Paul Madden <[email protected]>

* Update docs/shared/key_path.rst

Co-authored-by: Paul Madden <[email protected]>

* Add key_path documentation to drivers

* Update cli

* Update docs

* Update docs

* Update docs

* Update test case

---------

Co-authored-by: Paul Madden <[email protected]>

docs/sections/user_guide/cli/drivers/chgres_cube.rst

@@ -93,6 +93,9 @@ Its contents are described in depth in section :ref:`chgres_cube_yaml`. Each of
 
      $ uw chgres_cube run --config-file config.yaml --cycle 2023-12-15T18 --batch --dry-run
 
+
+.. include:: ../../../../shared/key_path.rst
+
 * The ``run`` task depends on the other available tasks and executes them as prerequisites. It is possible to execute any task directly, which entails execution of any of *its* dependencies. For example, to create a ``chgres_cube`` run directory provisioned with all the files, directories, symlinks, etc. required per the configuration file:
 
   .. code-block:: text

docs/sections/user_guide/cli/drivers/esg_grid.rst

@@ -89,6 +89,8 @@ The driver creates a ``runscript.esg_grid`` file in the directory specified by `
 
      $ uw esg_grid run --config-file config.yaml --batch --dry-run
 
+.. include:: ../../../../shared/key_path.rst
+
 * The ``run`` task depends on the other available tasks and executes them as prerequisites. It is possible to execute any task directly, which entails execution of any of *its* dependencies. For example, to create an ``esg_grid`` run directory provisioned with all the files, directories, symlinks, etc. required per the configuration file:
 
   .. code-block:: text

docs/sections/user_guide/cli/drivers/fv3.rst

@@ -102,6 +102,8 @@ The examples use a configuration file named ``config.yaml``. Its contents are de
 
      $ uw fv3 run --config-file config.yaml --cycle 2024-02-11T12 --batch --dry-run
 
+.. include:: ../../../../shared/key_path.rst
+
 * The ``run`` task depends on the other available tasks and executes them as prerequisites. It is possible to execute any task directly, which entails execution of any of *its* dependencies. For example, to create an FV3 run directory provisioned with all the files, directories, symlinks, etc. required per the configuration file:
 
   .. code-block:: text

docs/sections/user_guide/cli/drivers/global_equiv_resol.rst

@@ -88,3 +88,5 @@ Its contents are described in section :ref:`global_equiv_resol_yaml`.
   .. code-block:: text
 
      $ uw global_equiv_resol run --config-file config.yaml --batch --dry-run
+
+.. include:: ../../../../shared/key_path.rst

docs/sections/user_guide/cli/drivers/jedi.rst

@@ -81,20 +81,22 @@ Its contents are described in section :ref:`jedi_yaml`.
 
    .. code-block:: text
 
-      $ uw jedi run --config-file config.yaml
+      $ uw jedi run --config-file config.yaml --cycle 2024-05-22T12
 
 The driver creates a ``runscript.jedi`` file in the directory specified by ``run_dir:`` in the config and runs it, executing ``jedi``.
 
 * Run ``jedi`` via a batch job
 
    .. code-block:: text
 
-      $ uw jedi run --config-file config.yaml --batch
+      $ uw jedi run --config-file config.yaml --cycle 2024-05-22T12 --batch
 
 The driver creates a ``runscript.jedi`` file in the directory specified by ``run_dir:`` in the config and submits it to the batch system. Running with ``--batch`` requires a correctly configured ``platform:`` block in ``config.yaml``, as well as appropriate settings in the ``execution:`` block under ``jedi:``.
 
 * Specifying the ``--dry-run`` flag results in the driver logging messages about actions it would have taken, without actually taking any.
 
   .. code-block:: text
 
-     $ uw jedi run --config-file config.yaml --batch --dry-run
+     $ uw jedi run --config-file config.yaml --cycle 2024-05-22T12 --batch --dry-run
+
+.. include:: ../../../../shared/key_path.rst

docs/sections/user_guide/cli/drivers/make_hgrid.rst

@@ -91,3 +91,5 @@ Its contents are described in section :ref:`make_hgrid_yaml`.
   .. code-block:: text
 
      $ uw make_hgrid run --config-file config.yaml --batch --dry-run
+
+.. include:: ../../../../shared/key_path.rst

docs/sections/user_guide/cli/drivers/make_solo_mosaic.rst

@@ -92,3 +92,5 @@ Its contents are described in section :ref:`make_solo_mosaic_yaml`.
   .. code-block:: text
 
      $ uw make_solo_mosaic run --config-file config.yaml --batch --dry-run
+
+.. include:: ../../../../shared/key_path.rst

docs/sections/user_guide/cli/drivers/mpas.rst

@@ -101,6 +101,8 @@ Its contents are described in depth in section :ref:`mpas_yaml`.
 
      $ uw mpas run --config-file config.yaml --cycle 2025-02-12T12 --batch --dry-run
 
+.. include:: ../../../../shared/key_path.rst
+
 * The ``run`` task depends on the other available tasks and executes them as prerequisites. It is possible to execute any task directly, which entails execution of any of *its* dependencies. For example, to create an ``mpas`` run directory provisioned with all the files, directories, symlinks, etc. required per the configuration file:
 
   .. code-block:: text

docs/sections/user_guide/cli/drivers/mpas_init.rst

@@ -101,6 +101,8 @@ Its contents are described in depth in section :ref:`mpas_init_yaml`.
 
      $ uw mpas_init run --config-file config.yaml --cycle 2023-12-18T00 --batch --dry-run
 
+.. include:: ../../../../shared/key_path.rst
+
 * The ``run`` task depends on the other available tasks and executes them as prerequisites. It is possible to execute any task directly, which entails execution of any of *its* dependencies. For example, to create an ``mpas_init`` run directory provisioned with all the files, directories, symlinks, etc. required per the configuration file:
 
   .. code-block:: text

docs/sections/user_guide/cli/drivers/sfc_climo_gen.rst

@@ -89,6 +89,8 @@ Its contents are described in depth in section :ref:`sfc_climo_gen_yaml`.
 
      $ uw sfc_climo_gen run --config-file config.yaml --batch --dry-run
 
+.. include:: ../../../../shared/key_path.rst
+
 * The ``run`` task depends on the other available tasks and executes them as prerequisites. It is possible to execute any task directly, which entails execution of any of *its* dependencies. For example, to create an ``sfc_climo_gen`` run directory provisioned with all the files, directories, symlinks, etc. required per the configuration file:
 
   .. code-block:: text

docs/sections/user_guide/cli/drivers/shave.rst

@@ -86,3 +86,5 @@ Its contents are described in section :ref:`shave_yaml`.
   .. code-block:: text
 
      $ uw shave run --config-file config.yaml --batch --dry-run
+
+.. include:: ../../../../shared/key_path.rst

docs/sections/user_guide/cli/drivers/ungrib.rst

@@ -97,6 +97,8 @@ Its contents are described in depth in section :ref:`ungrib_yaml`.
 
      $ uw ungrib run --config-file config.yaml --cycle 2021-04-01T12 --batch --dry-run
 
+.. include:: ../../../../shared/key_path.rst
+
 * The ``run`` task depends on the other available tasks and executes them as prerequisites. It is possible to execute any task directly, which entails execution of any of *its* dependencies. For example, to create an ``ungrib`` run directory provisioned with all the files, directories, symlinks, etc. required per the configuration file:
 
   .. code-block:: text

docs/sections/user_guide/cli/drivers/upp.rst

@@ -99,6 +99,8 @@ Its contents are described in depth in section :ref:`upp_yaml`.
 
      $ uw upp run --config-file config.yaml --cycle 2024-05-06T12 --leadtime 6 --batch --dry-run
 
+.. include:: ../../../../shared/key_path.rst
+
 * The ``run`` task depends on the other available tasks and executes them as prerequisites. It is possible to execute any task directly, which entails execution of any of *its* dependencies. For example, to create an ``upp`` run directory provisioned with all the files, directories, symlinks, etc. required per the configuration file:
 
   .. code-block:: text

docs/shared/key_path.rst

@@ -0,0 +1,11 @@
+* The ``--key-path`` option can be used to navigate from the top of the config to the driver's configuration block. For example, specifying ``--key-path foo.bar`` with config
+
+  .. highlight:: yaml
+  .. literalinclude:: /shared/key_path_nested.yaml
+
+  is equivalent to using config
+
+  .. highlight:: yaml
+  .. literalinclude:: /shared/key_path_simple.yaml
+
+  without specifying ``--key-path``.

docs/shared/key_path_nested.yaml

@@ -0,0 +1,4 @@
+foo:
+  bar:
+    driver:
+      # driver config block

docs/shared/key_path_simple.yaml

@@ -0,0 +1,2 @@
+driver:
+  # driver config block

src/uwtools/cli.py

@@ -530,7 +530,7 @@ def _dispatch_template_translate(args: Args) -> bool:
 
 # Arguments
 
-# pylint: disable=missing-function-docstring
+# pylint: disable=line-too-long, missing-function-docstring
 
 
 def _add_arg_batch(group: Group) -> None:
@@ -641,6 +641,16 @@ def _add_arg_key_eq_val_pairs(group: Group) -> None:
     )
 
 
+def _add_arg_key_path(group: Group) -> None:
+    group.add_argument(
+        _switch(STR.keypath),
+        help="Dot-separated path of keys leading through the config to the driver's configuration block",
+        metavar="KEY[.KEY...]",
+        required=False,
+        type=lambda s: s.split("."),
+    )
+
+
 def _add_arg_keys(group: Group) -> None:
     group.add_argument(
         STR.keys,
@@ -888,6 +898,7 @@ def _add_subparser_for_driver_task(
     _add_arg_batch(optional)
     _add_arg_dry_run(optional)
     _add_arg_graph_file(optional)
+    _add_arg_key_path(optional)
     checks = _add_args_verbosity(optional)
     return checks
 
@@ -979,6 +990,7 @@ def _dispatch_to_driver(name: str, args: Args) -> bool:
         "batch": args[STR.batch],
         "dry_run": args[STR.dryrun],
         "graph_file": args[STR.graphfile],
+        "key_path": args[STR.keypath],
         "stdin_ok": True,
     }
     if cycle := args.get(STR.cycle):

src/uwtools/drivers/chgres_cube.py

@@ -4,7 +4,7 @@
 
 from datetime import datetime
 from pathlib import Path
-from typing import Optional
+from typing import List, Optional
 
 from iotaa import asset, task, tasks
 
@@ -25,6 +25,7 @@ def __init__(
         config: Optional[Path] = None,
         dry_run: bool = False,
         batch: bool = False,
+        key_path: Optional[List[str]] = None,
     ):
         """
         The driver.
@@ -33,8 +34,11 @@ def __init__(
         :param config: Path to config file (read stdin if missing or None).
         :param dry_run: Run in dry-run mode?
         :param batch: Run component via the batch system?
+        :param key_path: Keys leading through the config to the driver's configuration block.
         """
-        super().__init__(config=config, dry_run=dry_run, batch=batch, cycle=cycle)
+        super().__init__(
+            config=config, dry_run=dry_run, batch=batch, cycle=cycle, key_path=key_path
+        )
         self._cycle = cycle
 
     # Workflow tasks

src/uwtools/drivers/driver.py

@@ -9,7 +9,7 @@
 from datetime import datetime, timedelta
 from pathlib import Path
 from textwrap import dedent
-from typing import Any, Dict, List, Optional, Type
+from typing import Any, Dict, List, Optional, Type, Union
 
 from iotaa import asset, dryrun, external, task, tasks
 
@@ -29,10 +29,11 @@ class Driver(ABC):
 
     def __init__(
         self,
-        config: Optional[Path] = None,
+        config: Optional[Union[dict, Path]],
         dry_run: bool = False,
         batch: bool = False,
         cycle: Optional[datetime] = None,
+        key_path: Optional[List[str]] = None,
         leadtime: Optional[timedelta] = None,
     ) -> None:
         """
@@ -42,6 +43,7 @@ def __init__(
         :param dry_run: Run in dry-run mode?
         :param batch: Run component via the batch system?
         :param cycle: The cycle.
+        :param key_path: Keys leading through the config to the driver's configuration block.
         :param leadtime: The leadtime.
         """
         dryrun(enable=dry_run)
@@ -56,6 +58,9 @@ def __init__(
                 **self._config.data,
             }
         )
+        key_path = key_path or []
+        for key in key_path:
+            self._config = self._config[key]
         self._validate()
 
     # Workflow tasks

src/uwtools/drivers/esg_grid.py

@@ -3,7 +3,7 @@
 """
 
 from pathlib import Path
-from typing import Optional
+from typing import List, Optional
 
 from iotaa import asset, task, tasks
 
@@ -22,15 +22,17 @@ def __init__(
         config: Optional[Path] = None,
         dry_run: bool = False,
         batch: bool = False,
+        key_path: Optional[List[str]] = None,
     ):
         """
         The driver.
 
         :param config: Path to config file (read stdin if missing or None).
         :param dry_run: Run in dry-run mode?
         :param batch: Run component via the batch system?
+        :param key_path: Keys leading through the config to the driver's configuration block.
         """
-        super().__init__(config=config, dry_run=dry_run, batch=batch)
+        super().__init__(config=config, dry_run=dry_run, batch=batch, key_path=key_path)
 
     # Workflow tasks
 

src/uwtools/drivers/fv3.py

@@ -5,7 +5,7 @@
 from datetime import datetime
 from pathlib import Path
 from shutil import copy
-from typing import Optional
+from typing import List, Optional
 
 from iotaa import asset, task, tasks
 
@@ -28,6 +28,7 @@ def __init__(
         config: Optional[Path] = None,
         dry_run: bool = False,
         batch: bool = False,
+        key_path: Optional[List[str]] = None,
     ):
         """
         The driver.
@@ -36,8 +37,11 @@ def __init__(
         :param config: Path to config file (read stdin if missing or None).
         :param dry_run: Run in dry-run mode?
         :param batch: Run component via the batch system?
+        :param key_path: Keys leading through the config to the driver's configuration block.
         """
-        super().__init__(config=config, dry_run=dry_run, batch=batch, cycle=cycle)
+        super().__init__(
+            config=config, dry_run=dry_run, batch=batch, cycle=cycle, key_path=key_path
+        )
         self._cycle = cycle
 
     # Workflow tasks

src/uwtools/drivers/global_equiv_resol.py

@@ -3,7 +3,7 @@
 """
 
 from pathlib import Path
-from typing import Optional
+from typing import List, Optional
 
 from iotaa import asset, external, task, tasks
 
@@ -21,15 +21,17 @@ def __init__(
         config: Optional[Path] = None,
         dry_run: bool = False,
         batch: bool = False,
+        key_path: Optional[List[str]] = None,
     ):
         """
         The driver.
 
         :param config: Path to config file (read stdin if missing or None).
         :param dry_run: Run in dry-run mode?
         :param batch: Run component via the batch system?
+        :param key_path: Keys leading through the config to the driver's configuration block.
         """
-        super().__init__(config=config, dry_run=dry_run, batch=batch)
+        super().__init__(config=config, dry_run=dry_run, batch=batch, key_path=key_path)
 
     # Workflow tasks
 

src/uwtools/drivers/jedi.py

@@ -5,7 +5,7 @@
 import logging
 from datetime import datetime
 from pathlib import Path
-from typing import Optional
+from typing import List, Optional
 
 from iotaa import asset, refs, run, task, tasks
 
@@ -26,6 +26,7 @@ def __init__(
         config: Optional[Path] = None,
         dry_run: bool = False,
         batch: bool = False,
+        key_path: Optional[List[str]] = None,
     ):
         """
         The driver.
@@ -34,8 +35,11 @@ def __init__(
         :param config: Path to config file.
         :param dry_run: Run in dry-run mode?
         :param batch: Run component via the batch system?
+        :param key_path: Keys leading through the config to the driver's configuration block.
         """
-        super().__init__(config=config, dry_run=dry_run, batch=batch, cycle=cycle)
+        super().__init__(
+            config=config, dry_run=dry_run, batch=batch, cycle=cycle, key_path=key_path
+        )
         self._cycle = cycle
 
     # Workflow tasks

src/uwtools/drivers/make_hgrid.py

@@ -3,7 +3,7 @@
 """
 
 from pathlib import Path
-from typing import Optional
+from typing import List, Optional
 
 from iotaa import asset, task, tasks
 
@@ -21,15 +21,17 @@ def __init__(
         config: Optional[Path] = None,
         dry_run: bool = False,
         batch: bool = False,
+        key_path: Optional[List[str]] = None,
     ):
         """
         The driver.
 
         :param config: Path to config file (read stdin if missing or None).
         :param dry_run: Run in dry-run mode?
         :param batch: Run component via the batch system?
+        :param key_path: Keys leading through the config to the driver's configuration block.
         """
-        super().__init__(config=config, dry_run=dry_run, batch=batch)
+        super().__init__(config=config, dry_run=dry_run, batch=batch, key_path=key_path)
 
     # Workflow tasks