UW-590 Add --cycle and --leadtime to file tool (ufs-community#498)

maddenp-noaa · May 30, 2024 · a440342 · a440342
1 parent 96785cd
commit a440342
Show file tree

Hide file tree

Showing 20 changed files with 173 additions and 51 deletions.
diff --git a/docs/sections/user_guide/cli/tools/file.rst b/docs/sections/user_guide/cli/tools/file.rst
@@ -34,6 +34,15 @@ Given ``copy-config.yaml`` containing
 
 Here, ``foo`` and ``bar`` are copies of their respective source files.
 
+The ``--cycle`` and ``--leadtime`` options can be used to make Python ``datetime`` and ``timedelta`` objects, respectively, available for use in Jinja2 expression in the config. For example:
+
+.. literalinclude:: file/copy-config-timedep.yaml
+   :language: yaml
+.. literalinclude:: file/copy-exec-timedep.cmd
+   :emphasize-lines: 2
+.. literalinclude:: file/copy-exec-timedep.out
+   :language: text
+
 .. _cli_file_link_examples:
 
 ``link``
@@ -59,3 +68,12 @@ Given ``link-config.yaml`` containing
    :language: text
 
 Here, ``foo`` and ``bar`` are symbolic links.
+
+The ``--cycle`` and ``--leadtime`` options can be used to make Python ``datetime`` and ``timedelta`` objects, respectively, available for use in Jinja2 expression in the config. For example:
+
+.. literalinclude:: file/link-config-timedep.yaml
+   :language: yaml
+.. literalinclude:: file/link-exec-timedep.cmd
+   :emphasize-lines: 2
+.. literalinclude:: file/link-exec-timedep.out
+   :language: text
diff --git a/docs/sections/user_guide/cli/tools/file/.gitignore b/docs/sections/user_guide/cli/tools/file/.gitignore
@@ -1,2 +1,4 @@
-dst-copy
-dst-link
+copy-dst
+copy-dst-timedep
+link-dst
+link-dst-timedep
diff --git a/docs/sections/user_guide/cli/tools/file/copy-config-timedep.yaml b/docs/sections/user_guide/cli/tools/file/copy-config-timedep.yaml
@@ -0,0 +1,7 @@
+config:
+  files:
+    baz-{{ validtime }}: src/{{ yyyymmdd }}/{{ hh }}/{{ nnn }}/baz
+yyyymmdd: "{{ cycle.strftime('%Y%m%d') }}"
+hh: "{{ cycle.strftime('%H') }}"
+nnn: "{{ '%03d' % (leadtime.total_seconds() // 3600) }}"
+validtime: "{{ (cycle + leadtime).strftime('%Y-%m-%dT%H') }}"
diff --git a/docs/sections/user_guide/cli/tools/file/copy-exec-timedep.cmd b/docs/sections/user_guide/cli/tools/file/copy-exec-timedep.cmd
@@ -0,0 +1,4 @@
+rm -rf copy-dst-timedep
+uw file copy --target-dir copy-dst-timedep --config-file copy-config-timedep.yaml --cycle 2024-05-29T12 --leadtime 6 config files
+echo
+tree copy-dst-timedep
diff --git a/docs/sections/user_guide/cli/tools/file/copy-exec-timedep.out b/docs/sections/user_guide/cli/tools/file/copy-exec-timedep.out
@@ -0,0 +1,15 @@
+[2024-05-29T16:50:14]     INFO Validating config against internal schema files-to-stage
+[2024-05-29T16:50:14]     INFO 0 UW schema-validation errors found
+[2024-05-29T16:50:14]     INFO File copies: Initial state: Pending
+[2024-05-29T16:50:14]     INFO File copies: Checking requirements
+[2024-05-29T16:50:14]     INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Initial state: Pending
+[2024-05-29T16:50:14]     INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Checking requirements
+[2024-05-29T16:50:14]     INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Requirement(s) ready
+[2024-05-29T16:50:14]     INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Executing
+[2024-05-29T16:50:14]     INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Final state: Ready
+[2024-05-29T16:50:14]     INFO File copies: Final state: Ready
+
+copy-dst-timedep
+└── baz-2024-05-29T18
+
+1 directory, 1 file
diff --git a/docs/sections/user_guide/cli/tools/file/copy-exec.cmd b/docs/sections/user_guide/cli/tools/file/copy-exec.cmd
@@ -1,4 +1,4 @@
-rm -rf dst-copy
-uw file copy --target-dir dst-copy --config-file copy-config.yaml config files
+rm -rf copy-dst
+uw file copy --target-dir copy-dst --config-file copy-config.yaml config files
 echo
-tree dst-copy
+tree copy-dst
diff --git a/docs/sections/user_guide/cli/tools/file/copy-exec.out b/docs/sections/user_guide/cli/tools/file/copy-exec.out
@@ -1,20 +1,20 @@
-[2024-05-23T19:39:15]     INFO Validating config against internal schema files-to-stage
-[2024-05-23T19:39:15]     INFO 0 UW schema-validation errors found
-[2024-05-23T19:39:15]     INFO File copies: Initial state: Pending
-[2024-05-23T19:39:15]     INFO File copies: Checking requirements
-[2024-05-23T19:39:15]     INFO Copy src/foo -> dst-copy/foo: Initial state: Pending
-[2024-05-23T19:39:15]     INFO Copy src/foo -> dst-copy/foo: Checking requirements
-[2024-05-23T19:39:15]     INFO Copy src/foo -> dst-copy/foo: Requirement(s) ready
-[2024-05-23T19:39:15]     INFO Copy src/foo -> dst-copy/foo: Executing
-[2024-05-23T19:39:15]     INFO Copy src/foo -> dst-copy/foo: Final state: Ready
-[2024-05-23T19:39:15]     INFO Copy src/bar -> dst-copy/subdir/bar: Initial state: Pending
-[2024-05-23T19:39:15]     INFO Copy src/bar -> dst-copy/subdir/bar: Checking requirements
-[2024-05-23T19:39:15]     INFO Copy src/bar -> dst-copy/subdir/bar: Requirement(s) ready
-[2024-05-23T19:39:15]     INFO Copy src/bar -> dst-copy/subdir/bar: Executing
-[2024-05-23T19:39:15]     INFO Copy src/bar -> dst-copy/subdir/bar: Final state: Ready
-[2024-05-23T19:39:15]     INFO File copies: Final state: Ready
+[2024-05-29T16:50:14]     INFO Validating config against internal schema files-to-stage
+[2024-05-29T16:50:14]     INFO 0 UW schema-validation errors found
+[2024-05-29T16:50:14]     INFO File copies: Initial state: Pending
+[2024-05-29T16:50:14]     INFO File copies: Checking requirements
+[2024-05-29T16:50:14]     INFO Copy src/foo -> copy-dst/foo: Initial state: Pending
+[2024-05-29T16:50:14]     INFO Copy src/foo -> copy-dst/foo: Checking requirements
+[2024-05-29T16:50:14]     INFO Copy src/foo -> copy-dst/foo: Requirement(s) ready
+[2024-05-29T16:50:14]     INFO Copy src/foo -> copy-dst/foo: Executing
+[2024-05-29T16:50:14]     INFO Copy src/foo -> copy-dst/foo: Final state: Ready
+[2024-05-29T16:50:14]     INFO Copy src/bar -> copy-dst/subdir/bar: Initial state: Pending
+[2024-05-29T16:50:14]     INFO Copy src/bar -> copy-dst/subdir/bar: Checking requirements
+[2024-05-29T16:50:14]     INFO Copy src/bar -> copy-dst/subdir/bar: Requirement(s) ready
+[2024-05-29T16:50:14]     INFO Copy src/bar -> copy-dst/subdir/bar: Executing
+[2024-05-29T16:50:14]     INFO Copy src/bar -> copy-dst/subdir/bar: Final state: Ready
+[2024-05-29T16:50:14]     INFO File copies: Final state: Ready
 
-dst-copy
+copy-dst
 ├── foo
 └── subdir
     └── bar

diff --git a/docs/sections/user_guide/cli/tools/file/copy-help.out b/docs/sections/user_guide/cli/tools/file/copy-help.out
@@ -1,5 +1,6 @@
 usage: uw file copy --target-dir PATH [-h] [--version] [--config-file PATH]
-                    [--dry-run] [--quiet] [--verbose]
+                    [--cycle CYCLE] [--leadtime LEADTIME] [--dry-run]
+                    [--quiet] [--verbose]
                     [KEY ...]
 
 Copy files
@@ -15,6 +16,10 @@ Optional arguments:
       Show version info and exit
   --config-file PATH, -c PATH
       Path to UW YAML config file (default: read from stdin)
+  --cycle CYCLE
+      The cycle in ISO8601 format (e.g. 2024-05-29T12)
+  --leadtime LEADTIME
+      The leadtime as HH[:MM[:SS]]
   --dry-run
       Only log info, making no changes
   --quiet, -q

diff --git a/docs/sections/user_guide/cli/tools/file/link-config-timedep.yaml b/docs/sections/user_guide/cli/tools/file/link-config-timedep.yaml
@@ -0,0 +1,7 @@
+config:
+  files:
+    baz-{{ validtime }}: src/{{ yyyymmdd }}/{{ hh }}/{{ nnn }}/baz
+yyyymmdd: "{{ cycle.strftime('%Y%m%d') }}"
+hh: "{{ cycle.strftime('%H') }}"
+nnn: "{{ '%03d' % (leadtime.total_seconds() // 3600) }}"
+validtime: "{{ (cycle + leadtime).strftime('%Y-%m-%dT%H') }}"
diff --git a/docs/sections/user_guide/cli/tools/file/link-exec-timedep.cmd b/docs/sections/user_guide/cli/tools/file/link-exec-timedep.cmd
@@ -0,0 +1,4 @@
+rm -rf link-dst-timedep
+uw file link --target-dir link-dst-timedep --config-file link-config-timedep.yaml --cycle 2024-05-29T12 --leadtime 6 config files
+echo
+tree link-dst-timedep
diff --git a/docs/sections/user_guide/cli/tools/file/link-exec-timedep.out b/docs/sections/user_guide/cli/tools/file/link-exec-timedep.out
@@ -0,0 +1,15 @@
+[2024-05-29T16:50:14]     INFO Validating config against internal schema files-to-stage
+[2024-05-29T16:50:14]     INFO 0 UW schema-validation errors found
+[2024-05-29T16:50:14]     INFO File links: Initial state: Pending
+[2024-05-29T16:50:14]     INFO File links: Checking requirements
+[2024-05-29T16:50:14]     INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Initial state: Pending
+[2024-05-29T16:50:14]     INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Checking requirements
+[2024-05-29T16:50:14]     INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Requirement(s) ready
+[2024-05-29T16:50:14]     INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Executing
+[2024-05-29T16:50:14]     INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Final state: Ready
+[2024-05-29T16:50:14]     INFO File links: Final state: Ready
+
+link-dst-timedep
+└── baz-2024-05-29T18 -> ../src/20240529/12/006/baz
+
+1 directory, 1 file
diff --git a/docs/sections/user_guide/cli/tools/file/link-exec.cmd b/docs/sections/user_guide/cli/tools/file/link-exec.cmd
@@ -1,4 +1,4 @@
-rm -rf dst-link
-uw file link --target-dir dst-link --config-file link-config.yaml config files
+rm -rf link-dst
+uw file link --target-dir link-dst --config-file link-config.yaml config files
 echo
-tree dst-link
+tree link-dst
diff --git a/docs/sections/user_guide/cli/tools/file/link-exec.out b/docs/sections/user_guide/cli/tools/file/link-exec.out
@@ -1,20 +1,20 @@
-[2024-05-23T19:39:16]     INFO Validating config against internal schema files-to-stage
-[2024-05-23T19:39:16]     INFO 0 UW schema-validation errors found
-[2024-05-23T19:39:16]     INFO File links: Initial state: Pending
-[2024-05-23T19:39:16]     INFO File links: Checking requirements
-[2024-05-23T19:39:16]     INFO Link dst-link/foo -> src/foo: Initial state: Pending
-[2024-05-23T19:39:16]     INFO Link dst-link/foo -> src/foo: Checking requirements
-[2024-05-23T19:39:16]     INFO Link dst-link/foo -> src/foo: Requirement(s) ready
-[2024-05-23T19:39:16]     INFO Link dst-link/foo -> src/foo: Executing
-[2024-05-23T19:39:16]     INFO Link dst-link/foo -> src/foo: Final state: Ready
-[2024-05-23T19:39:16]     INFO Link dst-link/subdir/bar -> src/bar: Initial state: Pending
-[2024-05-23T19:39:16]     INFO Link dst-link/subdir/bar -> src/bar: Checking requirements
-[2024-05-23T19:39:16]     INFO Link dst-link/subdir/bar -> src/bar: Requirement(s) ready
-[2024-05-23T19:39:16]     INFO Link dst-link/subdir/bar -> src/bar: Executing
-[2024-05-23T19:39:16]     INFO Link dst-link/subdir/bar -> src/bar: Final state: Ready
-[2024-05-23T19:39:16]     INFO File links: Final state: Ready
+[2024-05-29T16:50:14]     INFO Validating config against internal schema files-to-stage
+[2024-05-29T16:50:14]     INFO 0 UW schema-validation errors found
+[2024-05-29T16:50:14]     INFO File links: Initial state: Pending
+[2024-05-29T16:50:14]     INFO File links: Checking requirements
+[2024-05-29T16:50:14]     INFO Link link-dst/foo -> src/foo: Initial state: Pending
+[2024-05-29T16:50:14]     INFO Link link-dst/foo -> src/foo: Checking requirements
+[2024-05-29T16:50:14]     INFO Link link-dst/foo -> src/foo: Requirement(s) ready
+[2024-05-29T16:50:14]     INFO Link link-dst/foo -> src/foo: Executing
+[2024-05-29T16:50:14]     INFO Link link-dst/foo -> src/foo: Final state: Ready
+[2024-05-29T16:50:14]     INFO Link link-dst/subdir/bar -> src/bar: Initial state: Pending
+[2024-05-29T16:50:14]     INFO Link link-dst/subdir/bar -> src/bar: Checking requirements
+[2024-05-29T16:50:14]     INFO Link link-dst/subdir/bar -> src/bar: Requirement(s) ready
+[2024-05-29T16:50:14]     INFO Link link-dst/subdir/bar -> src/bar: Executing
+[2024-05-29T16:50:14]     INFO Link link-dst/subdir/bar -> src/bar: Final state: Ready
+[2024-05-29T16:50:14]     INFO File links: Final state: Ready
 
-dst-link
+link-dst
 ├── foo -> ../src/foo
 └── subdir
     └── bar -> ../../src/bar

diff --git a/docs/sections/user_guide/cli/tools/file/link-help.out b/docs/sections/user_guide/cli/tools/file/link-help.out
@@ -1,5 +1,6 @@
 usage: uw file link --target-dir PATH [-h] [--version] [--config-file PATH]
-                    [--dry-run] [--quiet] [--verbose]
+                    [--cycle CYCLE] [--leadtime LEADTIME] [--dry-run]
+                    [--quiet] [--verbose]
                     [KEY ...]
 
 Link files
@@ -15,6 +16,10 @@ Optional arguments:
       Show version info and exit
   --config-file PATH, -c PATH
       Path to UW YAML config file (default: read from stdin)
+  --cycle CYCLE
+      The cycle in ISO8601 format (e.g. 2024-05-29T12)
+  --leadtime LEADTIME
+      The leadtime as HH[:MM[:SS]]
   --dry-run
       Only log info, making no changes
   --quiet, -q

diff --git a/docs/sections/user_guide/cli/tools/file/src/20240529/12/006/baz b/docs/sections/user_guide/cli/tools/file/src/20240529/12/006/baz
@@ -0,0 +1 @@
+baz
diff --git a/src/uwtools/api/file.py b/src/uwtools/api/file.py
@@ -2,6 +2,7 @@
 API access to ``uwtools`` file-management tools.
 """
 
+import datetime as dt
 from pathlib import Path
 from typing import List, Optional, Union
 
@@ -13,6 +14,8 @@
 def copy(
     target_dir: Union[Path, str],
     config: Optional[Union[dict, Path, str]] = None,
+    cycle: Optional[dt.datetime] = None,
+    leadtime: Optional[dt.timedelta] = None,
     keys: Optional[List[str]] = None,
     dry_run: bool = False,
     stdin_ok: bool = False,
@@ -22,6 +25,8 @@ def copy(
 
     :param target_dir: Path to target directory
     :param config: YAML-file path, or ``dict`` (read ``stdin`` if missing or ``None``).
+    :param cycle: A datetime object to make available for use in the config.
+    :param leadtime: A timedelta object to make available for use in the config.
     :param keys: YAML keys leading to file dst/src block
     :param dry_run: Do not copy files
     :param stdin_ok: OK to read from ``stdin``?
@@ -30,6 +35,8 @@ def copy(
     _FileCopier(
         target_dir=Path(target_dir),
         config=_ensure_data_source(config, stdin_ok),
+        cycle=cycle,
+        leadtime=leadtime,
         keys=keys,
         dry_run=dry_run,
     ).go()
@@ -39,6 +46,8 @@ def copy(
 def link(
     target_dir: Union[Path, str],
     config: Optional[Union[dict, Path, str]] = None,
+    cycle: Optional[dt.datetime] = None,
+    leadtime: Optional[dt.timedelta] = None,
     keys: Optional[List[str]] = None,
     dry_run: bool = False,
     stdin_ok: bool = False,
@@ -48,6 +57,8 @@ def link(
 
     :param target_dir: Path to target directory
     :param config: YAML-file path, or ``dict`` (read ``stdin`` if missing or ``None``).
+    :param cycle: A datetime object to make available for use in the config.
+    :param leadtime: A timedelta object to make available for use in the config.
     :param keys: YAML keys leading to file dst/src block
     :param dry_run: Do not link files
     :param stdin_ok: OK to read from ``stdin``?
@@ -56,6 +67,8 @@ def link(
     _FileLinker(
         target_dir=Path(target_dir),
         config=_ensure_data_source(config, stdin_ok),
+        cycle=cycle,
+        leadtime=leadtime,
         keys=keys,
         dry_run=dry_run,
     ).go()

diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py
@@ -271,7 +271,9 @@ def _add_subparser_file_common(parser: Parser) -> ActionChecks:
     required = parser.add_argument_group(TITLE_REQ_ARG)
     _add_arg_target_dir(required, required=True)
     optional = _basic_setup(parser)
-    _add_arg_config_file(group=optional)
+    _add_arg_config_file(optional)
+    _add_arg_cycle(optional)
+    _add_arg_leadtime(optional)
     _add_arg_dry_run(optional)
     checks = _add_args_verbosity(optional)
     _add_arg_keys(optional)
@@ -320,6 +322,8 @@ def _dispatch_file_copy(args: Args) -> bool:
     return uwtools.api.file.copy(
         target_dir=args[STR.targetdir],
         config=args[STR.cfgfile],
+        cycle=args[STR.cycle],
+        leadtime=args[STR.leadtime],
         keys=args[STR.keys],
         dry_run=args[STR.dryrun],
         stdin_ok=True,
@@ -335,6 +339,8 @@ def _dispatch_file_link(args: Args) -> bool:
     return uwtools.api.file.link(
         target_dir=args[STR.targetdir],
         config=args[STR.cfgfile],
+        cycle=args[STR.cycle],
+        leadtime=args[STR.leadtime],
         keys=args[STR.keys],
         dry_run=args[STR.dryrun],
         stdin_ok=True,
@@ -553,13 +559,13 @@ def _add_arg_config_file(group: Group, required: bool = False) -> None:
     )
 
 
-def _add_arg_cycle(group: Group) -> None:
+def _add_arg_cycle(group: Group, required: bool = False) -> None:
     offset = dt.timedelta(hours=(dt.datetime.now(dt.timezone.utc).hour // 6) * 6)
     cycle = dt.datetime.combine(dt.date.today(), dt.datetime.min.time()) + offset
     group.add_argument(
         _switch(STR.cycle),
         help="The cycle in ISO8601 format (e.g. %s)" % cycle.strftime("%Y-%m-%dT%H"),
-        required=True,
+        required=required,
         type=dt.datetime.fromisoformat,
     )
 
@@ -660,11 +666,11 @@ def _add_arg_keys(group: Group) -> None:
     )
 
 
-def _add_arg_leadtime(group: Group) -> None:
+def _add_arg_leadtime(group: Group, required: bool = False) -> None:
     group.add_argument(
         _switch(STR.leadtime),
         help=f"The leadtime as {LEADTIME_DESC}",
-        required=True,
+        required=required,
         type=_timedelta_from_str,
     )
 
@@ -880,11 +886,11 @@ def _add_subparser_for_driver_task(
     parser = _add_subparser(subparsers, task, helpmsg.rstrip("."))
     required = parser.add_argument_group(TITLE_REQ_ARG)
     if with_cycle:
-        _add_arg_cycle(required)
+        _add_arg_cycle(required, required=True)
     if with_leadtime:
-        _add_arg_leadtime(required)
+        _add_arg_leadtime(required, required=True)
     optional = _basic_setup(parser)
-    _add_arg_config_file(group=optional)
+    _add_arg_config_file(optional)
     _add_arg_batch(optional)
     _add_arg_dry_run(optional)
     _add_arg_graph_file(optional)