diff --git a/docs/sections/user_guide/cli/tools/config.rst b/docs/sections/user_guide/cli/tools/config.rst index 4383c22c9..b441c78a5 100644 --- a/docs/sections/user_guide/cli/tools/config.rst +++ b/docs/sections/user_guide/cli/tools/config.rst @@ -156,10 +156,10 @@ In ``uw`` terminology, to realize a configuration file is to transform it from i $ uw config realize --help usage: uw config realize [-h] [--version] [--input-file PATH] [--input-format {ini,nml,sh,yaml}] + [--update-file PATH] [--update-format {ini,nml,sh,yaml}] [--output-file PATH] [--output-format {ini,nml,sh,yaml}] [--output-block KEY[.KEY[.KEY]...]] [--values-needed] [--total] [--dry-run] [--quiet] [--verbose] - [PATH ...] Realize config @@ -172,6 +172,10 @@ In ``uw`` terminology, to realize a configuration file is to transform it from i Path to input file (defaults to stdin) --input-format {ini,nml,sh,yaml} Input format + --update-file PATH, -u PATH + Path to update file (defaults to stdin) + --update-format {ini,nml,sh,yaml} + Input format --output-file PATH, -o PATH Path to output file (defaults to stdout) --output-format {ini,nml,sh,yaml} @@ -188,13 +192,11 @@ In ``uw`` terminology, to realize a configuration file is to transform it from i Print no logging messages --verbose, -v Print all logging messages - PATH - Additional files to supplement primary input Examples ^^^^^^^^ -The examples in this section use the YAML file ``config.yaml`` with the following contents: +The initial examples in this section use YAML file ``config.yaml`` with the following contents: .. code-block:: yaml @@ -202,11 +204,11 @@ The examples in this section use the YAML file ``config.yaml`` with the followin date: '{{ yyyymmdd }}' empty: greeting: Hello - message: '{{ (greeting + " " + recipient + " ") * repeat }}' + message: '{{ ((greeting + " " + recipient + " ") * repeat) | trim }}' recipient: World repeat: 1 -and the supplemental YAML file ``values1.yaml`` with the following contents: +and YAML file ``update.yaml`` with the following contents: .. code-block:: yaml @@ -216,33 +218,25 @@ and the supplemental YAML file ``values1.yaml`` with the following contents: recipient: Moon repeat: 2 -and an additional supplemental YAML file ``values2.yaml`` with the following contents: - -.. code-block:: yaml - - values: - empty: false - repeat: 3 - * To show the values in the input config file that have unrendered Jinja2 variables/expressions or empty keys: .. code-block:: text $ uw config realize --input-file config.yaml --output-format yaml --values-needed - [2024-01-23T22:28:40] INFO Keys that are complete: - [2024-01-23T22:28:40] INFO values - [2024-01-23T22:28:40] INFO values.greeting - [2024-01-23T22:28:40] INFO values.message - [2024-01-23T22:28:40] INFO values.recipient - [2024-01-23T22:28:40] INFO values.repeat - [2024-01-23T22:28:40] INFO - [2024-01-23T22:28:40] INFO Keys with unrendered Jinja2 variables/expressions: - [2024-01-23T22:28:40] INFO values.date: {{ yyyymmdd }} - [2024-01-23T22:28:40] INFO - [2024-01-23T22:28:40] INFO Keys that are set to empty: - [2024-01-23T22:28:40] INFO values.empty - -* To realize the config to ``stdout``, a target output format must be explicitly specified: + [2024-05-20T18:33:01] INFO Keys that are complete: + [2024-05-20T18:33:01] INFO values + [2024-05-20T18:33:01] INFO values.greeting + [2024-05-20T18:33:01] INFO values.message + [2024-05-20T18:33:01] INFO values.recipient + [2024-05-20T18:33:01] INFO values.repeat + [2024-05-20T18:33:01] INFO + [2024-05-20T18:33:01] INFO Keys with unrendered Jinja2 variables/expressions: + [2024-05-20T18:33:01] INFO values.date: {{ yyyymmdd }} + [2024-05-20T18:33:01] INFO + [2024-05-20T18:33:01] INFO Keys that are set to empty: + [2024-05-20T18:33:01] INFO values.empty + +* To realize the config to ``stdout``, the output format must be explicitly specified: .. code-block:: text @@ -251,30 +245,30 @@ and an additional supplemental YAML file ``values2.yaml`` with the following con date: '{{ yyyymmdd }}' empty: null greeting: Hello - message: 'Hello World ' + message: Hello World recipient: World repeat: 1 Shell redirection via ``|``, ``>``, et al. may also be used to stream output to a file, another process, etc. -* Values in the input file can be overridden via one or more supplemental files specified as positional arguments. Priority increases from left to right. +* Values in the input file can be updated via an optional update file: .. code-block:: text - $ uw config realize --input-file config.yaml --output-format yaml values1.yaml values2.yaml + $ uw config realize --input-file config.yaml --update-file update.yaml --output-format yaml values: date: 20240105 - empty: false + empty: null greeting: Good Night - message: 'Good Night Moon Good Night Moon Good Night Moon ' + message: Good Night Moon Good Night Moon recipient: Moon - repeat: 3 + repeat: 2 * To realize the config to a file via command-line argument: .. code-block:: text - $ uw config realize --input-file config.yaml --output-file realized.yaml values1.yaml + $ uw config realize --input-file config.yaml --update-file update.yaml --output-file realized.yaml The contents of ``realized.yaml``: @@ -284,229 +278,224 @@ and an additional supplemental YAML file ``values2.yaml`` with the following con date: 20240105 empty: null greeting: Good Night - message: 'Good Night Moon Good Night Moon ' + message: Good Night Moon Good Night Moon recipient: Moon repeat: 2 -* By default, variables/expressions that cannot be rendered are passed through unchanged in the output. For example, given config file ``config.yaml`` with contents - - .. code-block:: yaml - - roses: "{{ color1 }}" - violets: "{{ color2 }}" - color1: red +* With the ``--dry-run`` flag specified, nothing is written to ``stdout`` (or to a file if ``--output-file`` is specified), but a report of what would have been written is logged to ``stderr``: .. code-block:: text - $ uw config realize --input-file config.yaml --output-format yaml values.yaml - roses: red - violets: '{{ color2 }}' - color1: red - $ echo $? - 0 + $ uw config realize --input-file config.yaml --update-file update.yaml --output-file realized.yaml --dry-run + [2024-05-20T19:05:55] INFO values: + [2024-05-20T19:05:55] INFO date: 20240105 + [2024-05-20T19:05:55] INFO empty: null + [2024-05-20T19:05:55] INFO greeting: Good Night + [2024-05-20T19:05:55] INFO message: Good Night Moon Good Night Moon + [2024-05-20T19:05:55] INFO recipient: Moon + [2024-05-20T19:05:55] INFO repeat: 2 - Adding the ``--total`` flag, however, requires ``uw`` to totally realize the config, and to exit with error status if it cannot: +* If the config file has an unrecognized (or no) extension, ``uw`` will not automatically know how to parse its contents: .. code-block:: text - $ uw config realize --input-file config.yaml --output-format yaml values.yaml --total - [2024-04-02T00:53:04] ERROR Config could not be realized. Try with --values-needed for details. - $ echo $? - 1 - -* Realization of individual values is all-or-nothing. If a single value contains a mix of renderable and unrenderable variables/expressions, then the entire value remains unrealized. For example, given ``config.yaml`` with contents - - .. code-block:: yaml - - roses: "{{ color1 }} or {{ color2 }}" - color1: red - - .. code-block:: text - - $ uw config realize --input-file config.yaml --output-format yaml - roses: '{{ color1 }} or {{ color2 }}' - color1: red + $ uw config realize --input-file config.txt --update-file update.yaml --output-format yaml + Cannot deduce format of 'config.txt' from unknown extension 'txt' -* With the ``--dry-run`` flag specified, nothing is written to ``stdout`` (or to a file if ``--output-file`` is specified), but a report of what would have been written is logged to ``stderr``: + The format must be explicitly specified (``config.txt`` is a copy of ``config.yaml``): .. code-block:: text - $ uw config realize --input-file config.yaml --output-file realized.yaml --dry-run values1.yaml - [2024-01-23T22:31:08] INFO values: - [2024-01-23T22:31:08] INFO date: 20240105 - [2024-01-23T22:31:08] INFO empty: null - [2024-01-23T22:31:08] INFO greeting: Good Night - [2024-01-23T22:31:08] INFO message: 'Good Night Moon Good Night Moon ' - [2024-01-23T22:31:08] INFO recipient: Moon - [2024-01-23T22:31:08] INFO repeat: 2 + $ uw config realize --input-file config.txt --update-file update.yaml --output-format yaml --input-format yaml + values: + date: 20240105 + empty: null + greeting: Good Night + message: Good Night Moon Good Night Moon + recipient: Moon + repeat: 2 -* If an input file is read alone from ``stdin``, ``uw`` will not know how to parse its contents: +* Similarly, if an input file is read from ``stdin``, ``uw`` will not automatically know how to parse its contents: .. code-block:: text - $ cat config.yaml | uw config realize --output-file realized.yaml values1.yaml + $ cat config.yaml | uw config realize --update-file update.yaml --output-format yaml Specify --input-format when --input-file is not specified -* To read the config from ``stdin`` and realize to ``stdout``: + The format must be explicitly specified: .. code-block:: text - $ cat config.yaml | uw config realize --input-format yaml --output-format yaml values1.yaml + $ cat config.yaml | uw config realize --update-file update.yaml --output-format yaml --input-format yaml values: date: 20240105 empty: null greeting: Good Night - message: 'Good Night Moon Good Night Moon ' + message: Good Night Moon Good Night Moon recipient: Moon repeat: 2 -* To read the config from ``stdin`` and realize a subsection to ``stdout`` in a different format: +* This example demonstrates: 1. Reading a config from ``stdin``, 2. Extracting a specific subsection with the ``--output-block`` option, and 3. Writing the output in a different format: .. code-block:: text - $ cat config.yaml | uw config realize --input-format yaml --output-format sh --output-block values values1.yaml + $ cat config.yaml | uw config realize --input-format yaml --update-file update.yaml --output-block values --output-format sh date=20240105 empty=None greeting='Good Night' - message='Good Night Moon Good Night Moon ' + message='Good Night Moon Good Night Moon' recipient=Moon repeat=2 -* If the config file has an unrecognized (or no) extension, ``uw`` will not know how to parse its contents: +.. note:: Combining configs with incompatible depths is not supported. ``ini`` and ``nml`` configs are depth-2, as they organize their key-value pairs (one level) under top-level sections or namelists (a second level). ``sh`` configs are depth-1, and ``yaml`` configs have arbitrary depth. - .. code-block:: text + For example, when attempting to generate a ``sh`` config from the original depth-2 ``config.yaml``: - $ uw config realize --input-file config.txt --output-format yaml values1.yaml - Cannot deduce format of 'config.txt' from unknown extension 'txt' + .. code-block:: text - In this case, the format can be explicitly specified (``config.txt`` is a copy of ``config.yaml``): + $ uw config realize --input-file config.yaml --output-format sh + [2024-05-20T19:17:02] ERROR Cannot realize depth-2 config to type-'sh' config + +* It is possible to provide the update config, rather than the input config, on ``stdin``. Usage rules are as follows: + + * Only if either ``--update-file`` or ``--update-config`` are specified will ``uw`` attempt to read and apply update values to the input config. + * If ``--update-file`` is provided with an unrecognized (or no) extension, or if the update values are provided on ``stdin``, ``--update-format`` must be used to specify the correct format. + * When updating, the input config, the update config, or both must be provided via file; they cannot be streamed from ``stdin`` simultaneously. + + For example, here the update config is provided on ``stdin`` and the input config is read from a file: .. code-block:: text - $ uw config realize --input-file config.txt --input-format yaml --output-format yaml values1.yaml + $ echo "yyyymmdd: 20240520" | uw config realize --input-file config.yaml --update-format yaml --output-format yaml values: - date: 20240105 + date: '20240520' empty: null - greeting: Good Night - message: 'Good Night Moon Good Night Moon ' - recipient: Moon - repeat: 2 + greeting: Hello + message: Hello World + recipient: World + repeat: 1 + yyyymmdd: 20240520 -* To request verbose log output: +* By default, variables/expressions that cannot be rendered are passed through unchanged in the output. For example, given config file ``flowers.yaml`` with contents + + .. code-block:: yaml + + roses: "{{ color1 }}" + violets: "{{ color2 }}" + color1: red .. code-block:: text - $ uw config realize --input-file config.yaml --output-format yaml --verbose values1.yaml - [2024-01-23T22:59:58] DEBUG Command: uw config realize --input-file config.yaml --output-format yaml --verbose values1.yaml - [2024-01-23T22:59:58] DEBUG Before update, config has depth 2 - [2024-01-23T22:59:58] DEBUG Supplemental config has depth 2 - [2024-01-23T22:59:58] DEBUG After update, config has depth 2 - [2024-01-23T22:59:58] DEBUG Dereferencing, current value: - [2024-01-23T22:59:58] DEBUG values: - [2024-01-23T22:59:58] DEBUG date: 20240105 - [2024-01-23T22:59:58] DEBUG empty: null - [2024-01-23T22:59:58] DEBUG greeting: Good Night - [2024-01-23T22:59:58] DEBUG message: '{{ (greeting + " " + recipient + " ") * repeat }}' - [2024-01-23T22:59:58] DEBUG recipient: Moon - [2024-01-23T22:59:58] DEBUG repeat: 2 - ... - [2024-01-23T22:59:58] DEBUG Dereferencing, final value: - [2024-01-23T22:59:58] DEBUG values: - [2024-01-23T22:59:58] DEBUG date: 20240105 - [2024-01-23T22:59:58] DEBUG empty: null - [2024-01-23T22:59:58] DEBUG greeting: Good Night - [2024-01-23T22:59:58] DEBUG message: 'Good Night Moon Good Night Moon ' - [2024-01-23T22:59:58] DEBUG recipient: Moon - [2024-01-23T22:59:58] DEBUG repeat: 2 - values: - date: 20240105 - empty: null - greeting: Good Night - message: 'Good Night Moon Good Night Moon ' - recipient: Moon - repeat: 2 + $ uw config realize --input-file flowers.yaml --output-format yaml + roses: red + violets: '{{ color2 }}' + color1: red + $ echo $? + 0 - Note that ``uw`` logs to ``stderr`` and writes non-log output to ``stdout``, so the streams can be redirected separately: + Adding the ``--total`` flag, however, requires ``uw`` to totally realize the config, and to exit with error status if it cannot: .. code-block:: text - $ uw config realize --input-file config.yaml --output-format yaml --verbose values1.yaml >realized.yaml 2>realized.log + $ uw config realize --input-file flowers.yaml --output-format yaml --total + [2024-05-20T18:39:37] ERROR Config could not be realized. Try with --values-needed for details. + $ echo $? + 1 - The contents of ``realized.yaml``: +* Realization of individual values is all-or-nothing. If a single value contains a mix of renderable and unrenderable variables/expressions, then the entire value remains unrealized. For example, given ``flowers.yaml`` with contents .. code-block:: yaml - values: - date: 20240105 - empty: null - greeting: Good Night - message: 'Good Night Moon Good Night Moon ' - recipient: Moon - repeat: 2 - - The contents of ``realized.log``: + roses: "{{ color1 }} or {{ color2 }}" + color1: red .. code-block:: text - [2024-01-23T23:01:23] DEBUG Command: uw config realize --input-file config.yaml --output-format yaml --verbose values1.yaml - [2024-01-23T23:01:23] DEBUG Before update, config has depth 2 - [2024-01-23T23:01:23] DEBUG Supplemental config has depth 2 - [2024-01-23T23:01:23] DEBUG After update, config has depth 2 - [2024-01-23T23:01:23] DEBUG Dereferencing, current value: - [2024-01-23T23:01:23] DEBUG values: - [2024-01-23T23:01:23] DEBUG date: 20240105 - [2024-01-23T23:01:23] DEBUG empty: null - [2024-01-23T23:01:23] DEBUG greeting: Good Night - [2024-01-23T23:01:23] DEBUG message: '{{ (greeting + " " + recipient + " ") * repeat }}' - [2024-01-23T23:01:23] DEBUG recipient: Moon - [2024-01-23T23:01:23] DEBUG repeat: 2 - [2024-01-23T23:01:23] DEBUG [dereference] Accepting: 20240105 - [2024-01-23T23:01:23] DEBUG [dereference] Accepting: None - [2024-01-23T23:01:23] DEBUG [dereference] Rendering: Good Night - [2024-01-23T23:01:23] DEBUG [dereference] Rendered: Good Night - [2024-01-23T23:01:23] DEBUG [dereference] Rendering: {{ (greeting + " " + recipient + " ") * repeat }} - [2024-01-23T23:01:23] DEBUG [dereference] Rendered: Good Night Moon Good Night Moon - [2024-01-23T23:01:23] DEBUG [dereference] Rendering: Moon - [2024-01-23T23:01:23] DEBUG [dereference] Rendered: Moon - [2024-01-23T23:01:23] DEBUG [dereference] Accepting: 2 - ... - [2024-01-23T23:01:23] DEBUG Dereferencing, final value: - [2024-01-23T23:01:23] DEBUG values: - [2024-01-23T23:01:23] DEBUG date: 20240105 - [2024-01-23T23:01:23] DEBUG empty: null - [2024-01-23T23:01:23] DEBUG greeting: Good Night - [2024-01-23T23:01:23] DEBUG message: 'Good Night Moon Good Night Moon ' - [2024-01-23T23:01:23] DEBUG recipient: Moon - [2024-01-23T23:01:23] DEBUG repeat: 2 + $ uw config realize --input-file flowers.yaml --output-format yaml + roses: '{{ color1 }} or {{ color2 }}' + color1: red -.. note:: Combining configs with incompatible depths is not supported. ``ini`` and ``nml`` configs are depth-2, as they organize their key-value pairs (one level) under top-level sections or namelists (a second level). ``sh`` configs are depth-1, and ``yaml`` configs have arbitrary depth. +* To request verbose log output: - For example, when attempting to generate a ``sh`` config from a depth-2 ``yaml``: + .. code-block:: text - .. code-block:: text + $ echo "{hello: '{{ recipient }}', recipient: world}" | uw config realize --input-format yaml --output-format yaml --verbose + [2024-05-20T19:09:21] DEBUG Command: uw config realize --input-format yaml --output-format yaml --verbose + [2024-05-20T19:09:21] DEBUG Reading input from stdin + [2024-05-20T19:09:21] DEBUG Dereferencing, current value: + [2024-05-20T19:09:21] DEBUG hello: '{{ recipient }}' + [2024-05-20T19:09:21] DEBUG recipient: world + [2024-05-20T19:09:21] DEBUG [dereference] Rendering: {{ recipient }} + [2024-05-20T19:09:21] DEBUG [dereference] Rendered: world + [2024-05-20T19:09:21] DEBUG [dereference] Rendering: hello + [2024-05-20T19:09:21] DEBUG [dereference] Rendered: hello + [2024-05-20T19:09:21] DEBUG [dereference] Rendering: world + [2024-05-20T19:09:21] DEBUG [dereference] Rendered: world + [2024-05-20T19:09:21] DEBUG [dereference] Rendering: recipient + [2024-05-20T19:09:21] DEBUG [dereference] Rendered: recipient + [2024-05-20T19:09:21] DEBUG Dereferencing, current value: + [2024-05-20T19:09:21] DEBUG hello: world + [2024-05-20T19:09:21] DEBUG recipient: world + [2024-05-20T19:09:21] DEBUG [dereference] Rendering: world + [2024-05-20T19:09:21] DEBUG [dereference] Rendered: world + [2024-05-20T19:09:21] DEBUG [dereference] Rendering: hello + [2024-05-20T19:09:21] DEBUG [dereference] Rendered: hello + [2024-05-20T19:09:21] DEBUG [dereference] Rendering: world + [2024-05-20T19:09:21] DEBUG [dereference] Rendered: world + [2024-05-20T19:09:21] DEBUG [dereference] Rendering: recipient + [2024-05-20T19:09:21] DEBUG [dereference] Rendered: recipient + [2024-05-20T19:09:21] DEBUG Dereferencing, final value: + [2024-05-20T19:09:21] DEBUG hello: world + [2024-05-20T19:09:21] DEBUG recipient: world + [2024-05-20T19:09:21] DEBUG Writing output to stdout + hello: world + recipient: world - $ uw config realize --input-file config.yaml --output-format sh - [2024-01-23T23:02:42] ERROR Cannot realize depth-2 config to type-'sh' config - Cannot realize depth-2 config to type-'sh' config + Note that ``uw`` logs to ``stderr`` and writes non-log output to ``stdout``, so the streams can be redirected separately: -.. note:: In recognition of the different sets of value types representable in each config format, ``uw`` supports two format-combination schemes: + .. code-block:: text - 1. **Output matches input:** The format of the output config matches that of the input config. - 2. **YAML:** YAML is accepted as either input or output with any other format. In the worst case, values always have a string representation, but note that, for example, the string representation of a YAML sequence (Python ``list``) in an INI output config may not be useful. + $ echo "{hello: '{{ recipient }}', recipient: world}" | uw config realize --input-format yaml --output-format yaml --verbose >realized.yaml 2>realized.log - In all cases, any supplemental configs must be in the same format as the input config and must have recognized extensions. + The contents of ``realized.yaml``: - ``uw`` considers invalid combination requests errors: + .. code-block:: yaml - .. code-block:: text + hello: world + recipient: world - $ uw config realize --input-file b.nml --output-file a.ini - Accepted output formats for input format nml are nml or yaml + The contents of ``realized.log``: - .. code-block:: text + .. code-block:: text - $ uw config realize --input-file a.yaml --output-file c.yaml b.nml - Supplemental config #1 format nml must match input format yaml + [2024-05-20T19:10:11] DEBUG Command: uw config realize --input-format yaml --output-format yaml --verbose + [2024-05-20T19:10:11] DEBUG Reading input from stdin + [2024-05-20T19:10:11] DEBUG Dereferencing, current value: + [2024-05-20T19:10:11] DEBUG hello: '{{ recipient }}' + [2024-05-20T19:10:11] DEBUG recipient: world + [2024-05-20T19:10:11] DEBUG [dereference] Rendering: {{ recipient }} + [2024-05-20T19:10:11] DEBUG [dereference] Rendered: world + [2024-05-20T19:10:11] DEBUG [dereference] Rendering: hello + [2024-05-20T19:10:11] DEBUG [dereference] Rendered: hello + [2024-05-20T19:10:11] DEBUG [dereference] Rendering: world + [2024-05-20T19:10:11] DEBUG [dereference] Rendered: world + [2024-05-20T19:10:11] DEBUG [dereference] Rendering: recipient + [2024-05-20T19:10:11] DEBUG [dereference] Rendered: recipient + [2024-05-20T19:10:11] DEBUG Dereferencing, current value: + [2024-05-20T19:10:11] DEBUG hello: world + [2024-05-20T19:10:11] DEBUG recipient: world + [2024-05-20T19:10:11] DEBUG [dereference] Rendering: world + [2024-05-20T19:10:11] DEBUG [dereference] Rendered: world + [2024-05-20T19:10:11] DEBUG [dereference] Rendering: hello + [2024-05-20T19:10:11] DEBUG [dereference] Rendered: hello + [2024-05-20T19:10:11] DEBUG [dereference] Rendering: world + [2024-05-20T19:10:11] DEBUG [dereference] Rendered: world + [2024-05-20T19:10:11] DEBUG [dereference] Rendering: recipient + [2024-05-20T19:10:11] DEBUG [dereference] Rendered: recipient + [2024-05-20T19:10:11] DEBUG Dereferencing, final value: + [2024-05-20T19:10:11] DEBUG hello: world + [2024-05-20T19:10:11] DEBUG recipient: world + [2024-05-20T19:10:11] DEBUG Writing output to stdout .. _cli_config_validate_examples: diff --git a/docs/sections/user_guide/yaml/tags.rst b/docs/sections/user_guide/yaml/tags.rst index 248a07eb2..3a603a0e0 100644 --- a/docs/sections/user_guide/yaml/tags.rst +++ b/docs/sections/user_guide/yaml/tags.rst @@ -88,7 +88,7 @@ Removes the tagged YAML key/value pair. For example, given ``input.yaml``: e: 2.718 pi: 3.141 -and ``supplemental.yaml``: +and ``update.yaml``: .. code-block:: yaml @@ -96,5 +96,5 @@ and ``supplemental.yaml``: .. code-block:: text - % uw config realize --input-file input.yaml supplemental.yaml --output-format yaml + % uw config realize --input-file input.yaml --update-file update.yaml --output-format yaml pi: 3.141 diff --git a/src/uwtools/api/config.py b/src/uwtools/api/config.py index 5a668f3ee..b40331608 100644 --- a/src/uwtools/api/config.py +++ b/src/uwtools/api/config.py @@ -47,7 +47,7 @@ def get_fieldtable_config( :param config: FieldTable file to load (``None`` or unspecified => read ``stdin``), or initial ``dict`` - :param stdin_ok: OK to read from stdin? + :param stdin_ok: OK to read from ``stdin``? :return: An initialized ``FieldTableConfig`` object """ return _FieldTableConfig(config=_ensure_data_source(config, stdin_ok)) @@ -61,7 +61,7 @@ def get_ini_config( Get an ``INIConfig`` object. :param config: INI file to load (``None`` or unspecified => read ``stdin``), or initial ``dict`` - :param stdin_ok: OK to read from stdin? + :param stdin_ok: OK to read from ``stdin``? :return: An initialized ``INIConfig`` object """ return _INIConfig(config=_ensure_data_source(config, stdin_ok)) @@ -76,7 +76,7 @@ def get_nml_config( :param config: Fortran namelist file to load (``None`` or unspecified => read ``stdin``), or initial ``dict`` - :param stdin_ok: OK to read from stdin? + :param stdin_ok: OK to read from ``stdin``? :return: An initialized ``NMLConfig`` object """ return _NMLConfig(config=_ensure_data_source(config, stdin_ok)) @@ -91,7 +91,7 @@ def get_sh_config( :param config: File of shell 'key=value' pairs to load (``None`` or unspecified => read ``stdin``), or initial ``dict`` - :param stdin_ok: OK to read from stdin? + :param stdin_ok: OK to read from ``stdin``? :return: An initialized ``SHConfig`` object """ return _SHConfig(config=_ensure_data_source(config, stdin_ok)) @@ -106,7 +106,7 @@ def get_yaml_config( :param config: YAML file to load (``None`` or unspecified => read ``stdin``), or initial ``dict`` - :param stdin_ok: OK to read from stdin? + :param stdin_ok: OK to read from ``stdin``? :return: An initialized ``YAMLConfig`` object """ return _YAMLConfig(config=_ensure_data_source(config, stdin_ok)) @@ -115,10 +115,11 @@ def get_yaml_config( def realize( input_config: Optional[Union[dict, _Config, Path, str]] = None, input_format: Optional[str] = None, - output_block: Optional[List[Union[str, int]]] = None, + update_config: Optional[Union[dict, _Config, Path, str]] = None, + update_format: Optional[str] = None, output_file: Optional[Union[Path, str]] = None, output_format: Optional[str] = None, - supplemental_configs: Optional[List[Union[dict, _Config, Path, str]]] = None, + output_block: Optional[List[Union[str, int]]] = None, values_needed: bool = False, total: bool = False, dry_run: bool = False, @@ -130,14 +131,14 @@ def realize( input_config = ( _YAMLConfig(config=input_config) if isinstance(input_config, dict) else input_config ) - scs = [_str2path(x) for x in supplemental_configs] if supplemental_configs else None _realize( input_config=_ensure_data_source(input_config, stdin_ok), input_format=input_format, - output_block=output_block, + update_config=_ensure_data_source(update_config, stdin_ok), + update_format=update_format, output_file=_str2path(output_file), output_format=output_format, - supplemental_configs=scs, + output_block=output_block, values_needed=values_needed, total=total, dry_run=dry_run, @@ -147,13 +148,14 @@ def realize( def realize_to_dict( # pylint: disable=unused-argument input_config: Optional[Union[dict, _Config, Path, str]] = None, input_format: Optional[str] = None, - supplemental_configs: Optional[List[Union[dict, _Config, Path, str]]] = None, + update_config: Optional[Union[dict, _Config, Path, str]] = None, + update_format: Optional[str] = None, values_needed: bool = False, dry_run: bool = False, stdin_ok: bool = False, ) -> dict: """ - Realize a config to a ``dict``, based on an input config and optional supplemental configs. + Realize a config to a ``dict``, based on a base input config and an optional update config. See ``realize()`` for details on arguments, etc. """ @@ -173,7 +175,7 @@ def validate( :param schema_file: The JSON Schema file to use for validation :param config: The config to validate - :param stdin_ok: OK to read from stdin? + :param stdin_ok: OK to read from ``stdin``? :return: ``True`` if the YAML file conforms to the schema, ``False`` otherwise """ return _validate_yaml( @@ -203,20 +205,21 @@ def validate( realize.__doc__ = """ -Realize a config based on an input config and optional supplemental configs. +Realize a config based on a base input config and an optional update config. -If no input is specified, ``stdin`` is read. A ``dict`` or ``Config`` object may also be provided as -input. If no output is specified, ``stdout`` is written to. When an input or output filename is -specified, its format will be deduced from its extension, if possible. This can be overridden by -specifying the format explicitly, and it is required to do so for reads from ``stdin`` or writes to -``stdout``, as no attempt is made to deduce the format of streamed data. +The input config may be specified as a filesystem path, a ``dict``, or a ``Config`` object. When it +is not, it will be read from ``stdin``. -If optional supplemental configs (which may likewise be file paths or ``Config`` / ``dict`` objects) -are provided, they will be merged, in the order specified, onto the input config. The format of all -input configs must match. +If an update config is specified, it is merged onto the input config, augmenting or overriding base +values. It may be specified as a filesystem path, a ``dict``, or a ``Config`` object. When it is +not, it will be read from ``stdin``. -If the input-config format is YAML, any supported output format may be specified. For all other -input formats, the output format must match the input. +At most one of the input config or the update config may be left unspecified, in which case the +other will be read from ``stdin``. If neither filename or format is specified for the update config, no +update will be performed. + +The output destination may be specified as a filesystem path. When it is not, it will be written to +``stdout``. If ``values_needed`` is ``True``, a report of values needed to realize the config is logged. In ``dry_run`` mode, output is written to ``stderr``. @@ -228,14 +231,15 @@ def validate( :param input_config: Input config file (``None`` or unspecified => read ``stdin``) :param input_format: Format of the input config (optional if file's extension is recognized) -:param output_block: Path through keys to the desired output block +:param update_config: Update config file (``None`` or unspecified => read ``stdin``) +:param update_format: Format of the update config (optional if file's extension is recognized) :param output_file: Output config file (``None`` or unspecified => write to ``stdout``) :param output_format: Format of the output config (optional if file's extension is recognized) -:param supplemental_configs: Configs to merge, in order, onto the input +:param output_block: Path through keys to the desired output block :param values_needed: Report complete, missing, and template values :param total: Require rendering of all Jinja2 variables/expressions :param dry_run: Log output instead of writing to output -:param stdin_ok: OK to read from stdin? +:param stdin_ok: OK to read from ``stdin``? :raises: UWConfigRealizeError if ``total`` is ``True`` and any Jinja2 variable/expression was not rendered """.format( extensions=", ".join(_FORMAT.extensions()) diff --git a/src/uwtools/api/file.py b/src/uwtools/api/file.py index 8d09886da..89f78c013 100644 --- a/src/uwtools/api/file.py +++ b/src/uwtools/api/file.py @@ -24,7 +24,7 @@ def copy( :param config: YAML-file path, or ``dict`` (read ``stdin`` if missing or ``None``). :param keys: YAML keys leading to file dst/src block :param dry_run: Do not copy files - :param stdin_ok: OK to read from stdin? + :param stdin_ok: OK to read from ``stdin``? :return: ``True`` if no exception is raised """ _FileCopier( @@ -50,7 +50,7 @@ def link( :param config: YAML-file path, or ``dict`` (read ``stdin`` if missing or ``None``). :param keys: YAML keys leading to file dst/src block :param dry_run: Do not link files - :param stdin_ok: OK to read from stdin? + :param stdin_ok: OK to read from ``stdin``? :return: ``True`` if no exception is raised """ _FileLinker( diff --git a/src/uwtools/api/rocoto.py b/src/uwtools/api/rocoto.py index eb984efd9..2c25f873c 100644 --- a/src/uwtools/api/rocoto.py +++ b/src/uwtools/api/rocoto.py @@ -28,7 +28,7 @@ def realize( ``YAMLConfig`` object :param output_file: Path to write rendered XML file (``None`` or unspecified => write to ``stdout``) - :param stdin_ok: OK to read from stdin? + :param stdin_ok: OK to read from ``stdin``? :return: ``True`` """ _realize(config=_ensure_data_source(config, stdin_ok), output_file=_str2path(output_file)) @@ -43,7 +43,7 @@ def validate( Validate purported Rocoto XML file against its schema. :param xml_file: Path to XML file (``None`` or unspecified => read ``stdin``) - :param stdin_ok: OK to read from stdin? + :param stdin_ok: OK to read from ``stdin``? :return: ``True`` if the XML conforms to the schema, ``False`` otherwise """ return _validate(xml_file=_ensure_data_source(xml_file, stdin_ok)) diff --git a/src/uwtools/api/template.py b/src/uwtools/api/template.py index 2c2bc9e0a..2b1f808c0 100644 --- a/src/uwtools/api/template.py +++ b/src/uwtools/api/template.py @@ -45,7 +45,7 @@ def render( :param searchpath: Paths to search for extra templates :param values_needed: Just report variables needed to render the template? :param dry_run: Run in dry-run mode? - :param stdin_ok: OK to read from stdin? + :param stdin_ok: OK to read from ``stdin``? :return: The rendered template string :raises: UWTemplateRenderError if template could not be rendered """ @@ -100,7 +100,7 @@ def translate( read ``stdin``) :param output_file: Path to the file to write the converted template to :param dry_run: Run in dry-run mode? - :param stdin_ok: OK to read from stdin? + :param stdin_ok: OK to read from ``stdin``? :return: ``True`` """ _convert_atparse_to_jinja2( diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index 1e7324758..773f0e986 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -145,6 +145,8 @@ def _add_subparser_config_realize(subparsers: Subparsers) -> ActionChecks: optional = _basic_setup(parser) _add_arg_input_file(optional) _add_arg_input_format(optional, choices=FORMATS) + _add_arg_update_file(optional) + _add_arg_update_format(optional, choices=FORMATS) _add_arg_output_file(optional) _add_arg_output_format(optional, choices=FORMATS) _add_arg_output_block(optional) @@ -152,10 +154,10 @@ def _add_subparser_config_realize(subparsers: Subparsers) -> ActionChecks: _add_arg_total(optional) _add_arg_dry_run(optional) checks = _add_args_verbosity(optional) - _add_arg_supplemental_files(optional) return checks + [ partial(_check_file_vs_format, STR.infile, STR.infmt), partial(_check_file_vs_format, STR.outfile, STR.outfmt), + _check_update, ] @@ -211,10 +213,11 @@ def _dispatch_config_realize(args: Args) -> bool: uwtools.api.config.realize( input_config=args[STR.infile], input_format=args[STR.infmt], - output_block=args[STR.outblock], + update_config=args[STR.updatefile], + update_format=args[STR.updatefmt], output_file=args[STR.outfile], output_format=args[STR.outfmt], - supplemental_configs=args[STR.suppfiles], + output_block=args[STR.outblock], values_needed=args[STR.valsneeded], total=args[STR.total], dry_run=args[STR.dryrun], @@ -716,16 +719,6 @@ def _add_arg_search_path(group: Group) -> None: ) -def _add_arg_supplemental_files(group: Group) -> None: - group.add_argument( - STR.suppfiles, - help="Additional files to supplement primary input", - metavar="PATH", - nargs="*", - type=Path, - ) - - def _add_arg_target_dir(group: Group, required: bool) -> None: group.add_argument( _switch(STR.targetdir), @@ -744,6 +737,27 @@ def _add_arg_total(group: Group) -> None: ) +def _add_arg_update_file(group: Group, required: bool = False) -> None: + group.add_argument( + _switch(STR.updatefile), + "-u", + help="Path to update file (defaults to stdin)", + metavar="PATH", + required=required, + type=Path, + ) + + +def _add_arg_update_format(group: Group, choices: List[str], required: bool = False) -> None: + group.add_argument( + _switch(STR.updatefmt), + choices=choices, + help="Update format", + required=required, + type=str, + ) + + def _add_arg_values_file(group: Group, required: bool = False) -> None: group.add_argument( _switch(STR.valsfile), @@ -928,6 +942,13 @@ def _check_template_render_vals_args(args: Args) -> Args: return args +def _check_update(args: Args) -> Args: + if args.get(STR.updatefile) is not None: + if args.get(STR.updatefmt) is None: + args[STR.updatefmt] = get_file_format(args[STR.updatefile]) + return args + + def _check_verbosity(args: Args) -> Args: if args.get(STR.quiet) and args.get(STR.verbose): _abort("%s may not be used with %s" % (_switch(STR.quiet), _switch(STR.verbose))) diff --git a/src/uwtools/config/formats/base.py b/src/uwtools/config/formats/base.py index 5112ffd1e..211a7729a 100644 --- a/src/uwtools/config/formats/base.py +++ b/src/uwtools/config/formats/base.py @@ -44,7 +44,7 @@ def __init__(self, config: Optional[Union[dict, Path]] = None) -> None: def __repr__(self) -> str: """ - Returns the string representation of a Config object. + Returns the YAML string representation of a Config object. """ s = StringIO() yaml.dump(self.data, s) diff --git a/src/uwtools/config/formats/ini.py b/src/uwtools/config/formats/ini.py index 6e010ae6d..63758257a 100644 --- a/src/uwtools/config/formats/ini.py +++ b/src/uwtools/config/formats/ini.py @@ -47,7 +47,6 @@ def dump(self, path: Optional[Path] = None) -> None: :param path: Path to dump config to. """ config_check_depths_dump(config_obj=self, target_format=FORMAT.ini) - self.dump_dict(self.data, path) @staticmethod diff --git a/src/uwtools/config/tools.py b/src/uwtools/config/tools.py index 701abf743..fb02d4aa5 100644 --- a/src/uwtools/config/tools.py +++ b/src/uwtools/config/tools.py @@ -3,13 +3,12 @@ """ from pathlib import Path -from typing import Callable, List, Optional, Union +from typing import Callable, List, Optional, Tuple, Union from uwtools.config.formats.base import Config -from uwtools.config.formats.yaml import YAMLConfig from uwtools.config.jinja2 import unrendered from uwtools.config.support import depth, format_to_config, log_and_error -from uwtools.exceptions import UWConfigRealizeError, UWError +from uwtools.exceptions import UWConfigError, UWConfigRealizeError, UWError from uwtools.logging import MSGWIDTH, log from uwtools.strings import FORMAT from uwtools.utils.file import get_file_format @@ -79,10 +78,11 @@ def config_check_depths_update(config_obj: Union[Config, dict], target_format: s def realize_config( input_config: Union[Config, Optional[Path]] = None, input_format: Optional[str] = None, - output_block: Optional[List[Union[str, int]]] = None, + update_config: Union[Config, Optional[Path]] = None, + update_format: Optional[str] = None, output_file: Optional[Path] = None, output_format: Optional[str] = None, - supplemental_configs: Optional[List[Union[dict, Config, Path]]] = None, + output_block: Optional[List[Union[str, int]]] = None, values_needed: bool = False, total: bool = False, dry_run: bool = False, @@ -90,22 +90,12 @@ def realize_config( """ NB: This docstring is dynamically replaced: See realize_config.__doc__ definition below. """ - input_format = _ensure_format("input", input_format, input_config) - output_format = _ensure_format("output", output_format, output_file) - _validate_format_output(input_format, output_format) - input_obj: Config = ( - input_config - if isinstance(input_config, Config) - else format_to_config(input_format)(config=input_config) - ) - if supplemental_configs: - input_obj = _realize_config_update(input_obj, input_format, supplemental_configs) + input_obj, input_format = _realize_config_input_setup(input_config, input_format) + input_obj = _realize_config_update(input_obj, update_config, update_format) input_obj.dereference() - output_data = input_obj.data - if output_block is not None: - for key in output_block: - output_data = output_data.get(key, {}) - config_check_depths_realize(output_data, output_format) + output_data, output_format = _realize_config_output_setup( + input_obj, output_file, output_format, output_block + ) if dry_run: for line in str(input_obj).strip().split("\n"): log.info(line) @@ -136,7 +126,7 @@ def _ensure_format( :raises: UWError if the format cannot be determined. """ if isinstance(config, Config): - return FORMAT.yaml + return config.get_format() if fmt is None: if config is not None: fmt = get_file_format(config) @@ -170,38 +160,82 @@ def _print_config_section(config: dict, key_path: List[str]) -> None: print("\n".join(sorted(output_lines))) +def _realize_config_input_setup( + input_config: Union[Config, Optional[Path]] = None, input_format: Optional[str] = None +) -> Tuple[Config, str]: + """ + Set up config-realize input. + + :param input_config: Input config source (None => read stdin). + :param input_format: Format of the input config. + :return: The input Config object and its format name. + """ + input_format = _ensure_format("input", input_format, input_config) + if not input_config: + log.debug("Reading input from stdin") + input_obj: Config = ( + input_config + if isinstance(input_config, Config) + else format_to_config(input_format)(config=input_config) + ) + return input_obj, input_format + + +def _realize_config_output_setup( + input_obj: Config, + output_file: Optional[Path] = None, + output_format: Optional[str] = None, + output_block: Optional[List[Union[str, int]]] = None, +) -> Tuple[dict, str]: + """ + Set up config-realize output. + + :param input_obj: The input Config object. + :param output_file: Output config destination (None => write to stdout). + :param output_format: Format of the output config. + :param output_block: Path through keys to the desired output block. + :return: The unrealized data to output and the output format name. + """ + output_format = _ensure_format("output", output_format, output_file) + log.debug("Writing output to %s" % (output_file or "stdout")) + _validate_format("output", output_format, input_obj.get_format()) + output_data = input_obj.data + if output_block is not None: + for key in output_block: + output_data = output_data[key] + config_check_depths_realize(output_data, output_format) + return output_data, output_format + + def _realize_config_update( - config_obj: Config, - config_fmt: str, - supplemental_configs: Optional[List[Union[dict, Config, Path]]] = None, + input_obj: Config, + update_config: Union[Config, Optional[Path]] = None, + update_format: Optional[str] = None, ) -> Config: """ - Update config with values from other configs, if given. + Set up config-realize update. - :param config_obj: The config to update. - :param config_fmt: Format of config's source. - :param supplemental_configs: Sources of values to modify input. - :return: The input config, possibly updated. + :param input_obj: The input Config object. + :param update_config: Input config source (None => read stdin). + :param update_format: Format of the update config. + :return: The updated but unrealized Config object. """ - if supplemental_configs: - log.debug("Before update, config has depth %s", config_obj.depth) - supplemental_obj: Config - for idx, supplemental_config in enumerate(supplemental_configs): - _validate_format_supplemental(config_fmt, supplemental_config, idx) - if isinstance(supplemental_config, dict): - supplemental_obj = YAMLConfig(config=supplemental_config) - elif isinstance(supplemental_config, Config): - supplemental_obj = supplemental_config - else: - supplemental_format = get_file_format(supplemental_config) - supplemental_obj = format_to_config(supplemental_format)(config=supplemental_config) - log.debug("Supplemental config has depth %s", supplemental_obj.depth) - config_check_depths_update(supplemental_obj, config_obj.get_format()) - config_obj.update_values(supplemental_obj) - log.debug("After update, config has depth %s", config_obj.depth) - else: - log.debug("Input config has depth %s", config_obj.depth) - return config_obj + if update_config or update_format: + update_format = _ensure_format("update", update_format, update_config) + if not update_config: + log.debug("Reading update from stdin") + _validate_format("update", update_format, input_obj.get_format()) + update_obj: Config = ( + update_config + if isinstance(update_config, Config) + else format_to_config(update_format)(config=update_config) + ) + log.debug("Initial input config depth: %s", input_obj.depth) + log.debug("Update config depth: %s", update_obj.depth) + config_check_depths_update(update_obj, input_obj.get_format()) + input_obj.update_values(update_obj) + log.debug("Final input config depth: %s", input_obj.depth) + return input_obj def _realize_config_values_needed(input_obj: Config) -> bool: @@ -248,48 +282,24 @@ def _validate_depth( target_class = format_to_config(target_format) config = config_obj.data if isinstance(config_obj, Config) else config_obj if bad_depth(target_class.get_depth_threshold(), depth(config)): - raise log_and_error( + raise UWConfigError( "Cannot %s depth-%s config to type-'%s' config" % (action, depth(config), target_format) ) -def _validate_format_output(input_fmt: str, output_fmt: str) -> None: +def _validate_format(other_fmt_desc: str, other_fmt: str, input_fmt: str) -> None: """ - Ensure output format agrees with input. + Ensure a format agrees with the input format. + :param other_fmt_desc: Description of other format. + :param other_fmt: Other format. :param input_fmt: Input format. - :param output_fmt: Output format. - :raises: UWError if output format is incompatible. - """ - if FORMAT.yaml not in (input_fmt, output_fmt) and input_fmt != output_fmt: - raise UWError( - "Accepted output formats for input format %s are %s or yaml" % (input_fmt, input_fmt) - ) - - -def _validate_format_supplemental( - config_fmt: str, supplemental_cfg: Union[dict, Config, Path], idx: int -) -> None: - """ - Ensure supplemental config format agrees with base config format. - - :param config_fmt: Base config format. - :param supplemental_cfg: Supplemental config to check. - :param idx: Index of supplemental config for identification purposes. - :raises: UWError if supplemental config format is incompatible. + :raises: UWError if other format is incompatible. """ - pre = f"Supplemental config #{idx + 1}" - if isinstance(supplemental_cfg, dict): - log.debug("%s is a dict: Cannot validate its format vs %s", pre, config_fmt) - return - sc_fmt = ( - supplemental_cfg.get_format() - if isinstance(supplemental_cfg, Config) - else _ensure_format(desc=pre, config=supplemental_cfg) - ) - if sc_fmt not in (FORMAT.yaml, config_fmt): + if FORMAT.yaml not in (input_fmt, other_fmt) and input_fmt != other_fmt: raise UWError( - "%s format %s must be %s or input format %s" % (pre, sc_fmt, FORMAT.yaml, config_fmt) + "Accepted %s formats for input format %s are %s or %s" + % (other_fmt_desc, input_fmt, input_fmt, FORMAT.yaml) ) @@ -319,12 +329,13 @@ def _validate_format_supplemental( Recognized file extensions are: {extensions} -:param input_config: Input config source (None => read stdin). +:param input_config: Input config source (None => read ``stdin``). :param input_format: Format of the input config. -:param output_block: Path through keys to the desired output block. -:param output_file: Output config destination (None => write to stdout). +:param update_config: Input config source (None => read ``stdin``). +:param update_format: Format of the update config. +:param output_file: Output config destination (None => write to ``stdout``). :param output_format: Format of the output config. -:param supplemental_configs: Sources of values used to modify input. +:param output_block: Path through keys to the desired output block. :param values_needed: Report complete, missing, and template values. :param total: Require rendering of all Jinja2 variables/expressions. :param dry_run: Log output instead of writing to output. diff --git a/src/uwtools/strings.py b/src/uwtools/strings.py index ce7f1f7b3..b5a64f718 100644 --- a/src/uwtools/strings.py +++ b/src/uwtools/strings.py @@ -106,7 +106,6 @@ class STR: searchpath: str = "search_path" sfcclimogen: str = "sfc_climo_gen" shave: str = "shave" - suppfiles: str = "supplemental_files" targetdir: str = "target_dir" task: str = "task" tasks: str = "tasks" @@ -114,6 +113,8 @@ class STR: total: str = "total" translate: str = "translate" ungrib: str = "ungrib" + updatefile: str = "update_file" + updatefmt: str = "update_format" upp: str = "upp" validate: str = "validate" valsfile: str = "values_file" diff --git a/src/uwtools/tests/api/test_config.py b/src/uwtools/tests/api/test_config.py index a88ab25c1..96af61300 100644 --- a/src/uwtools/tests/api/test_config.py +++ b/src/uwtools/tests/api/test_config.py @@ -50,10 +50,11 @@ def test_realize(): kwargs: dict = { "input_config": "path1", "input_format": "fmt1", + "update_config": "path2", + "update_format": "fmt2", + "output_file": "path3", + "output_format": "fmt3", "output_block": None, - "output_file": "path2", - "output_format": "fmt2", - "supplemental_configs": ["path3"], "values_needed": True, "total": True, "dry_run": False, @@ -64,8 +65,8 @@ def test_realize(): **{ **kwargs, "input_config": Path(kwargs["input_config"]), + "update_config": Path(kwargs["update_config"]), "output_file": Path(kwargs["output_file"]), - "supplemental_configs": [Path(x) for x in kwargs["supplemental_configs"]], } ) @@ -74,7 +75,8 @@ def test_realize_to_dict(): kwargs: dict = { "input_config": "path1", "input_format": "fmt1", - "supplemental_configs": ["path3"], + "update_config": None, + "update_format": None, "values_needed": True, "dry_run": False, "stdin_ok": False, diff --git a/src/uwtools/tests/config/formats/test_base.py b/src/uwtools/tests/config/formats/test_base.py index 6251527bf..032fec6d6 100644 --- a/src/uwtools/tests/config/formats/test_base.py +++ b/src/uwtools/tests/config/formats/test_base.py @@ -167,7 +167,7 @@ def test_derefernce_context_override(tmp_path): @pytest.mark.parametrize("fmt2", [FORMAT.ini, FORMAT.nml, FORMAT.sh]) -def test_invalid_config(caplog, fmt2, tmp_path): +def test_invalid_config(fmt2, tmp_path): """ Test that invalid config files will error when attempting to dump. """ @@ -177,9 +177,7 @@ def test_invalid_config(caplog, fmt2, tmp_path): depthin = depth(cfgin.data) with raises(UWConfigError) as e: tools.format_to_config(fmt2).dump_dict(cfg=cfgin.data, path=outfile) - msg = f"Cannot dump depth-{depthin} config to type-'{fmt2}' config" - assert logged(caplog, msg) - assert msg in str(e.value) + assert f"Cannot dump depth-{depthin} config to type-'{fmt2}' config" in str(e.value) def test_parse_include(config): diff --git a/src/uwtools/tests/config/test_tools.py b/src/uwtools/tests/config/test_tools.py index 6f9983f63..ad41fa420 100644 --- a/src/uwtools/tests/config/test_tools.py +++ b/src/uwtools/tests/config/test_tools.py @@ -4,8 +4,11 @@ """ import logging +import sys +from io import StringIO from pathlib import Path from textwrap import dedent +from unittest.mock import patch import f90nml # type: ignore import pytest @@ -15,12 +18,15 @@ from uwtools.config import tools from uwtools.config.formats.ini import INIConfig from uwtools.config.formats.nml import NMLConfig +from uwtools.config.formats.sh import SHConfig from uwtools.config.formats.yaml import YAMLConfig from uwtools.config.support import depth from uwtools.exceptions import UWConfigError, UWError from uwtools.logging import log +from uwtools.strings import FORMAT from uwtools.tests.support import compare_files, fixture_path, logged -from uwtools.utils.file import FORMAT, writable +from uwtools.utils.file import _stdinproxy as stdinproxy +from uwtools.utils.file import writable # Fixtures @@ -54,24 +60,25 @@ def realize_config_yaml_input(tmp_path): # Helpers -def help_realize_config_fmt2fmt(infn, infmt, cfgfn, tmpdir): - infile = fixture_path(infn) - cfgfile = fixture_path(cfgfn) - ext = Path(infile).suffix - outfile = tmpdir / f"outfile{ext}" +def help_realize_config_fmt2fmt(input_file, input_format, update_file, update_format, tmpdir): + input_file = fixture_path(input_file) + update_file = fixture_path(update_file) + ext = Path(input_file).suffix + output_file = tmpdir / f"output_file{ext}" tools.realize_config( - input_config=infile, - input_format=infmt, - output_file=outfile, - output_format=infmt, - supplemental_configs=[cfgfile], + input_config=input_file, + input_format=input_format, + update_config=update_file, + update_format=update_format, + output_file=output_file, + output_format=input_format, ) - cfgclass = tools.format_to_config(infmt) - cfgobj = cfgclass(infile) - cfgobj.update_values(cfgclass(cfgfile)) + cfgclass = tools.format_to_config(input_format) + cfgobj = cfgclass(input_file) + cfgobj.update_values(cfgclass(update_file)) reference = str(tmpdir / f"expected{ext}") cfgobj.dump(reference) - assert compare_files(reference, outfile) + assert compare_files(reference, output_file) def help_realize_config_simple(infn, infmt, tmpdir): @@ -139,26 +146,22 @@ def test_compare_configs_bad_format(caplog): assert logged(caplog, msg) -def test_config_check_depths_realize_fail(caplog, realize_config_testobj): +def test_config_check_depths_realize_fail(realize_config_testobj): depthin = depth(realize_config_testobj.data) with raises(UWConfigError) as e: tools.config_check_depths_realize( config_obj=realize_config_testobj, target_format=FORMAT.ini ) - msg = f"Cannot realize depth-{depthin} config to type-'ini' config" - assert logged(caplog, msg) - assert msg in str(e.value) + assert f"Cannot realize depth-{depthin} config to type-'ini' config" in str(e.value) -def test_config_check_depths_update_fail(caplog, realize_config_testobj): +def test_config_check_depths_update_fail(realize_config_testobj): depthin = depth(realize_config_testobj.data) with raises(UWConfigError) as e: tools.config_check_depths_update( config_obj=realize_config_testobj, target_format=FORMAT.ini ) - msg = f"Cannot update depth-{depthin} config to type-'ini' config" - assert logged(caplog, msg) - assert msg in str(e.value) + assert f"Cannot update depth-{depthin} config to type-'ini' config" in str(e.value) def test_realize_config_conversion_cfg_to_yaml(tmp_path): @@ -245,7 +248,7 @@ def test_realize_config_fmt2fmt_nml2nml(tmp_path): Test that providing a namelist base input file and a config file will create and update namelist config file. """ - help_realize_config_fmt2fmt("simple.nml", FORMAT.nml, "simple2.nml", tmp_path) + help_realize_config_fmt2fmt("simple.nml", FORMAT.nml, "simple2.nml", FORMAT.nml, tmp_path) def test_realize_config_fmt2fmt_ini2ini(tmp_path): @@ -253,7 +256,7 @@ def test_realize_config_fmt2fmt_ini2ini(tmp_path): Test that providing an INI base input file and an INI config file will create and update INI config file. """ - help_realize_config_fmt2fmt("simple.ini", FORMAT.ini, "simple2.ini", tmp_path) + help_realize_config_fmt2fmt("simple.ini", FORMAT.ini, "simple2.ini", FORMAT.ini, tmp_path) def test_realize_config_fmt2fmt_yaml2yaml(tmp_path): @@ -262,7 +265,7 @@ def test_realize_config_fmt2fmt_yaml2yaml(tmp_path): config file. """ help_realize_config_fmt2fmt( - "fruit_config.yaml", FORMAT.yaml, "fruit_config_similar.yaml", tmp_path + "fruit_config.yaml", FORMAT.yaml, "fruit_config_similar.yaml", FORMAT.yaml, tmp_path ) @@ -278,115 +281,69 @@ def test_realize_config_incompatible_file_type(): ) -def test_realize_config_output_block(tmp_path, realize_config_testobj): - """ - Test that --output-block subsets the input as expected. +def test_realize_config_output_file_format(tmp_path): """ - outfile = tmp_path / "test_output.yaml" - tools.realize_config( - input_config=realize_config_testobj, - output_block=[1], - output_file=outfile, - ) - expected = """ - 2: - 3: 88""" - expected_file = tmp_path / "expected.yaml" - with open(expected_file, "w", encoding="utf-8") as f: - f.write(dedent(expected).strip()) - assert compare_files(expected_file, outfile) - - -def test_realize_config_output_block_format_conversion(tmp_path): - """ - Test that --output-block subsets the input as expected for output format. - """ - outfile = tmp_path / "test_output.nml" - d = {"a": {"b": {"c": 88}}} - tools.realize_config( - input_config=YAMLConfig(d), - output_block=["a"], - output_file=outfile, - ) - expected = """ - &b - c = 88 - /""" - expected_file = tmp_path / "expected.nml" - with open(expected_file, "w", encoding="utf-8") as f: - f.write(dedent(expected).strip()) - assert compare_files(expected_file, outfile) - - -def test_realize_config_output_file_conversion(tmp_path): - """ - Test that --output-input-type converts config object to desired object type. + Test that output_format overrides bad output_file extension. """ infile = fixture_path("simple.nml") outfile = tmp_path / "test_ouput.cfg" tools.realize_config( input_config=infile, - input_format=FORMAT.nml, output_file=outfile, output_format=FORMAT.nml, ) - expected = NMLConfig(infile) - expected_file = tmp_path / "expected.nml" - expected.dump(expected_file) - assert compare_files(expected_file, outfile) - with open(outfile, "r", encoding="utf-8") as f: - assert f.read()[-1] == "\n" + assert compare_files(outfile, infile) def test_realize_config_remove_nml_to_nml(tmp_path): - nml = NMLConfig({"constants": {"pi": 3.141, "e": 2.718}}) + input_config = NMLConfig({"constants": {"pi": 3.141, "e": 2.718}}) s = """ constants: e: !remove """ - sup = tmp_path / "sup.yaml" - with open(sup, "w", encoding="utf-8") as f: + update_config = tmp_path / "update.yaml" + with open(update_config, "w", encoding="utf-8") as f: print(dedent(s).strip(), file=f) - cfg = tmp_path / "config.nml" - assert not cfg.is_file() + output_file = tmp_path / "config.nml" + assert not output_file.is_file() tools.realize_config( - input_config=nml, - output_file=cfg, - supplemental_configs=[sup], + input_config=input_config, + update_config=update_config, + output_file=output_file, ) - assert f90nml.read(cfg) == {"constants": {"pi": 3.141}} + assert f90nml.read(output_file) == {"constants": {"pi": 3.141}} def test_realize_config_remove_yaml_to_yaml_scalar(tmp_path): - yml = YAMLConfig({"a": {"b": {"c": 11, "d": 22, "e": 33}}}) + input_config = YAMLConfig({"a": {"b": {"c": 11, "d": 22, "e": 33}}}) s = """ a: b: d: !remove """ - sup = tmp_path / "sup.yaml" - with open(sup, "w", encoding="utf-8") as f: + update_config = tmp_path / "update.yaml" + with open(update_config, "w", encoding="utf-8") as f: print(dedent(s).strip(), file=f) assert {"a": {"b": {"c": 11, "e": 33}}} == tools.realize_config( - input_config=yml, - output_format="yaml", - supplemental_configs=[sup], + input_config=input_config, + update_config=update_config, + output_format=FORMAT.yaml, ) def test_realize_config_remove_yaml_to_yaml_subtree(tmp_path): - yml = YAMLConfig(yaml.safe_load("a: {b: {c: 11, d: 22, e: 33}}")) + input_config = YAMLConfig(yaml.safe_load("a: {b: {c: 11, d: 22, e: 33}}")) s = """ a: b: !remove """ - sup = tmp_path / "sup.yaml" - with open(sup, "w", encoding="utf-8") as f: + update_config = tmp_path / "update.yaml" + with open(update_config, "w", encoding="utf-8") as f: print(dedent(s).strip(), file=f) assert {"a": {}} == tools.realize_config( - input_config=yml, - output_format="yaml", - supplemental_configs=[sup], + input_config=input_config, + update_config=update_config, + output_format=FORMAT.yaml, ) @@ -420,76 +377,47 @@ def test_realize_config_simple_yaml(tmp_path): def test_realize_config_single_dereference(capsys, tmp_path): - path = tmp_path / "a.yaml" - supplemental_path = tmp_path / "b.yaml" - with writable(path) as f: + input_config = tmp_path / "a.yaml" + update_config = tmp_path / "b.yaml" + with writable(input_config) as f: yaml.dump({"1": "a", "2": "{{ deref }}", "3": "{{ temporalis }}"}, f) - with writable(supplemental_path) as f: + with writable(update_config) as f: yaml.dump({"2": "b", "temporalis": "c", "deref": "d"}, f) tools.realize_config( - input_config=path, - input_format=FORMAT.yaml, + input_config=input_config, + update_config=update_config, output_format=FORMAT.yaml, - supplemental_configs=[supplemental_path], ) - expected = """'1': a -'2': b -'3': c -deref: d -temporalis: c -""" - actual = capsys.readouterr().out - assert actual == expected + actual = capsys.readouterr().out.strip() + expected = """ + '1': a + '2': b + '3': c + deref: d + temporalis: c + """ + assert actual == dedent(expected).strip() -def test_realize_config_supp_bad_format(tmp_path): - path = tmp_path / "a.yaml" - supplemental_path = tmp_path / "b.clj" - msg = f"Cannot deduce format of '{supplemental_path}' from unknown extension 'clj'" - with writable(path) as f: +def test_realize_config_update_bad_format(tmp_path): + input_config = tmp_path / "a.yaml" + update_config = tmp_path / "b.clj" + with writable(input_config) as f: yaml.dump({"1": "a", "2": "{{ deref }}", "3": "{{ temporalis }}", "deref": "b"}, f) - with writable(supplemental_path) as f: + with writable(update_config) as f: yaml.dump({"2": "b", "temporalis": "c"}, f) with raises(UWError) as e: tools.realize_config( - input_config=path, - input_format=FORMAT.yaml, + input_config=input_config, + update_config=update_config, output_format=FORMAT.yaml, - supplemental_configs=[supplemental_path], dry_run=True, ) + msg = f"Cannot deduce format of '{update_config}' from unknown extension 'clj'" assert msg in str(e.value) -def test_realize_config_supp_list(capsys, tmp_path): - path = tmp_path / "a.yaml" - supplemental_path = tmp_path / "b.yaml" - second_supp_path = tmp_path / "c.yaml" - with writable(path) as f: - yaml.dump({"1": "a", "2": "{{ deref }}", "3": "{{ temporalis }}"}, f) - with writable(supplemental_path) as f: - yaml.dump({"2": "b", "temporalis": "c"}, f) - with writable(second_supp_path) as f: - yaml.dump({"4": "d", "tempus": "fugit", "deref": "{{ tempus }}"}, f) - tools.realize_config( - input_config=path, - input_format=FORMAT.yaml, - output_format=FORMAT.yaml, - supplemental_configs=[supplemental_path, second_supp_path], - ) - expected = """'1': a -'2': b -'3': c -temporalis: c -'4': d -deref: fugit -tempus: fugit -""" - actual = capsys.readouterr().out - assert actual == expected - - -def test_realize_config_supp_none(capsys, tmp_path): +def test_realize_config_update_none(capsys, tmp_path): path = tmp_path / "a.yaml" with writable(path) as f: yaml.dump({"1": "a", "2": "{{ deref }}", "3": "{{ temporalis }}", "deref": "b"}, f) @@ -498,19 +426,20 @@ def test_realize_config_supp_none(capsys, tmp_path): input_format=FORMAT.yaml, output_format=FORMAT.yaml, ) - expected = """'1': a -'2': b -'3': '{{ temporalis }}' -deref: b -""" - actual = capsys.readouterr().out - assert actual == expected + expected = """ + '1': a + '2': b + '3': '{{ temporalis }}' + deref: b + """ + actual = capsys.readouterr().out.strip() + assert actual == dedent(expected).strip() def test_realize_config_total_fail(): with raises(UWConfigError) as e: tools.realize_config( - input_config=YAMLConfig({"foo": "{{ bar }}"}), output_format="yaml", total=True + input_config=YAMLConfig({"foo": "{{ bar }}"}), output_format=FORMAT.yaml, total=True ) assert str(e.value) == "Config could not be totally realized" @@ -527,28 +456,28 @@ def test_realize_config_values_needed_ini(caplog): output_format=FORMAT.ini, values_needed=True, ) - expected = """ -Keys that are complete: - salad - salad.base - salad.fruit - salad.vegetable - salad.dressing - dessert - dessert.type - dessert.side - dessert.servings - -Keys with unrendered Jinja2 variables/expressions: - salad.how_many: {{ amount }} - dessert.flavor: {{ flavor }} - -Keys that are set to empty: - salad.toppings - salad.meat -""".strip() actual = "\n".join(record.message for record in caplog.records) - assert actual == expected + expected = """ + Keys that are complete: + salad + salad.base + salad.fruit + salad.vegetable + salad.dressing + dessert + dessert.type + dessert.side + dessert.servings + + Keys with unrendered Jinja2 variables/expressions: + salad.how_many: {{ amount }} + dessert.flavor: {{ flavor }} + + Keys that are set to empty: + salad.toppings + salad.meat + """ + assert actual.strip() == dedent(expected).strip() def test_realize_config_values_needed_yaml(caplog): @@ -565,25 +494,25 @@ def test_realize_config_values_needed_yaml(caplog): ) actual = "\n".join(record.message for record in caplog.records) expected = """ -Keys that are complete: - FV3GFS - FV3GFS.nomads - FV3GFS.nomads.protocol - FV3GFS.nomads.file_names - FV3GFS.nomads.file_names.grib2 - FV3GFS.nomads.file_names.testfalse - FV3GFS.nomads.file_names.testzero - -Keys with unrendered Jinja2 variables/expressions: - FV3GFS.nomads.url: https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.{{ yyyymmdd }}/{{ hh }}/atmos - FV3GFS.nomads.file_names.grib2.anl: ['gfs.t{{ hh }}z.atmanl.nemsio', 'gfs.t{{ hh }}z.sfcanl.nemsio'] - FV3GFS.nomads.file_names.grib2.fcst: ['gfs.t{{ hh }}z.pgrb2.0p25.f{{ fcst_hr03d }}'] - -Keys that are set to empty: - FV3GFS.nomads.file_names.nemsio - FV3GFS.nomads.testempty -""".strip() - assert actual == expected + Keys that are complete: + FV3GFS + FV3GFS.nomads + FV3GFS.nomads.protocol + FV3GFS.nomads.file_names + FV3GFS.nomads.file_names.grib2 + FV3GFS.nomads.file_names.testfalse + FV3GFS.nomads.file_names.testzero + + Keys with unrendered Jinja2 variables/expressions: + FV3GFS.nomads.url: https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.{{ yyyymmdd }}/{{ hh }}/atmos + FV3GFS.nomads.file_names.grib2.anl: ['gfs.t{{ hh }}z.atmanl.nemsio', 'gfs.t{{ hh }}z.sfcanl.nemsio'] + FV3GFS.nomads.file_names.grib2.fcst: ['gfs.t{{ hh }}z.pgrb2.0p25.f{{ fcst_hr03d }}'] + + Keys that are set to empty: + FV3GFS.nomads.file_names.nemsio + FV3GFS.nomads.testempty + """ + assert actual.strip() == dedent(expected).strip() def test__ensure_format_bad(): @@ -617,12 +546,12 @@ def test__print_config_section_ini(capsys): tools._print_config_section(config_obj.data, section) actual = capsys.readouterr().out expected = """ -flavor={{ flavor }} -servings=0 -side=False -type=pie -""".lstrip() - assert actual == expected + flavor={{ flavor }} + servings=0 + side=False + type=pie + """ + assert actual.strip() == dedent(expected).strip() def test__print_config_section_ini_missing_section(): @@ -640,10 +569,10 @@ def test__print_config_section_yaml(capsys): tools._print_config_section(config_obj.data, section) actual = capsys.readouterr().out expected = """ -name=fixed -surface_value=0.0 -""".lstrip() - assert actual == expected + name=fixed + surface_value=0.0 + """ + assert actual.strip() == dedent(expected).strip() def test__print_config_section_yaml_for_nonscalar(): @@ -670,51 +599,201 @@ def test__print_config_section_yaml_not_dict(): assert "must be a dictionary" in str(e.value) -@pytest.mark.parametrize( - "supplemental_configs", [YAMLConfig(config={1: {2: {3: 99}}}), {1: {2: {3: 99}}}] -) -def test__realize_config_update(realize_config_testobj, supplemental_configs): - assert realize_config_testobj[1][2][3] == 88 - o = tools._realize_config_update( - config_obj=realize_config_testobj, - config_fmt="yaml", - supplemental_configs=[supplemental_configs], - ) - assert o[1][2][3] == 99 +def test__realize_config_input_setup_ini_cfgobj(): + data = {"section": {"foo": "bar"}} + cfgobj = INIConfig(config=data) + input_obj, input_format = tools._realize_config_input_setup(input_config=cfgobj) + assert input_obj.data == data + assert input_format == FORMAT.ini -def test__realize_config_update_noop(realize_config_testobj): - assert realize_config_testobj == tools._realize_config_update( - config_obj=realize_config_testobj, config_fmt="yaml", supplemental_configs=None - ) +def test__realize_config_input_setup_ini_file(tmp_path): + data = """ + [section] + foo = bar + """ + path = tmp_path / "config.ini" + with open(path, "w", encoding="utf-8") as f: + print(dedent(data).strip(), file=f) + input_obj, input_format = tools._realize_config_input_setup(input_config=path) + assert input_obj.data == {"section": {"foo": "bar"}} + assert input_format == FORMAT.ini -def test__realize_config_update_file(realize_config_testobj, tmp_path): - values = {1: {2: {3: 99}}} +def test__realize_config_input_setup_ini_stdin(caplog): + data = """ + [section] + foo = bar + baz = 88 + """ + stdinproxy.cache_clear() + log.setLevel(logging.DEBUG) + s = StringIO() + print(dedent(data).strip(), file=s) + s.seek(0) + with patch.object(sys, "stdin", new=s): + input_obj, input_format = tools._realize_config_input_setup(input_format=FORMAT.ini) + assert input_obj.data == {"section": {"foo": "bar", "baz": "88"}} # note: 88 is str, not int + assert input_format == FORMAT.ini + assert logged(caplog, "Reading input from stdin") + + +def test__realize_config_input_setup_nml_cfgobj(): + data = {"nl": {"pi": 3.14}} + cfgobj = NMLConfig(config=data) + input_obj, input_format = tools._realize_config_input_setup(input_config=cfgobj) + assert input_obj.data == data + assert input_format == FORMAT.nml + + +def test__realize_config_input_setup_nml_file(tmp_path): + data = """ + &nl + pi = 3.14 + / + """ + path = tmp_path / "config.nml" + with open(path, "w", encoding="utf-8") as f: + print(dedent(data).strip(), file=f) + input_obj, input_format = tools._realize_config_input_setup(input_config=path) + assert input_obj["nl"]["pi"] == 3.14 + assert input_format == FORMAT.nml + + +def test__realize_config_input_setup_nml_stdin(caplog): + data = """ + &nl + pi = 3.14 + / + """ + stdinproxy.cache_clear() + log.setLevel(logging.DEBUG) + s = StringIO() + print(dedent(data).strip(), file=s) + s.seek(0) + with patch.object(sys, "stdin", new=s): + input_obj, input_format = tools._realize_config_input_setup(input_format=FORMAT.nml) + assert input_obj["nl"]["pi"] == 3.14 + assert input_format == FORMAT.nml + assert logged(caplog, "Reading input from stdin") + + +def test__realize_config_input_setup_sh_cfgobj(): + data = {"foo": "bar"} + cfgobj = SHConfig(config=data) + input_obj, input_format = tools._realize_config_input_setup(input_config=cfgobj) + assert input_obj.data == data + assert input_format == FORMAT.sh + + +def test__realize_config_input_setup_sh_file(tmp_path): + data = """ + foo=bar + """ + path = tmp_path / "config.sh" + with open(path, "w", encoding="utf-8") as f: + print(dedent(data).strip(), file=f) + input_obj, input_format = tools._realize_config_input_setup(input_config=path) + assert input_obj.data == {"foo": "bar"} + assert input_format == FORMAT.sh + + +def test__realize_config_input_setup_sh_stdin(caplog): + data = """ + foo=bar + """ + stdinproxy.cache_clear() + log.setLevel(logging.DEBUG) + s = StringIO() + print(dedent(data).strip(), file=s) + s.seek(0) + with patch.object(sys, "stdin", new=s): + input_obj, input_format = tools._realize_config_input_setup(input_format=FORMAT.sh) + assert input_obj.data == {"foo": "bar"} + assert input_format == FORMAT.sh + assert logged(caplog, "Reading input from stdin") + + +def test__realize_config_input_setup_yaml_cfgobj(): + data = {"foo": "bar"} + cfgobj = YAMLConfig(config=data) + input_obj, input_format = tools._realize_config_input_setup(input_config=cfgobj) + assert input_obj.data == data + assert input_format == FORMAT.yaml + + +def test__realize_config_input_setup_yaml_file(tmp_path): + data = """ + foo: bar + """ path = tmp_path / "config.yaml" with open(path, "w", encoding="utf-8") as f: - yaml.dump(values, f) + print(dedent(data).strip(), file=f) + input_obj, input_format = tools._realize_config_input_setup(input_config=path) + assert input_obj.data == {"foo": "bar"} + assert input_format == FORMAT.yaml + + +def test__realize_config_input_setup_yaml_stdin(caplog): + data = """ + foo: bar + """ + stdinproxy.cache_clear() + log.setLevel(logging.DEBUG) + s = StringIO() + print(dedent(data).strip(), file=s) + s.seek(0) + with patch.object(sys, "stdin", new=s): + input_obj, input_format = tools._realize_config_input_setup(input_format=FORMAT.yaml) + assert input_obj.data == {"foo": "bar"} + assert input_format == FORMAT.yaml + assert logged(caplog, "Reading input from stdin") + + +def test__realize_config_output_setup(caplog, tmp_path): + log.setLevel(logging.DEBUG) + input_obj = YAMLConfig({"a": {"b": {"foo": "bar"}}}) + output_file = tmp_path / "output.yaml" + assert tools._realize_config_output_setup( + input_obj=input_obj, output_file=output_file, output_block=["a", "b"] + ) == ({"foo": "bar"}, FORMAT.yaml) + assert logged(caplog, f"Writing output to {output_file}") + + +def test__realize_config_update_cfgobj(realize_config_testobj): assert realize_config_testobj[1][2][3] == 88 - o = tools._realize_config_update( - config_obj=realize_config_testobj, config_fmt="yaml", supplemental_configs=[path] - ) + update_config = YAMLConfig(config={1: {2: {3: 99}}}) + o = tools._realize_config_update(input_obj=realize_config_testobj, update_config=update_config) + assert o[1][2][3] == 99 + + +def test__realize_config_update_stdin(caplog, realize_config_testobj): + stdinproxy.cache_clear() + log.setLevel(logging.DEBUG) + assert realize_config_testobj[1][2][3] == 88 + s = StringIO() + print("{1: {2: {3: 99}}}", file=s) + s.seek(0) + with patch.object(sys, "stdin", new=s): + o = tools._realize_config_update( + input_obj=realize_config_testobj, update_format=FORMAT.yaml + ) assert o[1][2][3] == 99 + assert logged(caplog, "Reading update from stdin") + + +def test__realize_config_update_noop(realize_config_testobj): + assert tools._realize_config_update(input_obj=realize_config_testobj) == realize_config_testobj -def test__realize_config_update_list(realize_config_testobj, tmp_path): +def test__realize_config_update_file(realize_config_testobj, tmp_path): + assert realize_config_testobj[1][2][3] == 88 values = {1: {2: {3: 99}}} - path = tmp_path / "config.yaml" - with open(path, "w", encoding="utf-8") as f: + update_config = tmp_path / "config.yaml" + with open(update_config, "w", encoding="utf-8") as f: yaml.dump(values, f) - values2 = {1: {2: {3: 77}}} - path2 = tmp_path / "config2.yaml" - with open(path2, "w", encoding="utf-8") as f: - yaml.dump(values2, f) - assert realize_config_testobj[1][2][3] == 88 - o = tools._realize_config_update( - config_obj=realize_config_testobj, config_fmt="yaml", supplemental_configs=[path, path2] - ) - assert o[1][2][3] == 77 + o = tools._realize_config_update(input_obj=realize_config_testobj, update_config=update_config) + assert o[1][2][3] == 99 def test__realize_config_values_needed(caplog, tmp_path): @@ -744,59 +823,16 @@ def test__realize_config_values_needed_negative_results(caplog, tmp_path): @pytest.mark.parametrize("input_fmt", FORMAT.extensions()) -@pytest.mark.parametrize("output_fmt", FORMAT.extensions()) -def test__validate_format_output(input_fmt, output_fmt): - call = lambda: tools._validate_format_output(input_fmt=input_fmt, output_fmt=output_fmt) - if FORMAT.yaml in (input_fmt, output_fmt) or input_fmt == output_fmt: +@pytest.mark.parametrize("other_fmt", FORMAT.extensions()) +def test__validate_format(input_fmt, other_fmt): + call = lambda: tools._validate_format( + other_fmt_desc="other", other_fmt=other_fmt, input_fmt=input_fmt + ) + if FORMAT.yaml in (input_fmt, other_fmt) or input_fmt == other_fmt: call() # no exception raised else: with raises(UWError) as e: call() - assert ( - str(e.value) == "Accepted output formats for input format " - f"{input_fmt} are {input_fmt} or yaml" + assert str(e.value) == "Accepted other formats for input format {x} are {x} or yaml".format( + x=input_fmt ) - - -def test__validate_format_supplemental_fail_obj(): - config_fmt = FORMAT.ini - sc = NMLConfig(config={"n": {"k": "v"}}) - with raises(UWError) as e: - tools._validate_format_supplemental(config_fmt=config_fmt, supplemental_cfg=sc, idx=87) - assert str(e.value) == "Supplemental config #88 format %s must be yaml or input format %s" % ( - FORMAT.nml, - config_fmt, - ) - - -def test__validate_format_supplemental_fail_path(): - config_fmt = FORMAT.ini - sc = Path("/path/to/config.nml") - with raises(UWError) as e: - tools._validate_format_supplemental(config_fmt=config_fmt, supplemental_cfg=sc, idx=87) - assert str(e.value) == "Supplemental config #%s format %s must be yaml or input format %s" % ( - 88, - FORMAT.nml, - config_fmt, - ) - - -def test__validate_format_supplemental_pass_dict(caplog): - log.setLevel(logging.DEBUG) - config_fmt = FORMAT.yaml - sc: dict = {} - tools._validate_format_supplemental(config_fmt=config_fmt, supplemental_cfg=sc, idx=87) - msg = "Supplemental config #%s is a dict: Cannot validate its format vs %s" % (88, config_fmt) - assert logged(caplog, msg) - - -def test__validate_format_supplemental_pass_match_obj(): - config_fmt = FORMAT.yaml - sc = YAMLConfig(config={}) - tools._validate_format_supplemental(config_fmt=config_fmt, supplemental_cfg=sc, idx=87) - - -def test__validate_format_supplemental_pass_match_path(): - config_fmt = FORMAT.yaml - sc = Path("/path/to/config.yaml") - tools._validate_format_supplemental(config_fmt=config_fmt, supplemental_cfg=sc, idx=87) diff --git a/src/uwtools/tests/support.py b/src/uwtools/tests/support.py index ee5634558..c62344d41 100644 --- a/src/uwtools/tests/support.py +++ b/src/uwtools/tests/support.py @@ -4,7 +4,7 @@ from copy import deepcopy from importlib import resources from pathlib import Path -from typing import Any, Callable +from typing import Any, Callable, Union import yaml from _pytest.logging import LogCaptureFixture @@ -13,7 +13,7 @@ from uwtools.utils.file import resource_path -def compare_files(path1: str, path2: str) -> bool: +def compare_files(path1: Union[Path, str], path2: Union[Path, str]) -> bool: """ Determines whether the two given files are identical up to any number of trailing newlines, which are ignored. Print the contents of both files when they do not match. diff --git a/src/uwtools/tests/test_cli.py b/src/uwtools/tests/test_cli.py index aeb79bae6..43afcce85 100644 --- a/src/uwtools/tests/test_cli.py +++ b/src/uwtools/tests/test_cli.py @@ -37,6 +37,22 @@ def actions(parser: Parser) -> List[str]: # Fixtures +@fixture +def args_config_realize(): + return { + STR.infile: "in", + STR.infmt: "yaml", + STR.updatefile: "update", + STR.updatefmt: "yaml", + STR.outfile: "out", + STR.outfmt: "yaml", + STR.outblock: "foo.bar", + STR.valsneeded: False, + STR.total: False, + STR.dryrun: False, + } + + @fixture def args_dispatch_file(): return { @@ -219,6 +235,26 @@ def test__check_template_render_vals_args_noop_explicit_valsfmt(): assert cli._check_template_render_vals_args(args) == args +@pytest.mark.parametrize( + "fmt,fn,ok", + [ + (None, "update.txt", False), + ("yaml", "udpate.txt", True), + (None, "update.yaml", True), + ("yaml", "update.yaml", True), + ("jpg", "udpate.yaml", True), + ], +) +def test__check_update(fmt, fn, ok): + args = {STR.updatefile: fn, STR.updatefmt: fmt} + if ok: + assert cli._check_update(args) == args + else: + with raises(UWError) as e: + cli._check_update(args) + assert "Cannot deduce format" in str(e.value) + + def test__check_verbosity_fail(capsys): log.setLevel(logging.INFO) args = {STR.quiet: True, STR.verbose: True} @@ -266,83 +302,31 @@ def test__dispatch_config_compare(): ) -def test__dispatch_config_realize(): - args = { - STR.infile: 1, - STR.infmt: 2, - STR.outblock: 3, - STR.outfile: 4, - STR.outfmt: 5, - STR.suppfiles: 6, - STR.valsneeded: 7, - STR.total: 8, - STR.dryrun: 9, - } +def test__dispatch_config_realize(args_config_realize): with patch.object(cli.uwtools.api.config, "realize") as realize: - cli._dispatch_config_realize(args) + cli._dispatch_config_realize(args_config_realize) realize.assert_called_once_with( - input_config=1, - input_format=2, - output_block=3, - output_file=4, - output_format=5, - supplemental_configs=6, - values_needed=7, - total=8, - dry_run=9, + input_config="in", + input_format="yaml", + update_config="update", + update_format="yaml", + output_file="out", + output_format="yaml", + output_block="foo.bar", + values_needed=False, + total=False, + dry_run=False, stdin_ok=True, ) -def test__dispatch_config_realize_fail(caplog): +def test__dispatch_config_realize_fail(caplog, args_config_realize): log.setLevel(logging.ERROR) - args = { - x: None - for x in ( - STR.infile, - STR.infmt, - STR.outblock, - STR.outfile, - STR.outfmt, - STR.suppfiles, - STR.valsneeded, - STR.total, - STR.dryrun, - ) - } with patch.object(cli.uwtools.api.config, "realize", side_effect=UWConfigRealizeError): - assert cli._dispatch_config_realize(args) is False + assert cli._dispatch_config_realize(args_config_realize) is False assert regex_logged(caplog, "Config could not be realized") -def test__dispatch_config_realize_no_optional(): - args = { - STR.infile: None, - STR.infmt: None, - STR.outblock: None, - STR.outfile: None, - STR.outfmt: None, - STR.suppfiles: ["/foo.vals"], - STR.valsneeded: False, - STR.total: False, - STR.dryrun: False, - } - with patch.object(cli.uwtools.api.config, "realize") as realize: - cli._dispatch_config_realize(args) - realize.assert_called_once_with( - input_config=None, - input_format=None, - output_block=None, - output_file=None, - output_format=None, - supplemental_configs=["/foo.vals"], - values_needed=False, - total=False, - dry_run=False, - stdin_ok=True, - ) - - def test__dispatch_config_validate_config_obj(): _dispatch_config_validate_args = { STR.schemafile: Path("/path/to/a.jsonschema"),