From 5487b8740c03e08506c1b7c3ecf8247a4ae88863 Mon Sep 17 00:00:00 2001 From: Alberto Contreras Date: Fri, 26 Jul 2024 21:14:06 +0200 Subject: [PATCH] doc: improve drop-in custom modules (#5548) Add group of pages for drop-in custom modules and restructure existing docs under it. Add doc for custom datasources and config modules. SC-1836 Fixes GH-4649 --- doc/rtd/development/datasource_creation.rst | 2 + doc/rtd/development/module_creation.rst | 11 ++++ doc/rtd/explanation/format.rst | 2 + doc/rtd/reference/base_config_reference.rst | 4 ++ doc/rtd/reference/cli.rst | 26 +------- doc/rtd/reference/custom_modules.rst | 24 ++++++++ .../custom_modules/custom_clean_scripts.rst | 25 ++++++++ .../custom_configuration_module.rst | 23 +++++++ .../custom_modules/custom_datasource.rst | 19 ++++++ .../custom_modules/custom_mergers.rst | 60 +++++++++++++++++++ doc/rtd/reference/index.rst | 1 + doc/rtd/reference/merging.rst | 60 +------------------ 12 files changed, 175 insertions(+), 82 deletions(-) create mode 100644 doc/rtd/reference/custom_modules.rst create mode 100644 doc/rtd/reference/custom_modules/custom_clean_scripts.rst create mode 100644 doc/rtd/reference/custom_modules/custom_configuration_module.rst create mode 100644 doc/rtd/reference/custom_modules/custom_datasource.rst create mode 100644 doc/rtd/reference/custom_modules/custom_mergers.rst diff --git a/doc/rtd/development/datasource_creation.rst b/doc/rtd/development/datasource_creation.rst index 98f9f88419ae..1b6e525b1224 100644 --- a/doc/rtd/development/datasource_creation.rst +++ b/doc/rtd/development/datasource_creation.rst @@ -170,6 +170,8 @@ Datasources included in upstream cloud-init benefit from ongoing maintenance, compatibility with the rest of the codebase, and security fixes by the upstream development team. +If this is not possible, one can add +:ref:`custom out-of-tree datasources` to cloud-init. .. _make-mime: https://cloudinit.readthedocs.io/en/latest/explanation/instancedata.html#storage-locations .. _DMI: https://www.dmtf.org/sites/default/files/standards/documents/DSP0005.pdf diff --git a/doc/rtd/development/module_creation.rst b/doc/rtd/development/module_creation.rst index 32240ab3e910..3e10a1ee00b5 100644 --- a/doc/rtd/development/module_creation.rst +++ b/doc/rtd/development/module_creation.rst @@ -163,6 +163,17 @@ in the correct location based on dependencies. If your module has no particular dependencies or is not necessary for a later boot stage, it should be placed in the ``cloud_final_modules`` section before the ``final-message`` module. +Benefits of including your config module in upstream cloud-init +=============================================================== + +Config modules included in upstream cloud-init benefit from ongoing +maintenance, +compatibility with the rest of the codebase, and security fixes by the upstream +development team. + +If this is not possible, one can add +:ref:`custom out-of-tree config modules` +to cloud-init. .. _MetaSchema: https://github.com/canonical/cloud-init/blob/3bcffacb216d683241cf955e4f7f3e89431c1491/cloudinit/config/schema.py#L58 .. _OSFAMILIES: https://github.com/canonical/cloud-init/blob/3bcffacb216d683241cf955e4f7f3e89431c1491/cloudinit/distros/__init__.py#L35 diff --git a/doc/rtd/explanation/format.rst b/doc/rtd/explanation/format.rst index b154dd66036e..8f14ccdb6c54 100644 --- a/doc/rtd/explanation/format.rst +++ b/doc/rtd/explanation/format.rst @@ -202,6 +202,8 @@ Example of once-per-instance script fi sudo echo $INSTANCE_ID > $PERSIST_ID +.. _user_data_formats-part_handler: + Part-handler ============ diff --git a/doc/rtd/reference/base_config_reference.rst b/doc/rtd/reference/base_config_reference.rst index 9686d456d11d..824841185537 100644 --- a/doc/rtd/reference/base_config_reference.rst +++ b/doc/rtd/reference/base_config_reference.rst @@ -28,6 +28,8 @@ distribution supported by ``cloud-init``. Base configuration keys ======================= +.. _base_config_module_keys: + Module keys ----------- @@ -221,6 +223,8 @@ Other keys The :ref:`network configuration` to be applied to this instance. +.. _base_config_datasource_pkg_list: + ``datasource_pkg_list`` ^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/doc/rtd/reference/cli.rst b/doc/rtd/reference/cli.rst index 9c0bbe9c3ee9..eb800b22a759 100644 --- a/doc/rtd/reference/cli.rst +++ b/doc/rtd/reference/cli.rst @@ -83,30 +83,8 @@ re-run all stages as it did on first boot. .. note:: - Cloud-init provides the directory :file:`/etc/cloud/clean.d/` for third party - applications which need additional configuration artifact cleanup from - the filesystem when the `clean` command is invoked. - - The :command:`clean` operation is typically performed by image creators - when preparing a golden image for clone and redeployment. The clean command - removes any cloud-init semaphores, allowing cloud-init to treat the next - boot of this image as the "first boot". When the image is next booted - cloud-init will performing all initial configuration based on any valid - datasource meta-data and user-data. - - Any executable scripts in this subdirectory will be invoked in lexicographical - order with run-parts when running the :command:`clean` command. - - Typical format of such scripts would be a ##- like the following: - :file:`/etc/cloud/clean.d/99-live-installer` - - An example of a script is: - - .. code-block:: bash - - sudo rm -rf /var/lib/installer_imgs/ - sudo rm -rf /var/log/installer/ - + The operations performed by `clean` can be supplemented / customized. See: + :ref:`custom_clean_scripts`. .. _cli_collect_logs: diff --git a/doc/rtd/reference/custom_modules.rst b/doc/rtd/reference/custom_modules.rst new file mode 100644 index 000000000000..3145e723bd78 --- /dev/null +++ b/doc/rtd/reference/custom_modules.rst @@ -0,0 +1,24 @@ +Custom Modules +************** + +This includes reference documentation on how to extend cloud-init with +custom / out-of-tree functionality. + +.. _custom_formats: + +Custom Formats +============== + +One can define custom data formats by presenting a +:ref:`#part-handler` +config via user-data or vendor-data. + +----- + +.. toctree:: + :maxdepth: 1 + + custom_modules/custom_clean_scripts.rst + custom_modules/custom_configuration_module.rst + custom_modules/custom_datasource.rst + custom_modules/custom_mergers.rst diff --git a/doc/rtd/reference/custom_modules/custom_clean_scripts.rst b/doc/rtd/reference/custom_modules/custom_clean_scripts.rst new file mode 100644 index 000000000000..955668fb2668 --- /dev/null +++ b/doc/rtd/reference/custom_modules/custom_clean_scripts.rst @@ -0,0 +1,25 @@ +.. _custom_clean_scripts: + +Custom Clean Scripts +******************** + +Cloud-init provides the directory :file:`/etc/cloud/clean.d/` for third party +applications which need additional configuration artifact cleanup from +the filesystem when the :ref:`cloud-init clean` command is invoked. + +The :command:`clean` operation is typically performed by image creators +when preparing a golden image for clone and redeployment. The clean command +removes any cloud-init internal state, allowing cloud-init to treat the next +boot of this image as the "first boot". +Any executable scripts in this subdirectory will be invoked in lexicographical +order when running the :command:`clean` command. + +Example +======= + +.. code-block:: bash + + $ cat /etc/cloud/clean.d/99-live-installer + #!/bin/sh + sudo rm -rf /var/lib/installer_imgs/ + sudo rm -rf /var/log/installer/ diff --git a/doc/rtd/reference/custom_modules/custom_configuration_module.rst b/doc/rtd/reference/custom_modules/custom_configuration_module.rst new file mode 100644 index 000000000000..a26adf26eb48 --- /dev/null +++ b/doc/rtd/reference/custom_modules/custom_configuration_module.rst @@ -0,0 +1,23 @@ +.. _custom_configuration_module: + +Custom Configuration Module +*************************** + +Custom 3rd-party out-of-tree configuration modules can be added to cloud-init +by: + +#. :ref:`Implement a config module` in a Python file with its + name starting with ``cc_``. + +#. Place the file where the rest of config modules are located. + On Ubuntu this path is typically: + `/usr/lib/python3/dist-packages/cloudinit/config/`. + +#. Extend the :ref:`base-configuration's ` + ``cloud_init_modules``, ``cloud_config_modules`` or ``cloud_final_modules`` + to let the config module run on one of those stages. + +.. warning :: + The config jsonschema validation functionality is going to complain about + unknown config keys introduced by custom modules and there is not an easy + way for custom modules to define their keys schema-wise. diff --git a/doc/rtd/reference/custom_modules/custom_datasource.rst b/doc/rtd/reference/custom_modules/custom_datasource.rst new file mode 100644 index 000000000000..2d5aa6c84630 --- /dev/null +++ b/doc/rtd/reference/custom_modules/custom_datasource.rst @@ -0,0 +1,19 @@ +.. _custom_datasource: + +Custom DataSource +***************** + +Custom 3rd-party out-of-tree DataSources can be added to cloud-init by: + +#. :ref:`Implement a DataSource` in a Python file. + +#. Place that file in as a single Python module or package in folder included + in ``$PYTHONPATH``. + +#. Extend the base configuration's + :ref:`datasource_pkg_list` to include the + Python package where the DataSource is located. + +#. Extend the :ref:`base-configuration`'s + :ref:`datasource_list` to include the name of + the custom DataSource. diff --git a/doc/rtd/reference/custom_modules/custom_mergers.rst b/doc/rtd/reference/custom_modules/custom_mergers.rst new file mode 100644 index 000000000000..b1af2c1d9f64 --- /dev/null +++ b/doc/rtd/reference/custom_modules/custom_mergers.rst @@ -0,0 +1,60 @@ +.. _custom_mergers: + +Custom Mergers +************** + +It is possible for users to inject their own :ref:`merging` +files to handle specific types of merging as they choose (the +basic ones included will handle lists, dicts, and strings). + +A `merge class` is a class definition providing functions that can be used +to merge a given type with another given type. + +An example of one of these `merging classes` is the following: + +.. code-block:: python + + class Merger: + def __init__(self, merger, opts): + self._merger = merger + self._overwrite = 'overwrite' in opts + + # This merging algorithm will attempt to merge with + # another dictionary, on encountering any other type of object + # it will not merge with said object, but will instead return + # the original value + # + # On encountering a dictionary, it will create a new dictionary + # composed of the original and the one to merge with, if 'overwrite' + # is enabled then keys that exist in the original will be overwritten + # by keys in the one to merge with (and associated values). Otherwise + # if not in overwrite mode the 2 conflicting keys themselves will + # be merged. + def _on_dict(self, value, merge_with): + if not isinstance(merge_with, (dict)): + return value + merged = dict(value) + for (k, v) in merge_with.items(): + if k in merged: + if not self._overwrite: + merged[k] = self._merger.merge(merged[k], v) + else: + merged[k] = v + else: + merged[k] = v + return merged + +There is an ``_on_dict`` method here that will be given a +source value, and a value to merge with. The result will be the merged object. + +This code itself is called by another merging class which "directs" the +merging to happen by analysing the object types to merge, and attempting to +find a known object that will merge that type. An example of this can be found +in the :file:`mergers/__init__.py` file (see ``LookupMerger`` and +``UnknownMerger``). + +Note how each +merge can have options associated with it, which affect how the merging is +performed. For example, a dictionary merger can be told to overwrite instead +of attempting to merge, or a string merger can be told to append strings +instead of discarding other strings to merge with. diff --git a/doc/rtd/reference/index.rst b/doc/rtd/reference/index.rst index 14e754b295f7..d1791fa96314 100644 --- a/doc/rtd/reference/index.rst +++ b/doc/rtd/reference/index.rst @@ -25,3 +25,4 @@ matrices and so on. ubuntu_stable_release_updates.rst breaking_changes.rst user_files.rst + custom_modules.rst diff --git a/doc/rtd/reference/merging.rst b/doc/rtd/reference/merging.rst index 7f1fc022f178..097892e25360 100644 --- a/doc/rtd/reference/merging.rst +++ b/doc/rtd/reference/merging.rst @@ -94,64 +94,8 @@ merging is done on other types. Customisation ============= -Because the above merging algorithm may not always be desired (just as the -previous merging algorithm was not always the preferred one), the concept of -customised merging was introduced through `merge classes`. - -A `merge class` is a class definition providing functions that can be used -to merge a given type with another given type. - -An example of one of these `merging classes` is the following: - -.. code-block:: python - - class Merger: - def __init__(self, merger, opts): - self._merger = merger - self._overwrite = 'overwrite' in opts - - # This merging algorithm will attempt to merge with - # another dictionary, on encountering any other type of object - # it will not merge with said object, but will instead return - # the original value - # - # On encountering a dictionary, it will create a new dictionary - # composed of the original and the one to merge with, if 'overwrite' - # is enabled then keys that exist in the original will be overwritten - # by keys in the one to merge with (and associated values). Otherwise - # if not in overwrite mode the 2 conflicting keys themselves will - # be merged. - def _on_dict(self, value, merge_with): - if not isinstance(merge_with, (dict)): - return value - merged = dict(value) - for (k, v) in merge_with.items(): - if k in merged: - if not self._overwrite: - merged[k] = self._merger.merge(merged[k], v) - else: - merged[k] = v - else: - merged[k] = v - return merged - -As you can see, there is an ``_on_dict`` method here that will be given a -source value, and a value to merge with. The result will be the merged object. - -This code itself is called by another merging class which "directs" the -merging to happen by analysing the object types to merge, and attempting to -find a known object that will merge that type. An example of this can be found -in the :file:`mergers/__init__.py` file (see ``LookupMerger`` and -``UnknownMerger``). - -So, following the typical ``cloud-init`` approach of allowing source code to -be downloaded and used dynamically, it is possible for users to inject their -own merging files to handle specific types of merging as they choose (the -basic ones included will handle lists, dicts, and strings). Note how each -merge can have options associated with it, which affect how the merging is -performed. For example, a dictionary merger can be told to overwrite instead -of attempting to merge, or a string merger can be told to append strings -instead of discarding other strings to merge with. +Custom 3rd party mergers can be defined, for more info visit +:ref:`custom_mergers`. How to activate ===============