diff --git a/doc/examples/part-handler.txt b/doc/examples/part-handler.txt index 7cc356f6346..020c8302f80 100644 --- a/doc/examples/part-handler.txt +++ b/doc/examples/part-handler.txt @@ -1,22 +1,58 @@ #part-handler +"""This is a trivial example part-handler that creates a file with the path +specified in the payload. It performs no input checking or error handling. + +To use it, first save the file you are currently viewing into your current +working directory. Then run the following: +``` +$ echo '/var/tmp/my_path' > part +$ cloud-init devel make-mime -a part-handler.py:part-handler -a part:x-my-path --force > user-data +``` + +This will create a mime file with the contents of 'part' and the +part-handler. You can now pass 'user-data' to your cloud of choice. + +When run, cloud-init will have created an empty file at /var/tmp/my_path. +""" + +import pathlib +from typing import Any + +from cloudinit.cloud import Cloud + + def list_types(): - # return a list of mime-types that are handled by this module - return(["text/plain", "text/go-cubs-go"]) - -def handle_part(data, ctype, filename, payload): - # data: the cloudinit object - # ctype: '__begin__', '__end__', or the specific mime-type of the part - # filename: the filename for the part, or dynamically generated part if - # no filename is given attribute is present - # payload: the content of the part (empty for begin or end) + """Return a list of mime-types that are handled by this module.""" + return ["text/x-my-path"] + + +def handle_part(data: Cloud, ctype: str, filename: str, payload: Any): + """Handle a part with the given mime-type. + + This function will get called multiple times. The first time is + to allow any initial setup needed to handle parts. It will then get + called once for each part matching the mime-type returned by `list_types`. + Finally, it will get called one last time to allow for any final + teardown. + + :data: A `Cloud` instance. This will be the same instance for each call + to handle_part. + :ctype: '__begin__', '__end__', or the mime-type + (for this example 'text/x-my-path') of the part + :filename: The filename for the part as defined in the MIME archive, + or dynamically generated part if no filename is given + :payload: The content of the part. This will be + `None` when `ctype` is '__begin__' or '__end__'. + """ if ctype == "__begin__": - print("my handler is beginning") - return + # Any custom setup needed before handling payloads + return + if ctype == "__end__": - print("my handler is ending") - return + # Any custom teardown needed after handling payloads can happen here + return - print(f"==== received ctype={ctype} filename={filename} ====") - print(payload) - print(f"==== end ctype={ctype} filename={filename}") + # If we've made it here, we're dealing with a real payload, so handle + # it appropriately + pathlib.Path(payload.strip()).touch() diff --git a/doc/rtd/explanation/boot.rst b/doc/rtd/explanation/boot.rst index b1421a209a5..a975ca7a093 100644 --- a/doc/rtd/explanation/boot.rst +++ b/doc/rtd/explanation/boot.rst @@ -137,7 +137,7 @@ mounted, including ones that have stale (previous instance) references in :file:`/etc/fstab`. As such, entries in :file:`/etc/fstab` other than those necessary for cloud-init to run should not be done until after this stage. -A part-handler and :ref:`boothooks` +A part-handler and :ref:`boothooks` will run at this stage. After this stage completes, expect to be able to access the system via serial diff --git a/doc/rtd/explanation/format.rst b/doc/rtd/explanation/format.rst index 8f14ccdb6c5..bed2b61af11 100644 --- a/doc/rtd/explanation/format.rst +++ b/doc/rtd/explanation/format.rst @@ -3,18 +3,53 @@ User data formats ***************** -User data is opaque configuration data provided by a platform to an instance at -launch configure the instance. User data can be one of the following types. +User data is configuration data provided by a user of a cloud platform to an +instance at launch. User data can be passed to cloud-init in any of many +formats documented here. + +Configuration types +=================== + +User data formats can be categorized into those that directly configure the +instance, and those that serve as a container, template, or means to obtain +or modify another configuration. + +Formats that directly configure the instance: + +- `Cloud config data`_ +- `User data script`_ +- `Cloud boothook`_ + +Formats that deal with other user data formats: + +- `Include file`_ +- `Jinja template`_ +- `MIME multi-part archive`_ +- `Cloud config archive`_ +- `Part handler`_ +- `Gzip compressed content`_ .. _user_data_formats-cloud_config: Cloud config data ================= -Cloud-config is the preferred user data format. The cloud config format is a -declarative syntax which uses `YAML version 1.1`_ with keys which describe -desired instance state. Cloud-config can be used to define how an instance -should be configured in a human-friendly format. +Example +------- + +.. code-block:: yaml + + #cloud-config + password: password + chpasswd: + expire: False + +Explanation +----------- + +Cloud-config can be used to define how an instance should be configured +in a human-friendly format. The cloud config format uses `YAML`_ with +keys which describe desired instance state. These things may include: @@ -24,93 +59,190 @@ These things may include: - importing certain SSH keys or host keys - *and many more...* -See the :ref:`yaml_examples` section for a commented set of examples of -supported cloud config formats. - -Begins with: ``#cloud-config`` or ``Content-Type: text/cloud-config`` when -using a MIME archive. +Many modules are available to process cloud-config data. These modules +may run once per instance, every boot, or once ever. See the associated +module to determine the run frequency. -.. note:: - Cloud config data can also render cloud instance metadata variables using - :ref:`jinja templates `. +For more information, see the cloud config +:ref:`example configurations ` or the cloud config +:ref:`modules reference`. .. _user_data_script: User data script ================ -Typically used by those who just want to execute a shell script. +Example +------- + +.. code-block:: shell + + #!/bin/sh + echo "Hello World" > /var/tmp/output.txt + +Explanation +----------- -Begins with: ``#!`` or ``Content-Type: text/x-shellscript`` when using a MIME -archive. +A user data script is a single script to be executed once per instance. +User data scripts are run relatively late in the boot process, during +cloud-init's :ref:`final stage` as part of the +:ref:`cc_scripts_user` module. When run, +the environment variable ``INSTANCE_ID`` is set to the current instance ID +for use within the script. -User data scripts can optionally render cloud instance metadata variables using -:ref:`jinja templates `. +.. _user_data_formats-cloud_boothook: -Example script +Cloud boothook +============== + +Simple Example -------------- -Create a script file :file:`myscript.sh` that contains the following: +.. code-block:: shell -.. code-block:: + #cloud-boothook + #!/bin/sh + echo 192.168.1.130 us.archive.ubuntu.com > /etc/hosts + +Example of once-per-instance script +----------------------------------- +.. code-block:: bash + + #cloud-boothook #!/bin/sh - echo "Hello World. The time is now $(date -R)!" | tee /root/output.txt -Now run: + PERSIST_ID=/var/lib/cloud/first-instance-id + _id="" + if [ -r $PERSIST_ID ]; then + _id=$(cat /var/lib/cloud/first-instance-id) + fi -.. code-block:: shell-session + if [ -z $_id ] || [ $INSTANCE_ID != $_id ]; then + echo 192.168.1.130 us.archive.ubuntu.com >> /etc/hosts + fi + sudo echo $INSTANCE_ID > $PERSIST_ID - $ euca-run-instances --key mykey --user-data-file myscript.sh ami-a07d95c9 +Explanation +----------- -Kernel command line -=================== +A cloud boothook is similar to a :ref:`user data script` +in that it is a script run on boot. When run, +the environment variable ``INSTANCE_ID`` is set to the current instance ID +for use within the script. -When using the NoCloud datasource, users can pass user data via the kernel -command line parameters. See the :ref:`NoCloud datasource` -and :ref:`explanation/kernel-command-line:Kernel command line` documentation -for more details. +The boothook is different in that: -Gzip compressed content -======================= +* It is run very early in boot, during the :ref:`network` stage, + before any cloud-init modules are run. +* It is run on every boot -Content found to be gzip compressed will be uncompressed. -The uncompressed data will then be used as if it were not compressed. -This is typically useful because user data is limited to ~16384 [#]_ bytes. +Include file +============ + +Example +------- + +.. code-block:: text + + #include + https://raw.githubusercontent.com/canonical/cloud-init/403f70b930e3ce0f05b9b6f0e1a38d383d058b53/doc/examples/cloud-config-run-cmds.txt + https://raw.githubusercontent.com/canonical/cloud-init/403f70b930e3ce0f05b9b6f0e1a38d383d058b53/doc/examples/cloud-config-boot-cmds.txt + +Explanation +----------- + +An include file contains a list of URLs, one per line. Each of the URLs will +be read and their content can be any kind of user data format, both base +config and meta config. If an error occurs reading a file the remaining files +will not be read. + +Jinja template +============== + +Example cloud-config +-------------------- + +.. code-block:: yaml + + ## template: jinja + #cloud-config + runcmd: + - echo 'Running on {{ v1.cloud_name }}' > /var/tmp/cloud_name + +Example user data script +------------------------ + +.. code-block:: shell + + ## template: jinja + #!/bin/sh + echo 'Current instance id: {{ v1.instance_id }}' > /var/tmp/instance_id + +Explanation +----------- + +`Jinja templating `_ may be used for +cloud-config and user data scripts. Any +:ref:`instance-data variables` may be used +as jinja template variables. Any jinja templated configuration must contain +the original header along with the new jinja header above it. + +.. note:: + Use of Jinja templates is ONLY supported for cloud-config and user data + scripts. Jinja templates are not supported for cloud-boothooks or + meta configs. + +.. _user_data_formats-mime_archive: MIME multi-part archive ======================= -This list of rules is applied to each part of this multi-part file. +Example +------- + +.. code-block:: + + Content-Type: multipart/mixed; boundary="===============2389165605550749110==" + MIME-Version: 1.0 + Number-Attachments: 2 + + --===============2389165605550749110== + Content-Type: text/cloud-boothook; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Disposition: attachment; filename="part-001" + + #!/bin/sh + echo "this is from a boothook." > /var/tmp/boothook.txt + + --===============2389165605550749110== + Content-Type: text/cloud-config; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Disposition: attachment; filename="part-002" + + bootcmd: + - echo "this is from a cloud-config." > /var/tmp/bootcmd.txt + --===============2389165605550749110==-- + +Explanation +----------- + Using a MIME multi-part file, the user can specify more than one type of data. For example, both a user data script and a cloud-config type could be specified. -Supported content-types are listed from the ``cloud-init`` subcommand +Each part must specify a valid +:ref:`content types`. Supported content-types +may also be listed from the ``cloud-init`` subcommand :command:`make-mime`: .. code-block:: shell-session $ cloud-init devel make-mime --list-types -Example output: - -.. code-block:: - - cloud-boothook - cloud-config - cloud-config-archive - cloud-config-jsonp - jinja2 - part-handler - x-include-once-url - x-include-url - x-shellscript - x-shellscript-per-boot - x-shellscript-per-instance - x-shellscript-per-once - Helper subcommand to generate MIME messages ------------------------------------------- @@ -121,8 +253,7 @@ The :command:`make-mime` subcommand takes pairs of (filename, "text/" mime subtype) separated by a colon (e.g., ``config.yaml:cloud-config``) and emits a MIME multipart message to :file:`stdout`. -Examples --------- +**MIME subcommand Examples** Create user data containing both a cloud-config (:file:`config.yaml`) and a shell script (:file:`script.sh`) @@ -141,102 +272,56 @@ Create user data containing 3 shell scripts: $ cloud-init devel make-mime -a always.sh:x-shellscript-per-boot -a instance.sh:x-shellscript-per-instance -a once.sh:x-shellscript-per-once -``include`` file -================ - -This content is an :file:`include` file. - -The file contains a list of URLs, one per line. Each of the URLs will be read -and their content will be passed through this same set of rules, i.e., the -content read from the URL can be gzipped, MIME multi-part, or plain text. If -an error occurs reading a file the remaining files will not be read. -Begins with: ``#include`` or ``Content-Type: text/x-include-url`` when using -a MIME archive. +Cloud config archive +==================== -``cloud-boothook`` -================== +Example +------- -One line ``#cloud-boothook`` header and then executable payload. +.. code-block:: shell -This is run very early on the boot process, during the -:ref:`Network boot stage`, even before ``cc_bootcmd``. + #cloud-config-archive + - type: "text/cloud-boothook" + content: | + #!/bin/sh + echo "this is from a boothook." > /var/tmp/boothook.txt + - type: "text/cloud-config" + content: | + bootcmd: + - echo "this is from a cloud-config." > /var/tmp/bootcmd.txt -This can be used when something has to be configured very early on boot, -potentially on every boot, with less convenience as ``cc_bootcmd`` but more -flexibility. +Explanation +----------- -.. note:: - Boothooks are executed on every boot. - The environment variable ``INSTANCE_ID`` will be set to the current instance - ID. ``INSTANCE_ID`` can be used to implement a `once-per-instance` type of - functionality. +A cloud-config-archive is a way to specify more than one type of data +using YAML. Since building a MIME multipart archive can be somewhat unwieldly +to build by hand or requires using a cloud-init helper utility, the +cloud-config-archive provides a simpler alternative to building the MIME +multi-part archive for those that would prefer to use YAML. -Begins with: ``#cloud-boothook``. +The format is a list of dictionaries. -Example with simple script --------------------------- +Required fields: -.. code-block:: bash +* ``type``: The :ref:`Content-Type` + identifier for the type of user data in content +* ``content``: The user data configuration - #cloud-boothook - #!/bin/sh - echo 192.168.1.130 us.archive.ubuntu.com > /etc/hosts +Optional fields: -Example of once-per-instance script ------------------------------------ +* ``launch-index``: The EC2 Launch-Index (if applicable) +* ``filename``: This field is only used if using a user data format that + requires a filename in a MIME part. This is unrelated to any local system + file. -.. code-block:: bash - - #cloud-boothook - #!/bin/sh - - PERSIST_ID=/var/lib/cloud/first-instance-id - _id="" - if [ -r $PERSIST_ID ]; then - _id=$(cat /var/lib/cloud/first-instance-id) - fi - - if [ -z $_id ] || [ $INSTANCE_ID != $_id ]; then - echo 192.168.1.130 us.archive.ubuntu.com >> /etc/hosts - fi - sudo echo $INSTANCE_ID > $PERSIST_ID +All other fields will be interpreted as a MIME part header. .. _user_data_formats-part_handler: -Part-handler +Part handler ============ -This is a `part-handler`: It contains custom code for either supporting new -mime-types in multi-part user data, or overriding the existing handlers for -supported mime-types. It will be written to a file in -:file:`/var/lib/cloud/data` based on its filename (which is generated). - -This must be Python code that contains a ``list_types`` function and a -``handle_part`` function. Once the section is read the ``list_types`` method -will be called. It must return a list of mime-types that this `part-handler` -handles. Since MIME parts are processed in order, a `part-handler` part -must precede any parts with mime-types it is expected to handle in the same -user data. - -The ``handle_part`` function must be defined like: - -.. code-block:: python - - def handle_part(data, ctype, filename, payload): - # data = the cloudinit object - # ctype = "__begin__", "__end__", or the mime-type of the part that is being handled. - # filename = the filename of the part (or a generated filename if none is present in mime data) - # payload = the parts' content - -``Cloud-init`` will then call the ``handle_part`` function once before it -handles any parts, once per part received, and once after all parts have been -handled. The ``'__begin__'`` and ``'__end__'`` sentinels allow the part -handler to do initialisation or teardown before or after receiving any parts. - -Begins with: ``#part-handler`` or ``Content-Type: text/part-handler`` when -using a MIME archive. - Example ------- @@ -244,17 +329,63 @@ Example :language: python :linenos: -Also, `this blog post`_ offers another example for more advanced usage. -Disabling user data -=================== +Explanation +----------- + +A part handler contains custom code for either supporting new +mime-types in multi-part user data or for overriding the existing handlers for +supported mime-types. + +See the :ref:`custom part handler` reference documentation +for details on writing custom handlers along with an annotated example. + +`This blog post`_ offers another example for more advanced usage. + +Gzip compressed content +======================= + +Content found to be gzip compressed will be uncompressed. +The uncompressed data will then be used as if it were not compressed. +This is typically useful because user data size may be limited based on +cloud platform. + +.. _user_data_formats-content_types: + +Headers and content types +========================= + +In order for cloud-init to recognize which user data format is being used, +the user data must contain a header. Additionally, if the user data +is being passed as a multi-part message, such as MIME, cloud-config-archive, +or part-handler, the content-type for each part must also be set +appropriately. + +The table below lists the headers and content types for each user data format. +Note that gzip compressed content is not represented here as it gets passed +as binary data and so may be processed automatically. + ++--------------------+-----------------------------+-------------------------+ +|User data format |Header |Content-Type | ++====================+=============================+=========================+ +|Cloud config data |#cloud-config |text/cloud-config | ++--------------------+-----------------------------+-------------------------+ +|User data script |#! |text/x-shellscript | ++--------------------+-----------------------------+-------------------------+ +|Cloud boothook |#cloud-boothook |text/cloud-boothook | ++--------------------+-----------------------------+-------------------------+ +|MIME multi-part |Content-Type: multipart/mixed|multipart/mixed | ++--------------------+-----------------------------+-------------------------+ +|Cloud config archive|#cloud-config-archive |text/cloud-config-archive| ++--------------------+-----------------------------+-------------------------+ +|Jinja template |## template: jinja |text/jinja | ++--------------------+-----------------------------+-------------------------+ +|Include file |#include |text/x-include-url | ++--------------------+-----------------------------+-------------------------+ +|Part handler |#part-handler |text/part-handler | ++--------------------+-----------------------------+-------------------------+ -``Cloud-init`` can be configured to ignore any user data provided to instance. -This allows custom images to prevent users from accidentally breaking closed -appliances. Setting ``allow_userdata: false`` in the configuration will disable -``cloud-init`` from processing user data. .. _make-mime: https://github.com/canonical/cloud-init/blob/main/cloudinit/cmd/devel/make_mime.py -.. _YAML version 1.1: https://yaml.org/spec/1.1/current.html -.. [#] See your cloud provider for applicable user-data size limitations... -.. _this blog post: http://foss-boss.blogspot.com/2011/01/advanced-cloud-init-custom-handlers.html +.. _YAML: https://yaml.org/spec/1.1/current.html +.. _This blog post: http://foss-boss.blogspot.com/2011/01/advanced-cloud-init-custom-handlers.html diff --git a/doc/rtd/explanation/instancedata.rst b/doc/rtd/explanation/instancedata.rst index 650efa79452..d2aadc083ee 100644 --- a/doc/rtd/explanation/instancedata.rst +++ b/doc/rtd/explanation/instancedata.rst @@ -165,7 +165,10 @@ Storage locations unredacted JSON blob. * :file:`/run/cloud-init/combined-cloud-config.json`: root-readable unredacted JSON blob. Any meta-data, vendor-data and user-data overrides - are applied to the :file:`/run/cloud-init/combined-cloud-config.json` config values. + are applied to the :file:`/run/cloud-init/combined-cloud-config.json` config + values. + +.. _instance_metadata-keys: :file:`instance-data.json` top level keys ----------------------------------------- diff --git a/doc/rtd/explanation/vendordata.rst b/doc/rtd/explanation/vendordata.rst index 621fcdeb3d9..a2340c2fab9 100644 --- a/doc/rtd/explanation/vendordata.rst +++ b/doc/rtd/explanation/vendordata.rst @@ -20,19 +20,7 @@ caveats: required for the instance to run, then vendor data should not be used. 4. User-supplied cloud-config is merged over cloud-config from vendor data. -Users providing cloud-config data can use the ``#cloud-config-jsonp`` method -to more finely control their modifications to the vendor-supplied -cloud-config. For example, if both vendor and user have provided ``runcmd`` -then the default merge handler will cause the user's ``runcmd`` to override -the one provided by the vendor. To append to ``runcmd``, the user could better -provide multi-part input with a ``cloud-config-jsonp`` part like: - -.. code:: yaml - - #cloud-config-jsonp - [{ "op": "add", "path": "/runcmd", "value": ["my", "command", "here"]}] - -Further, we strongly advise vendors to not "be evil". By evil, we mean any +Further, we strongly advise vendors to ensure you protect against any action that could compromise a system. Since users trust you, please take care to make sure that any vendor data is safe, atomic, idempotent and does not put your users at risk. diff --git a/doc/rtd/reference/base_config_reference.rst b/doc/rtd/reference/base_config_reference.rst index 82484118553..2d13675e68c 100644 --- a/doc/rtd/reference/base_config_reference.rst +++ b/doc/rtd/reference/base_config_reference.rst @@ -267,6 +267,14 @@ Format is a dict with ``enabled`` and ``prefix`` keys: ``vendor_data``. * ``prefix``: A path to prepend to any ``vendor_data``-provided script. +``allow_userdata`` +^^^^^^^^^^^^^^^^^^ + +A boolean value to disable the use of user data. +This allows custom images to prevent users from accidentally breaking closed +appliances. Setting ``allow_userdata: false`` in the configuration will disable +``cloud-init`` from processing user data. + ``manual_cache_clean`` ^^^^^^^^^^^^^^^^^^^^^^ diff --git a/doc/rtd/reference/custom_modules.rst b/doc/rtd/reference/custom_modules.rst index 3145e723bd7..4ce423dd52b 100644 --- a/doc/rtd/reference/custom_modules.rst +++ b/doc/rtd/reference/custom_modules.rst @@ -6,13 +6,6 @@ custom / out-of-tree functionality. .. _custom_formats: -Custom Formats -============== - -One can define custom data formats by presenting a -:ref:`#part-handler` -config via user-data or vendor-data. - ----- .. toctree:: @@ -22,3 +15,4 @@ config via user-data or vendor-data. custom_modules/custom_configuration_module.rst custom_modules/custom_datasource.rst custom_modules/custom_mergers.rst + custom_modules/custom_part_handlers.rst diff --git a/doc/rtd/reference/custom_modules/custom_part_handlers.rst b/doc/rtd/reference/custom_modules/custom_part_handlers.rst new file mode 100644 index 00000000000..501dc7af7be --- /dev/null +++ b/doc/rtd/reference/custom_modules/custom_part_handlers.rst @@ -0,0 +1,32 @@ +.. _custom_part_handler: + +Custom Part Handler +******************* + +This must be Python code that contains a ``list_types`` function and a +``handle_part`` function. + +The ``list_types`` function takes no arguments and must return a list +of :ref:`content types` that this +part handler handles. These can include custom content types or built-in +content types that this handler will override. + +The ``handle_part`` function takes 4 arguments and returns nothing. See the +example for how exactly each argument is used. + +To use this part handler, it must be included in a MIME multipart file as +part of the :ref:`user data`. +Since MIME parts are processed in order, a part handler part must precede +any parts with mime-types that it is expected to handle in the same user data. + +``Cloud-init`` will then call the ``handle_part`` function once before it +handles any parts, once per part received, and once after all parts have been +handled. These additional calls allow for initialisation or teardown before +or after receiving any parts. + +Example +======= + +.. literalinclude:: ../../../examples/part-handler.txt + :language: python + :linenos: diff --git a/doc/rtd/spelling_word_list.txt b/doc/rtd/spelling_word_list.txt index 239b3b49475..5f4783af65b 100644 --- a/doc/rtd/spelling_word_list.txt +++ b/doc/rtd/spelling_word_list.txt @@ -24,6 +24,7 @@ bigstep boolean bootcmd boothook +boothooks btrfs busybox byobu @@ -211,6 +212,7 @@ scaleway seedurl serverurl setup-keymap +shellscript shortid sigonly sk