From 7a0d957c2080ec5ee9aa99e48588a6467797776d Mon Sep 17 00:00:00 2001 From: James Falcon Date: Mon, 9 Sep 2024 08:37:30 -0500 Subject: [PATCH] chore: Prefer other methods over $INSTANCE_ID (#5661) $INSTANCE_ID is an environment variable that contains a reference to the current instance id. Its use is no longer needed now that we can use jinja templates with {{ v1.instance_id }}. Furthermore, `cloud-init-per` is a better replacement for running scripts once per instance. In particular, this commit does the following: - Add jinja templating functionality to boothooks to be consistent with other core user data types - Document cloud-id and cloud-init-per as they were previously undocumented - Replace all $INSTANCE_ID references in docs to either use {{ v1.instance_id }} or the `cloud-init-per` script - Update documentation of $INSTANCE_ID to now be deprecated - Update tests as necessary --- cloudinit/config/cc_phone_home.py | 4 +- cloudinit/stages.py | 10 ++- doc/examples/cloud-config-boot-cmds.txt | 1 - doc/examples/cloud-config.txt | 7 +-- doc/man/cloud-init-per.1 | 2 +- doc/module-docs/cc_bootcmd/data.yaml | 7 ++- doc/module-docs/cc_phone_home/data.yaml | 20 +++--- doc/module-docs/cc_phone_home/example1.yaml | 3 +- doc/module-docs/cc_phone_home/example2.yaml | 3 +- doc/rtd/explanation/format.rst | 34 +++++----- doc/rtd/explanation/instancedata.rst | 2 + doc/rtd/howto/module_run_frequency.rst | 3 +- doc/rtd/reference/cli.rst | 63 ++++++++++++++----- doc/userdata.txt | 14 ++--- tests/data/merge_sources/expected9.yaml | 3 +- tests/data/merge_sources/source9-1.yaml | 3 +- .../modules/test_boothook.py | 3 +- .../test_multi_part_user_data_handling.py | 2 +- tests/unittests/config/test_cc_bootcmd.py | 16 ----- 19 files changed, 120 insertions(+), 80 deletions(-) diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index b9dc22a4cfb..212ab52e7cc 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -35,12 +35,12 @@ LOG = logging.getLogger(__name__) # phone_home: -# url: http://my.foo.bar/$INSTANCE/ +# url: http://my.foo.bar/{{ v1.instance_id }}/ # post: all # tries: 10 # # phone_home: -# url: http://my.foo.bar/$INSTANCE_ID/ +# url: http://my.foo.bar/{{ v1.instance_id }}/ # post: [ pub_key_rsa, pub_key_ecdsa, instance_id, hostname, # fqdn ] # diff --git a/cloudinit/stages.py b/cloudinit/stages.py index b6394ffbd3c..ff0e336e80b 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -640,15 +640,21 @@ def _default_handlers(self, opts=None) -> List[handlers.Handler]: # TODO(harlowja) Hmmm, should we dynamically import these?? cloudconfig_handler = CloudConfigPartHandler(**opts) shellscript_handler = ShellScriptPartHandler(**opts) + boothook_handler = BootHookPartHandler(**opts) def_handlers = [ cloudconfig_handler, shellscript_handler, ShellScriptByFreqPartHandler(PER_ALWAYS, **opts), ShellScriptByFreqPartHandler(PER_INSTANCE, **opts), ShellScriptByFreqPartHandler(PER_ONCE, **opts), - BootHookPartHandler(**opts), + boothook_handler, JinjaTemplatePartHandler( - **opts, sub_handlers=[cloudconfig_handler, shellscript_handler] + **opts, + sub_handlers=[ + cloudconfig_handler, + shellscript_handler, + boothook_handler, + ], ), ] return def_handlers diff --git a/doc/examples/cloud-config-boot-cmds.txt b/doc/examples/cloud-config-boot-cmds.txt index 4eebe029bb5..f0954abd2a0 100644 --- a/doc/examples/cloud-config-boot-cmds.txt +++ b/doc/examples/cloud-config-boot-cmds.txt @@ -5,7 +5,6 @@ # This is very similar to runcmd, but commands run very early # in the boot process, only slightly after a 'boothook' would run. # - bootcmd will run on every boot -# - INSTANCE_ID variable will be set to the current instance ID # - 'cloud-init-per' command can be used to make bootcmd run exactly once bootcmd: - echo 192.168.1.130 us.archive.ubuntu.com >> /etc/hosts diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 2b85fa8dc66..88fb8393c94 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -117,7 +117,6 @@ runcmd: # This is very similar to runcmd above, but commands run very early # in the boot process, only slightly after a 'boothook' would run. # - bootcmd will run on every boot -# - INSTANCE_ID variable will be set to the current instance ID # - 'cloud-init-per' command can be used to make bootcmd run exactly once bootcmd: - echo 192.168.1.130 us.archive.ubuntu.com > /etc/hosts @@ -315,15 +314,15 @@ final_message: "The system is finally up, after $UPTIME seconds" # phone_home: if this dictionary is present, then the phone_home # cloud-config module will post specified data back to the given -# url +# url. Note that this example requires a `## template: jinja` header # default: none # phone_home: -# url: http://my.foo.bar/$INSTANCE/ +# url: http://my.foo.bar/{{ v1.instance_id }}/ # post: all # tries: 10 # phone_home: - url: http://my.example.com/$INSTANCE_ID/ + url: http://my.example.com/{{ v1.instance_id }}/ post: [ pub_key_rsa, pub_key_ecdsa, instance_id ] # timezone: set the timezone for this instance diff --git a/doc/man/cloud-init-per.1 b/doc/man/cloud-init-per.1 index 3668232e0b1..6b7530571e8 100644 --- a/doc/man/cloud-init-per.1 +++ b/doc/man/cloud-init-per.1 @@ -25,7 +25,7 @@ This can be one of the following values: run only once and do not re-run for new instance-id .BR "instance" ":" -run only the first boot for a given instance-id +run only once for a given instance-id and re-run for new instance-id .BR "always" ":" run every boot diff --git a/doc/module-docs/cc_bootcmd/data.yaml b/doc/module-docs/cc_bootcmd/data.yaml index 552e16887c9..0489114bd5a 100644 --- a/doc/module-docs/cc_bootcmd/data.yaml +++ b/doc/module-docs/cc_bootcmd/data.yaml @@ -2,8 +2,7 @@ cc_bootcmd: description: | This module runs arbitrary commands very early in the boot process, only slightly after a boothook would run. This is very similar to a boothook, - but more user friendly. The environment variable ``INSTANCE_ID`` will be - set to the current instance ID for all run commands. Commands can be + but more user friendly. Commands can be specified either as lists or strings. For invocation details, see ``runcmd``. @@ -14,6 +13,10 @@ cc_bootcmd: .. note:: When writing files, do not use ``/tmp`` dir as it races with ``systemd-tmpfiles-clean`` (LP: #1707222). Use ``/run/somedir`` instead. + .. warning:: + Use of ``INSTANCE_ID`` variable within this module is deprecated. + Use :ref:`jinja templates` with + :ref:`v1.instance_id` instead. examples: - comment: | Example 1: diff --git a/doc/module-docs/cc_phone_home/data.yaml b/doc/module-docs/cc_phone_home/data.yaml index f5af3f6bde3..2a34f7996fe 100644 --- a/doc/module-docs/cc_phone_home/data.yaml +++ b/doc/module-docs/cc_phone_home/data.yaml @@ -1,34 +1,38 @@ cc_phone_home: description: | This module can be used to post data to a remote host after boot is - complete. If the post URL contains the string ``$INSTANCE_ID`` it will be - replaced with the ID of the current instance. + complete. Either all data can be posted, or a list of keys to post. Available keys are: - + - ``pub_key_rsa`` - ``pub_key_ecdsa`` - ``pub_key_ed25519`` - ``instance_id`` - ``hostname`` - ``fdqn`` - + Data is sent as ``x-www-form-urlencoded`` arguments. - + **Example HTTP POST**: - + .. code-block:: http - + POST / HTTP/1.1 Content-Length: 1337 User-Agent: Cloud-Init/21.4 Accept-Encoding: gzip, deflate Accept: */* Content-Type: application/x-www-form-urlencoded - + pub_key_rsa=rsa_contents&pub_key_ecdsa=ecdsa_contents&pub_key_ed25519=ed25519_contents&instance_id=i-87018aed&hostname=myhost&fqdn=myhost.internal + + .. warning:: + Use of ``INSTANCE_ID`` variable within this module is deprecated. + Use :ref:`jinja templates` with + :ref:`v1.instance_id` instead. examples: - comment: | Example 1: diff --git a/doc/module-docs/cc_phone_home/example1.yaml b/doc/module-docs/cc_phone_home/example1.yaml index 1278f497eb5..2221f82ae78 100644 --- a/doc/module-docs/cc_phone_home/example1.yaml +++ b/doc/module-docs/cc_phone_home/example1.yaml @@ -1,2 +1,3 @@ +## template: jinja #cloud-config -phone_home: {post: all, url: 'http://example.com/$INSTANCE_ID/'} +phone_home: {post: all, url: 'http://example.com/{{ v1.instance_id }}/'} diff --git a/doc/module-docs/cc_phone_home/example2.yaml b/doc/module-docs/cc_phone_home/example2.yaml index fe9ec638f3b..eb6e8bd342c 100644 --- a/doc/module-docs/cc_phone_home/example2.yaml +++ b/doc/module-docs/cc_phone_home/example2.yaml @@ -1,5 +1,6 @@ +## template: jinja #cloud-config phone_home: post: [pub_key_rsa, pub_key_ecdsa, pub_key_ed25519, instance_id, hostname, fqdn] tries: 5 - url: http://example.com/$INSTANCE_ID/ + url: http://example.com/{{ v1.instance_id }}/ diff --git a/doc/rtd/explanation/format.rst b/doc/rtd/explanation/format.rst index 7d8a4a2176c..92f37df0132 100644 --- a/doc/rtd/explanation/format.rst +++ b/doc/rtd/explanation/format.rst @@ -88,9 +88,12 @@ Explanation A user data script is a single script to be executed once per instance. User data scripts are run relatively late in the boot process, during cloud-init's :ref:`final stage` as part of the -:ref:`cc_scripts_user` module. When run, -the environment variable ``INSTANCE_ID`` is set to the current instance ID -for use within the script. +:ref:`cc_scripts_user` module. + +.. warning:: + Use of ``INSTANCE_ID`` variable within user data scripts is deprecated. + Use :ref:`jinja templates` with + :ref:`v1.instance_id` instead. .. _user_data_formats-cloud_boothook: @@ -114,16 +117,10 @@ Example of once-per-instance script #cloud-boothook #!/bin/sh - PERSIST_ID=/var/lib/cloud/first-instance-id - _id="" - if [ -r $PERSIST_ID ]; then - _id=$(cat /var/lib/cloud/first-instance-id) - fi - - if [ -z $_id ] || [ $INSTANCE_ID != $_id ]; then - echo 192.168.1.130 us.archive.ubuntu.com >> /etc/hosts - fi - sudo echo $INSTANCE_ID > $PERSIST_ID + # Early exit 0 when script has already run for this instance-id, + # continue if new instance boot. + cloud-init-per instance do-hosts /bin/false && exit 0 + echo 192.168.1.130 us.archive.ubuntu.com >> /etc/hosts Explanation ----------- @@ -139,6 +136,11 @@ The boothook is different in that: before any cloud-init modules are run. * It is run on every boot +.. warning:: + Use of ``INSTANCE_ID`` variable within boothooks is deprecated. + Use :ref:`jinja templates` with + :ref:`v1.instance_id` instead. + Include file ============ @@ -159,6 +161,8 @@ be read and their content can be any kind of user data format, both base config and meta config. If an error occurs reading a file the remaining files will not be read. +.. _user_data_formats-jinja: + Jinja template ============== @@ -191,8 +195,8 @@ as jinja template variables. Any jinja templated configuration must contain the original header along with the new jinja header above it. .. note:: - Use of Jinja templates is ONLY supported for cloud-config and user data - scripts. Jinja templates are not supported for cloud-boothooks or + Use of Jinja templates is supported for cloud-config, user data + scripts, and cloud-boothooks. Jinja templates are not supported for meta configs. .. _user_data_formats-mime_archive: diff --git a/doc/rtd/explanation/instancedata.rst b/doc/rtd/explanation/instancedata.rst index 1196fcb3793..af1ffd5cce6 100644 --- a/doc/rtd/explanation/instancedata.rst +++ b/doc/rtd/explanation/instancedata.rst @@ -310,6 +310,8 @@ Example output: - sles, 12.3, x86_64 - ubuntu, 20.04, focal +.. _v1_instance_id: + ``v1.instance_id`` ^^^^^^^^^^^^^^^^^^ diff --git a/doc/rtd/howto/module_run_frequency.rst b/doc/rtd/howto/module_run_frequency.rst index 58b161c61ba..bbe872772f7 100644 --- a/doc/rtd/howto/module_run_frequency.rst +++ b/doc/rtd/howto/module_run_frequency.rst @@ -34,7 +34,8 @@ Then your user data could then be: .. code-block:: yaml + ## template: jinja #cloud-config phone_home: - url: http://example.com/$INSTANCE_ID/ + url: http://example.com/{{ v1.instance_id }}/ post: all diff --git a/doc/rtd/reference/cli.rst b/doc/rtd/reference/cli.rst index bdc59c2808a..704ca06805e 100644 --- a/doc/rtd/reference/cli.rst +++ b/doc/rtd/reference/cli.rst @@ -3,6 +3,16 @@ CLI commands ************ +Cloud-init ships multiple executables that are intended for user interaction. + +These executables include: + +- `cloud-init`_ +- `cloud-init-per`_ + +cloud-init +========== + For the latest list of subcommands and arguments use ``cloud-init``'s ``--help`` option. This can be used against ``cloud-init`` itself, or on any of its subcommands. @@ -44,7 +54,7 @@ The rest of this document will give an overview of each of the subcommands. .. _cli_analyze: :command:`analyze` -================== +------------------ Get detailed reports of where ``cloud-init`` spends its time during the boot process. For more complete reference see :ref:`analyze`. @@ -62,7 +72,7 @@ Possible subcommands include: .. _cli_clean: :command:`clean` -================ +---------------- Remove ``cloud-init`` artifacts from :file:`/var/lib/cloud` and config files (best effort) to simulate a clean instance. On reboot, ``cloud-init`` will @@ -89,7 +99,7 @@ re-run all stages as it did on first boot. .. _cli_collect_logs: :command:`collect-logs` -======================= +----------------------- Collect and tar ``cloud-init``-generated logs, data files, and system information for triage. This subcommand is integrated with apport. @@ -111,7 +121,7 @@ Logs collected include: .. _cli_devel: :command:`devel` -================ +---------------- Collection of development tools under active development. These tools will likely be promoted to top-level subcommands when stable. @@ -144,18 +154,18 @@ for debugging purposes. :command:`query` -^^^^^^^^^^^^^^^^ +---------------- Query if hotplug is enabled for a given subsystem. :command:`handle` -^^^^^^^^^^^^^^^^^ +----------------- Respond to newly added system devices by retrieving updated system metadata and bringing up/down the corresponding device. :command:`enable` -^^^^^^^^^^^^^^^^^ +----------------- Enable hotplug for a given subsystem. This is a last resort command for administrators to enable hotplug in running instances. The recommended @@ -165,7 +175,7 @@ datasource. .. _cli_features: :command:`features` -=================== +------------------- Print out each feature supported. If ``cloud-init`` does not have the :command:`features` subcommand, it also does not support any features @@ -186,7 +196,7 @@ Example output: .. _cli_init: :command:`init` (deprecated) -============================ +---------------------------- Generally run by OS init systems to execute ``cloud-init``'s stages: *init* and *init-local*. See :ref:`boot_stages` for more info. @@ -201,7 +211,7 @@ generally gated to run only once due to semaphores in .. _cli_modules: :command:`modules` (deprecated) -=============================== +------------------------------- Generally run by OS init systems to execute ``modules:config`` and ``modules:final`` boot stages. This executes cloud config :ref:`modules` @@ -231,7 +241,7 @@ run only once due to semaphores in :file:`/var/lib/cloud/`. .. _cli_query: :command:`query` -================ +---------------- Query standardised cloud instance metadata crawled by ``cloud-init`` and stored in :file:`/run/cloud-init/instance-data.json`. This is a convenience @@ -332,7 +342,7 @@ and region: .. _cli_schema: :command:`schema` -================= +----------------- Validate cloud-config files using jsonschema. @@ -359,7 +369,7 @@ errors on :file:`stdout`. .. _cli_single: :command:`single` -================= +----------------- Attempt to run a single, named, cloud config module. @@ -384,7 +394,7 @@ module default frequency of ``instance``: .. _cli_status: :command:`status` -================= +----------------- Report cloud-init's current status. @@ -487,4 +497,29 @@ Which would produce the following example output: "status": "done" } +.. _cloud-init-per: + +cloud-init-per +============== + +``cloud-init-per`` will run a command with arguments at a specific frequency. + +For example, with the following command: + +.. code-block:: shell-session + + $ cloud-init-per once hello bash -c 'echo "Hello, world!" >> /tmp/hello' + +You will find 'Hello, world!' in the file :file:`/tmp/hello`. + +If we run the same command again, it will not run, as it has already run once. +:file:`/tmp/hello` still only contains one line rather than two. + +See the +`cloud-init-per man page `_ +for more details. + + + + .. _More details on machine-id: https://www.freedesktop.org/software/systemd/man/machine-id.html diff --git a/doc/userdata.txt b/doc/userdata.txt index e44a4952bc8..c13a418ee07 100644 --- a/doc/userdata.txt +++ b/doc/userdata.txt @@ -1,11 +1,11 @@ === Overview === Userdata is data provided by the entity that launches an instance. The cloud provider makes this data available to the instance via in one -way or another. +way or another. In EC2, the data is provided by the user via the '--user-data' or 'user-data-file' argument to ec2-run-instances. The EC2 cloud makes the -data available to the instance via its meta-data service at +data available to the instance via its meta-data service at http://169.254.169.254/latest/user-data cloud-init can read this input and act on it in different ways. @@ -15,7 +15,7 @@ cloud-init will download and cache to filesystem any user-data that it finds. However, certain types of user-data are handled specially. * Gzip Compressed Content - content found to be gzip compressed will be uncompressed, and + content found to be gzip compressed will be uncompressed, and these rules applied to the uncompressed data * Mime Multi Part archive @@ -52,17 +52,15 @@ finds. However, certain types of user-data are handled specially. This content is "cloud-config" data. See the examples for a commented example of supported config formats. - * Cloud Boothook + * Cloud Boothook begins with #cloud-boothook or Content-Type: text/cloud-boothook This content is "boothook" data. It is stored in a file under - /var/lib/cloud and then executed immediately. + ``/var/lib/cloud`` and then executed immediately. This is the earliest "hook" available. Note, that there is no mechanism provided for running only once. The boothook must take - care of this itself. It is provided with the instance id in the - environment variable "INSTANCE_ID". This could be made use of to - provide a 'once-per-instance' + care of this itself. === Examples === There are examples in the examples subdirectory. diff --git a/tests/data/merge_sources/expected9.yaml b/tests/data/merge_sources/expected9.yaml index 912276b46df..fa5a9dd1076 100644 --- a/tests/data/merge_sources/expected9.yaml +++ b/tests/data/merge_sources/expected9.yaml @@ -1,5 +1,6 @@ +## template: jinja #cloud-config phone_home: - url: http://my.example.com/$INSTANCE_ID/$BLAH_BLAH + url: http://my.example.com/{{ v1.instance_id }}/$BLAH_BLAH post: [ pub_key_rsa, pub_key_ecdsa, instance_id ] diff --git a/tests/data/merge_sources/source9-1.yaml b/tests/data/merge_sources/source9-1.yaml index d8dc9f17464..0b6401108a4 100644 --- a/tests/data/merge_sources/source9-1.yaml +++ b/tests/data/merge_sources/source9-1.yaml @@ -1,5 +1,6 @@ +## template: jinja #cloud-config phone_home: - url: http://my.example.com/$INSTANCE_ID/ + url: http://my.example.com/{{ v1.instance_id }}/ post: [ pub_key_rsa, pub_key_ecdsa, instance_id ] diff --git a/tests/integration_tests/modules/test_boothook.py b/tests/integration_tests/modules/test_boothook.py index e2a289b4f29..57effdb41c5 100644 --- a/tests/integration_tests/modules/test_boothook.py +++ b/tests/integration_tests/modules/test_boothook.py @@ -7,12 +7,13 @@ from tests.integration_tests.util import verify_clean_log USER_DATA = """\ +## template: jinja #cloud-boothook #!/bin/sh # Error below will generate stderr BOOTHOOK/0 echo BOOTHOOKstdout -echo "BOOTHOOK: $INSTANCE_ID: is called every boot." >> /boothook.txt +echo "BOOTHOOK: {{ v1.instance_id }}: is called every boot." >> /boothook.txt """ diff --git a/tests/integration_tests/test_multi_part_user_data_handling.py b/tests/integration_tests/test_multi_part_user_data_handling.py index a81c9fd9aea..0a0233eb868 100644 --- a/tests/integration_tests/test_multi_part_user_data_handling.py +++ b/tests/integration_tests/test_multi_part_user_data_handling.py @@ -73,7 +73,7 @@ - type: 'text/cloud-config' content: | bootcmd: - - [sh, -c, 'echo "BOOTCMD: $(date -R): $INSTANCE_ID" | tee /run/bootcmd.txt'] + - [sh, -c, 'echo "BOOTCMD: $(date -R): from cloud-config" | tee /run/bootcmd.txt'] """ # noqa: E501 diff --git a/tests/unittests/config/test_cc_bootcmd.py b/tests/unittests/config/test_cc_bootcmd.py index 507c1063e44..29292405e54 100644 --- a/tests/unittests/config/test_cc_bootcmd.py +++ b/tests/unittests/config/test_cc_bootcmd.py @@ -84,22 +84,6 @@ def test_handler_invalid_command_set(self): str(context_manager.exception), ) - def test_handler_creates_and_runs_bootcmd_script_with_instance_id(self): - """Valid schema runs a bootcmd script with INSTANCE_ID in the env.""" - cc = get_cloud() - out_file = self.tmp_path("bootcmd.out", self.new_root) - my_id = "b6ea0f59-e27d-49c6-9f87-79f19765a425" - valid_config = { - "bootcmd": ["echo {0} $INSTANCE_ID > {1}".format(my_id, out_file)] - } - - with mock.patch(self._etmpfile_path, FakeExtendedTempFile): - with self.allow_subp(["/bin/sh"]): - handle("cc_bootcmd", valid_config, cc, []) - self.assertEqual( - my_id + " iid-datasource-none\n", util.load_text_file(out_file) - ) - def test_handler_runs_bootcmd_script_with_error(self): """When a valid script generates an error, that error is raised.""" cc = get_cloud()